├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ └── open_an_issue.md ├── config.yml ├── dependabot.yml └── workflows │ ├── generated-pr.yml │ ├── js-test-and-release.yml │ ├── semantic-pull-request.yml │ └── stale.yml ├── .gitignore ├── .npmrc ├── CHANGELOG.md ├── FUNDING.json ├── LICENSE ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── package.json ├── packages ├── blockstore-core │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ ├── base.ts │ │ ├── black-hole.ts │ │ ├── identity.ts │ │ ├── index.ts │ │ ├── memory.ts │ │ └── tiered.ts │ ├── test │ │ ├── identity.spec.ts │ │ ├── memory.spec.ts │ │ └── tiered.spec.ts │ ├── tsconfig.json │ └── typedoc.json ├── blockstore-fs │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── benchmarks │ │ └── encoding │ │ │ ├── package.json │ │ │ ├── src │ │ │ ├── README.md │ │ │ └── index.ts │ │ │ └── tsconfig.json │ ├── package.json │ ├── src │ │ ├── index.ts │ │ └── sharding.ts │ ├── test │ │ ├── fixtures │ │ │ └── writer-worker.ts │ │ ├── index.spec.ts │ │ └── sharding.spec.ts │ ├── tsconfig.json │ └── typedoc.json ├── blockstore-idb │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── test │ │ └── index.spec.ts │ ├── tsconfig.json │ └── typedoc.json ├── blockstore-level │ ├── .aegir.js │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── test │ │ ├── fixtures │ │ │ └── test-level-iterator-destroy.ts │ │ ├── index.spec.ts │ │ └── node.ts │ ├── tsconfig.json │ └── typedoc.json ├── blockstore-s3 │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── examples │ │ └── helia │ │ │ ├── README.md │ │ │ ├── index.js │ │ │ └── package.json │ ├── package.json │ ├── src │ │ ├── index.ts │ │ └── sharding.ts │ ├── test │ │ ├── index.spec.ts │ │ └── utils │ │ │ └── s3-mock.ts │ ├── tsconfig.json │ └── typedoc.json ├── datastore-core │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ ├── base.ts │ │ ├── black-hole.ts │ │ ├── index.ts │ │ ├── keytransform.ts │ │ ├── memory.ts │ │ ├── mount.ts │ │ ├── namespace.ts │ │ ├── shard.ts │ │ ├── sharding.ts │ │ ├── tiered.ts │ │ └── utils.ts │ ├── test │ │ ├── keytransform.spec.ts │ │ ├── memory.spec.ts │ │ ├── mount.spec.ts │ │ ├── namespace.spec.ts │ │ ├── shard.spec.ts │ │ ├── sharding.spec.ts │ │ ├── tiered.spec.ts │ │ └── utils.spec.ts │ ├── tsconfig.json │ └── typedoc.json ├── datastore-fs │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── test │ │ ├── fixtures │ │ │ └── writer-worker.ts │ │ └── index.spec.ts │ ├── tsconfig.json │ └── typedoc.json ├── datastore-idb │ ├── .aegir.js │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── benchmark │ │ └── datastore-level │ │ │ ├── package.json │ │ │ ├── src │ │ │ └── index.ts │ │ │ └── tsconfig.json │ ├── package.json │ ├── src │ │ └── index.ts │ ├── test │ │ └── index.spec.ts │ ├── tsconfig.json │ └── typedoc.json ├── datastore-level │ ├── .aegir.js │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── test │ │ ├── browser.ts │ │ ├── fixtures │ │ │ └── test-level-iterator-destroy.ts │ │ ├── index.spec.ts │ │ └── node.ts │ ├── tsconfig.json │ └── typedoc.json ├── datastore-s3 │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── examples │ │ └── helia │ │ │ ├── README.md │ │ │ ├── index.js │ │ │ └── package.json │ ├── package.json │ ├── src │ │ └── index.ts │ ├── test │ │ ├── index.spec.ts │ │ └── utils │ │ │ └── s3-mock.ts │ ├── tsconfig.json │ └── typedoc.json ├── interface-blockstore-tests │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── tsconfig.json │ └── typedoc.json ├── interface-blockstore │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── tsconfig.json │ └── typedoc.json ├── interface-datastore-tests │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ └── index.ts │ ├── tsconfig.json │ └── typedoc.json ├── interface-datastore │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ │ ├── index.ts │ │ └── key.ts │ ├── test │ │ └── key.spec.ts │ ├── tsconfig.json │ └── typedoc.json └── interface-store │ ├── CHANGELOG.md │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE-APACHE │ ├── LICENSE-MIT │ ├── README.md │ ├── package.json │ ├── src │ ├── errors.ts │ └── index.ts │ ├── tsconfig.json │ └── typedoc.json └── typedoc.json /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Getting Help on IPFS 4 | url: https://ipfs.io/help 5 | about: All information about how and where to get help on IPFS. 6 | - name: IPFS Official Forum 7 | url: https://discuss.ipfs.io 8 | about: Please post general questions, support requests, and discussions here. 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/open_an_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Open an issue 3 | about: Only for actionable issues relevant to this repository. 4 | title: '' 5 | labels: need/triage 6 | assignees: '' 7 | 8 | --- 9 | 20 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for welcome - https://github.com/behaviorbot/welcome 2 | 3 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 4 | # Comment to be posted to on first time issues 5 | newIssueWelcomeComment: > 6 | Thank you for submitting your first issue to this repository! A maintainer 7 | will be here shortly to triage and review. 8 | 9 | In the meantime, please double-check that you have provided all the 10 | necessary information to make this process easy! Any information that can 11 | help save additional round trips is useful! We currently aim to give 12 | initial feedback within **two business days**. If this does not happen, feel 13 | free to leave a comment. 14 | 15 | Please keep an eye on how this issue will be labeled, as labels give an 16 | overview of priorities, assignments and additional actions requested by the 17 | maintainers: 18 | 19 | - "Priority" labels will show how urgent this is for the team. 20 | - "Status" labels will show if this is ready to be worked on, blocked, or in progress. 21 | - "Need" labels will indicate if additional input or analysis is required. 22 | 23 | Finally, remember to use https://discuss.ipfs.io if you just need general 24 | support. 25 | 26 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 27 | # Comment to be posted to on PRs from first time contributors in your repository 28 | newPRWelcomeComment: > 29 | Thank you for submitting this PR! 30 | 31 | A maintainer will be here shortly to review it. 32 | 33 | We are super grateful, but we are also overloaded! Help us by making sure 34 | that: 35 | 36 | * The context for this PR is clear, with relevant discussion, decisions 37 | and stakeholders linked/mentioned. 38 | 39 | * Your contribution itself is clear (code comments, self-review for the 40 | rest) and in its best form. Follow the [code contribution 41 | guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines) 42 | if they apply. 43 | 44 | Getting other community members to do a review would be great help too on 45 | complex PRs (you can ask in the chats/forums). If you are unsure about 46 | something, just leave us a comment. 47 | 48 | Next steps: 49 | 50 | * A maintainer will triage and assign priority to this PR, commenting on 51 | any missing things and potentially assigning a reviewer for high 52 | priority items. 53 | 54 | * The PR gets reviews, discussed and approvals as needed. 55 | 56 | * The PR is merged by maintainers when it has been approved and comments addressed. 57 | 58 | We currently aim to provide initial feedback/triaging within **two business 59 | days**. Please keep an eye on any labelling actions, as these will indicate 60 | priorities and status of your contribution. 61 | 62 | We are very grateful for your contribution! 63 | 64 | 65 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 66 | # Comment to be posted to on pull requests merged by a first time user 67 | # Currently disabled 68 | #firstPRMergeComment: "" 69 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directories: 5 | - "/" 6 | schedule: 7 | interval: daily 8 | time: "10:00" 9 | open-pull-requests-limit: 20 10 | commit-message: 11 | prefix: "deps" 12 | prefix-development: "chore" 13 | groups: 14 | interplanetary-deps: # Helia/libp2p deps 15 | patterns: 16 | - "*helia*" 17 | - "*libp2p*" 18 | - "*multiformats*" 19 | - "*blockstore*" 20 | - "*datastore*" 21 | kubo-deps: # kubo deps 22 | patterns: 23 | - "*kubo*" 24 | - "ipfsd-ctl" 25 | - package-ecosystem: "github-actions" 26 | directory: "/" 27 | schedule: 28 | interval: "weekly" 29 | commit-message: 30 | prefix: chore 31 | -------------------------------------------------------------------------------- /.github/workflows/generated-pr.yml: -------------------------------------------------------------------------------- 1 | name: Close Generated PRs 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 15 | -------------------------------------------------------------------------------- /.github/workflows/js-test-and-release.yml: -------------------------------------------------------------------------------- 1 | name: test & maybe release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | workflow_dispatch: 9 | 10 | permissions: 11 | contents: write 12 | id-token: write 13 | packages: write 14 | pull-requests: write 15 | 16 | concurrency: 17 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 18 | cancel-in-progress: true 19 | 20 | jobs: 21 | js-test-and-release: 22 | uses: ipdxco/unified-github-workflows/.github/workflows/js-test-and-release.yml@v1.0 23 | secrets: 24 | DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} 25 | DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} 26 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 27 | UCI_GITHUB_TOKEN: ${{ secrets.UCI_GITHUB_TOKEN }} 28 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 29 | -------------------------------------------------------------------------------- /.github/workflows/semantic-pull-request.yml: -------------------------------------------------------------------------------- 1 | name: Semantic PR 2 | 3 | on: 4 | pull_request_target: 5 | types: 6 | - opened 7 | - edited 8 | - synchronize 9 | 10 | jobs: 11 | main: 12 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-semantic-pull-request.yml@v1 13 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close and mark stale issue 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | dist 4 | .docs 5 | .coverage 6 | node_modules 7 | package-lock.json 8 | yarn.lock 9 | .vscode 10 | .tmp-compiled-docs 11 | tsconfig-doc-check.aegir.json 12 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | ; package-lock with tarball deps breaks lerna/nx - remove when https://github.com/semantic-release/github/pull/487 is merged 2 | package-lock=false 3 | -------------------------------------------------------------------------------- /FUNDING.json: -------------------------------------------------------------------------------- 1 | { 2 | "opRetro": { 3 | "projectId": "0x7f330267969cf845a983a9d4e7b7dbcca5c700a5191269af377836d109e0bb69" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This project is dual licensed under MIT and Apache-2.0. 2 | 3 | MIT: https://www.opensource.org/licenses/mit 4 | Apache-2.0: https://www.apache.org/licenses/license-2.0 5 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 2 | 3 | http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 6 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # stores 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Blockstores and datastores used by IP-JS internals 9 | 10 | # Packages 11 | 12 | - [`packages/blockstore-core`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-core) Contains various implementations of the API contract described in interface-blockstore 13 | - [`packages/blockstore-fs`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-fs) Blockstore implementation with file system backend 14 | - [`packages/blockstore-idb`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-idb) Blockstore implementation with IndexedDB backend 15 | - [`packages/blockstore-level`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-level) Blockstore implementation with level(up|down) backend 16 | - [`packages/blockstore-s3`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-s3) IPFS blockstore implementation backed by s3 17 | - [`packages/datastore-core`](https://github.com/ipfs/js-stores/tree/main/packages/datastore-core) Wrapper implementation for interface-datastore 18 | - [`packages/datastore-fs`](https://github.com/ipfs/js-stores/tree/main/packages/datastore-fs) Datastore implementation with file system backend 19 | - [`packages/datastore-idb`](https://github.com/ipfs/js-stores/tree/main/packages/datastore-idb) Datastore implementation with IndexedDB backend. 20 | - [`packages/datastore-level`](https://github.com/ipfs/js-stores/tree/main/packages/datastore-level) Datastore implementation with level(up|down) backend 21 | - [`packages/datastore-s3`](https://github.com/ipfs/js-stores/tree/main/packages/datastore-s3) IPFS datastore implementation backed by s3 22 | - [`packages/interface-blockstore`](https://github.com/ipfs/js-stores/tree/main/packages/interface-blockstore) An interface for storing and retrieving blocks 23 | - [`packages/interface-blockstore-tests`](https://github.com/ipfs/js-stores/tree/main/packages/interface-blockstore-tests) Compliance tests for the blockstore interface 24 | - [`packages/interface-datastore`](https://github.com/ipfs/js-stores/tree/main/packages/interface-datastore) datastore interface 25 | - [`packages/interface-datastore-tests`](https://github.com/ipfs/js-stores/tree/main/packages/interface-datastore-tests) Compliance tests for the datastore interface 26 | - [`packages/interface-store`](https://github.com/ipfs/js-stores/tree/main/packages/interface-store) A generic interface for storing and retrieving data 27 | 28 | # API Docs 29 | 30 | - 31 | 32 | # License 33 | 34 | Licensed under either of 35 | 36 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/LICENSE-APACHE) / ) 37 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/LICENSE-MIT) / ) 38 | 39 | # Contribute 40 | 41 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 42 | 43 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 44 | 45 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 46 | 47 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 48 | 49 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 50 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stores", 3 | "version": "1.0.0", 4 | "description": "Blockstores and datastores used by IP-JS internals", 5 | "license": "Apache-2.0 OR MIT", 6 | "homepage": "https://github.com/ipfs/js-stores#readme", 7 | "repository": { 8 | "type": "git", 9 | "url": "git+https://github.com/ipfs/js-stores.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/ipfs/js-stores/issues" 13 | }, 14 | "private": true, 15 | "scripts": { 16 | "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", 17 | "test": "aegir run test", 18 | "test:node": "aegir run test:node", 19 | "test:chrome": "aegir run test:chrome", 20 | "test:chrome-webworker": "aegir run test:chrome-webworker", 21 | "test:firefox": "aegir run test:firefox", 22 | "test:firefox-webworker": "aegir run test:firefox-webworker", 23 | "test:electron-main": "aegir run test:electron-main", 24 | "test:electron-renderer": "aegir run test:electron-renderer", 25 | "clean": "aegir run clean", 26 | "generate": "aegir run generate", 27 | "build": "aegir run build", 28 | "lint": "aegir run lint", 29 | "dep-check": "aegir run dep-check", 30 | "release": "run-s build docs:no-publish npm:release docs", 31 | "npm:release": "aegir run release --concurrency 1", 32 | "docs": "aegir docs", 33 | "docs:no-publish": "aegir docs --publish false" 34 | }, 35 | "devDependencies": { 36 | "aegir": "^47.0.16", 37 | "npm-run-all": "^4.1.5" 38 | }, 39 | "workspaces": [ 40 | "packages/*" 41 | ] 42 | } 43 | -------------------------------------------------------------------------------- /packages/blockstore-core/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/blockstore-core/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/blockstore-core/src/base.ts: -------------------------------------------------------------------------------- 1 | import type { Blockstore, Pair } from 'interface-blockstore' 2 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 3 | import type { CID } from 'multiformats/cid' 4 | 5 | export class BaseBlockstore implements Blockstore { 6 | has (key: CID, options?: AbortOptions): Await { 7 | return Promise.reject(new Error('.has is not implemented')) 8 | } 9 | 10 | put (key: CID, val: Uint8Array, options?: AbortOptions): Await { 11 | return Promise.reject(new Error('.put is not implemented')) 12 | } 13 | 14 | async * putMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { 15 | for await (const { cid, block } of source) { 16 | await this.put(cid, block, options) 17 | yield cid 18 | } 19 | } 20 | 21 | get (key: CID, options?: AbortOptions): Await { 22 | return Promise.reject(new Error('.get is not implemented')) 23 | } 24 | 25 | async * getMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { 26 | for await (const key of source) { 27 | yield { 28 | cid: key, 29 | block: await this.get(key, options) 30 | } 31 | } 32 | } 33 | 34 | delete (key: CID, options?: AbortOptions): Await { 35 | return Promise.reject(new Error('.delete is not implemented')) 36 | } 37 | 38 | async * deleteMany (source: AwaitIterable, options?: AbortOptions): AwaitIterable { 39 | for await (const key of source) { 40 | await this.delete(key, options) 41 | yield key 42 | } 43 | } 44 | 45 | /** 46 | * Extending classes should override `query` or implement this method 47 | */ 48 | async * getAll (options?: AbortOptions): AwaitIterable { // eslint-disable-line require-yield 49 | throw new Error('.getAll is not implemented') 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /packages/blockstore-core/src/black-hole.ts: -------------------------------------------------------------------------------- 1 | import { NotFoundError } from 'interface-store' 2 | import { BaseBlockstore } from './base.js' 3 | import type { Pair } from 'interface-blockstore' 4 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 5 | import type { CID } from 'multiformats/cid' 6 | 7 | export class BlackHoleBlockstore extends BaseBlockstore { 8 | put (key: CID, value: Uint8Array, options?: AbortOptions): Await { 9 | options?.signal?.throwIfAborted() 10 | return key 11 | } 12 | 13 | get (key: CID, options?: AbortOptions): Await { 14 | options?.signal?.throwIfAborted() 15 | throw new NotFoundError() 16 | } 17 | 18 | has (key: CID, options?: AbortOptions): Await { 19 | options?.signal?.throwIfAborted() 20 | return false 21 | } 22 | 23 | async delete (cid: CID, options?: AbortOptions): Promise { 24 | options?.signal?.throwIfAborted() 25 | } 26 | 27 | // eslint-disable-next-line require-yield 28 | async * getAll (options?: AbortOptions): AwaitIterable { 29 | options?.signal?.throwIfAborted() 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /packages/blockstore-core/src/identity.ts: -------------------------------------------------------------------------------- 1 | import { NotFoundError } from 'interface-store' 2 | import { BaseBlockstore } from './base.js' 3 | import type { Blockstore, Pair } from 'interface-blockstore' 4 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 5 | import type { CID } from 'multiformats/cid' 6 | 7 | // https://github.com/multiformats/multicodec/blob/d06fc6194710e8909bac64273c43f16b56ca4c34/table.csv#L2 8 | const IDENTITY_CODEC = 0x00 9 | 10 | export class IdentityBlockstore extends BaseBlockstore { 11 | private readonly child?: Blockstore 12 | 13 | constructor (child?: Blockstore) { 14 | super() 15 | 16 | this.child = child 17 | } 18 | 19 | put (key: CID, block: Uint8Array, options?: AbortOptions): Await { 20 | if (key.multihash.code === IDENTITY_CODEC) { 21 | options?.signal?.throwIfAborted() 22 | return key 23 | } 24 | 25 | if (this.child == null) { 26 | options?.signal?.throwIfAborted() 27 | return key 28 | } 29 | 30 | return this.child.put(key, block, options) 31 | } 32 | 33 | get (key: CID, options?: AbortOptions): Await { 34 | if (key.multihash.code === IDENTITY_CODEC) { 35 | options?.signal?.throwIfAborted() 36 | return key.multihash.digest 37 | } 38 | 39 | if (this.child == null) { 40 | options?.signal?.throwIfAborted() 41 | throw new NotFoundError() 42 | } 43 | 44 | return this.child.get(key, options) 45 | } 46 | 47 | has (key: CID, options?: AbortOptions): Await { 48 | if (key.multihash.code === IDENTITY_CODEC) { 49 | options?.signal?.throwIfAborted() 50 | return true 51 | } 52 | 53 | if (this.child == null) { 54 | options?.signal?.throwIfAborted() 55 | return false 56 | } 57 | 58 | return this.child.has(key, options) 59 | } 60 | 61 | delete (key: CID, options?: AbortOptions): Await { 62 | if (key.code === IDENTITY_CODEC) { 63 | options?.signal?.throwIfAborted() 64 | return 65 | } 66 | 67 | if (this.child != null) { 68 | return this.child.delete(key, options) 69 | } 70 | } 71 | 72 | getAll (options?: AbortOptions): AwaitIterable { 73 | if (this.child != null) { 74 | return this.child.getAll(options) 75 | } 76 | 77 | options?.signal?.throwIfAborted() 78 | return [] 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /packages/blockstore-core/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @packageDocumentation 3 | * 4 | * Various Blockstore implementations are available. 5 | * 6 | * ## Implementations 7 | * 8 | * - Base: [`src/base`](./src/base.ts) 9 | * - Memory: [`src/memory`](./src/memory.ts) 10 | * - BlackHole: ['src/black-hole](./src/black-hole.ts) 11 | * - Tiered: ['src/tiered](./src/tiered.ts) 12 | * 13 | * @example BaseBlockstore 14 | * 15 | * Provides a complete implementation of the Blockstore interface. You must implement `.get`, `.put`, etc. 16 | * 17 | * ```js 18 | * import { BaseBlockstore } from 'blockstore-core/base' 19 | * 20 | * class MyCustomBlockstore extends BaseBlockstore { 21 | * put (key, val, options) { 22 | * // store a block 23 | * } 24 | * 25 | * get (key, options) { 26 | * // retrieve a block 27 | * } 28 | * 29 | * // ...etc 30 | * } 31 | * ``` 32 | * 33 | * @example MemoryBlockstore 34 | * 35 | * A simple Blockstore that stores blocks in memory. 36 | * 37 | * ```js 38 | * import { MemoryBlockstore } from 'blockstore-core/memory' 39 | * 40 | * const store = new MemoryBlockstore() 41 | * ``` 42 | * 43 | * @example BlackHoleBlockstore 44 | * 45 | * A Blockstore that does not store any blocks. 46 | * 47 | * ```js 48 | * import { BlackHoleBlockstore } from 'blockstore-core/black-hole' 49 | * 50 | * const store = new BlackHoleBlockstore() 51 | * ``` 52 | * 53 | * @example TieredBlockstore 54 | * 55 | * A tiered blockstore wraps one or more blockstores and will query each in parallel to retrieve a block - the operation will succeed if any wrapped store has the block. 56 | * 57 | * Writes are invoked on all wrapped blockstores. 58 | * 59 | * ```js 60 | * import { TieredBlockstore } from 'blockstore-core/tiered' 61 | * 62 | * const store = new TieredBlockstore([ 63 | * store1, 64 | * store2, 65 | * // ...etc 66 | * ]) 67 | * ``` 68 | * 69 | * @example IdentityBlockstore 70 | * 71 | * An identity blockstore is one that deals exclusively in Identity CIDs - this is a special CID with the codec [0x00](https://github.com/multiformats/multicodec/blob/d06fc6194710e8909bac64273c43f16b56ca4c34/table.csv#L2) where the multihash digest is the data that makes up the block. 72 | * 73 | * ```TypeScript 74 | * import { IdentityBlockstore } from 'blockstore-core/identity' 75 | * import { CID } from 'multiformats/cid' 76 | * 77 | * const blockstore = new IdentityBlockstore() 78 | * 79 | * blockstore.has(CID.parse('QmFoo')) // false 80 | * 81 | * blockstore.has(CID.parse('bafkqac3imvwgy3zao5xxe3de')) // true 82 | * ``` 83 | */ 84 | 85 | export { BaseBlockstore } from './base.js' 86 | export { MemoryBlockstore } from './memory.js' 87 | export { BlackHoleBlockstore } from './black-hole.js' 88 | export { TieredBlockstore } from './tiered.js' 89 | -------------------------------------------------------------------------------- /packages/blockstore-core/src/memory.ts: -------------------------------------------------------------------------------- 1 | import { NotFoundError } from 'interface-store' 2 | import { base32 } from 'multiformats/bases/base32' 3 | import { CID } from 'multiformats/cid' 4 | import * as raw from 'multiformats/codecs/raw' 5 | import * as Digest from 'multiformats/hashes/digest' 6 | import { BaseBlockstore } from './base.js' 7 | import type { Pair } from 'interface-blockstore' 8 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 9 | 10 | export class MemoryBlockstore extends BaseBlockstore { 11 | private readonly data: Map 12 | 13 | constructor () { 14 | super() 15 | 16 | this.data = new Map() 17 | } 18 | 19 | put (key: CID, val: Uint8Array, options?: AbortOptions): Await { 20 | options?.signal?.throwIfAborted() 21 | this.data.set(base32.encode(key.multihash.bytes), val) 22 | 23 | return key 24 | } 25 | 26 | get (key: CID, options?: AbortOptions): Await { 27 | options?.signal?.throwIfAborted() 28 | const buf = this.data.get(base32.encode(key.multihash.bytes)) 29 | 30 | if (buf == null) { 31 | throw new NotFoundError() 32 | } 33 | 34 | return buf 35 | } 36 | 37 | has (key: CID, options?: AbortOptions): Await { 38 | options?.signal?.throwIfAborted() 39 | return this.data.has(base32.encode(key.multihash.bytes)) 40 | } 41 | 42 | async delete (key: CID, options?: AbortOptions): Promise { 43 | options?.signal?.throwIfAborted() 44 | this.data.delete(base32.encode(key.multihash.bytes)) 45 | } 46 | 47 | async * getAll (options?: AbortOptions): AwaitIterable { 48 | options?.signal?.throwIfAborted() 49 | 50 | for (const [key, value] of this.data.entries()) { 51 | yield { 52 | cid: CID.createV1(raw.code, Digest.decode(base32.decode(key))), 53 | block: value 54 | } 55 | options?.signal?.throwIfAborted() 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /packages/blockstore-core/src/tiered.ts: -------------------------------------------------------------------------------- 1 | import { logger } from '@libp2p/logger' 2 | import { NotFoundError } from 'interface-store' 3 | import filter from 'it-filter' 4 | import merge from 'it-merge' 5 | import { BaseBlockstore } from './base.js' 6 | import type { Blockstore, Pair } from 'interface-blockstore' 7 | import type { AbortOptions, AwaitIterable } from 'interface-store' 8 | import type { CID } from 'multiformats/cid' 9 | 10 | const log = logger('blockstore:core:tiered') 11 | 12 | /** 13 | * A blockstore that can combine multiple stores. Puts and deletes 14 | * will write through to all blockstores. Has and get will 15 | * try each store sequentially. getAll will use every store but also 16 | * deduplicate any yielded pairs. 17 | */ 18 | export class TieredBlockstore extends BaseBlockstore { 19 | private readonly stores: Blockstore[] 20 | 21 | constructor (stores: Blockstore[]) { 22 | super() 23 | 24 | this.stores = stores.slice() 25 | } 26 | 27 | async put (key: CID, value: Uint8Array, options?: AbortOptions): Promise { 28 | await Promise.all( 29 | this.stores.map(async store => { 30 | await store.put(key, value, options) 31 | }) 32 | ) 33 | 34 | return key 35 | } 36 | 37 | async get (key: CID, options?: AbortOptions): Promise { 38 | let error: Error | undefined 39 | 40 | for (const store of this.stores) { 41 | try { 42 | const res = await store.get(key, options) 43 | 44 | if (res != null) { 45 | return res 46 | } 47 | } catch (err: any) { 48 | error = err 49 | log.error(err) 50 | } 51 | } 52 | 53 | throw error ?? new NotFoundError() 54 | } 55 | 56 | async has (key: CID, options?: AbortOptions): Promise { 57 | for (const s of this.stores) { 58 | if (await s.has(key, options)) { 59 | return true 60 | } 61 | } 62 | 63 | return false 64 | } 65 | 66 | async delete (key: CID, options?: AbortOptions): Promise { 67 | await Promise.all( 68 | this.stores.map(async store => { 69 | await store.delete(key, options) 70 | }) 71 | ) 72 | } 73 | 74 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 75 | for await (const pair of source) { 76 | await this.put(pair.cid, pair.block, options) 77 | yield pair.cid 78 | } 79 | } 80 | 81 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 82 | for await (const cid of source) { 83 | await this.delete(cid, options) 84 | yield cid 85 | } 86 | } 87 | 88 | async * getAll (options?: AbortOptions): AwaitIterable { 89 | // deduplicate yielded pairs 90 | const seen = new Set() 91 | 92 | yield * filter(merge(...this.stores.map(s => s.getAll(options))), (pair) => { 93 | const cidStr = pair.cid.toString() 94 | 95 | if (seen.has(cidStr)) { 96 | return false 97 | } 98 | 99 | seen.add(cidStr) 100 | 101 | return true 102 | }) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /packages/blockstore-core/test/identity.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import all from 'it-all' 5 | import drain from 'it-drain' 6 | import { CID } from 'multiformats/cid' 7 | import * as raw from 'multiformats/codecs/raw' 8 | import { identity } from 'multiformats/hashes/identity' 9 | import { sha256 } from 'multiformats/hashes/sha2' 10 | import { IdentityBlockstore } from '../src/identity.js' 11 | import { MemoryBlockstore } from '../src/memory.js' 12 | import type { Blockstore } from 'interface-blockstore' 13 | 14 | describe('identity', () => { 15 | let blockstore: Blockstore 16 | let child: Blockstore 17 | 18 | beforeEach(() => { 19 | blockstore = new IdentityBlockstore() 20 | child = new MemoryBlockstore() 21 | }) 22 | 23 | it('has an identity CID', () => { 24 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 25 | const multihash = identity.digest(block) 26 | const cid = CID.createV1(identity.code, multihash) 27 | 28 | expect(blockstore.has(cid)).to.be.true() 29 | expect(blockstore.get(cid)).to.equalBytes(block) 30 | }) 31 | 32 | it('does not have a non-identity CID', async () => { 33 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 34 | const multihash = await sha256.digest(block) 35 | const cid = CID.createV1(raw.code, multihash) 36 | 37 | expect(blockstore.has(cid)).to.be.false() 38 | 39 | await blockstore.put(cid, block) 40 | 41 | expect(blockstore.has(cid)).to.be.false() 42 | }) 43 | 44 | it('cannot delete an identity CID', async () => { 45 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 46 | const multihash = identity.digest(block) 47 | const cid = CID.createV1(identity.code, multihash) 48 | 49 | await blockstore.delete(cid) 50 | 51 | expect(blockstore.has(cid)).to.be.true() 52 | }) 53 | 54 | it('cannot delete many identity CIDs', async () => { 55 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 56 | const multihash = identity.digest(block) 57 | const cid = CID.createV1(identity.code, multihash) 58 | 59 | await drain(blockstore.deleteMany([cid])) 60 | 61 | expect(blockstore.has(cid)).to.be.true() 62 | }) 63 | 64 | it('puts CIDs to child', async () => { 65 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 66 | const multihash = await sha256.digest(block) 67 | const cid = CID.createV1(raw.code, multihash) 68 | 69 | blockstore = new IdentityBlockstore(child) 70 | 71 | await blockstore.put(cid, block) 72 | expect(child.has(cid)).to.be.true() 73 | expect(child.get(cid)).to.equalBytes(block) 74 | }) 75 | 76 | it('gets CIDs from child', async () => { 77 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 78 | const multihash = await sha256.digest(block) 79 | const cid = CID.createV1(raw.code, multihash) 80 | 81 | await child.put(cid, block) 82 | 83 | blockstore = new IdentityBlockstore(child) 84 | expect(blockstore.has(cid)).to.be.true() 85 | expect(blockstore.get(cid)).to.equalBytes(block) 86 | }) 87 | 88 | it('has CIDs from child', async () => { 89 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 90 | const multihash = await sha256.digest(block) 91 | const cid = CID.createV1(raw.code, multihash) 92 | 93 | await child.put(cid, block) 94 | 95 | blockstore = new IdentityBlockstore(child) 96 | expect(blockstore.has(cid)).to.be.true() 97 | }) 98 | 99 | it('deletes CIDs from child', async () => { 100 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 101 | const multihash = await sha256.digest(block) 102 | const cid = CID.createV1(raw.code, multihash) 103 | 104 | await child.put(cid, block) 105 | 106 | blockstore = new IdentityBlockstore(child) 107 | expect(blockstore.has(cid)).to.be.true() 108 | 109 | await blockstore.delete(cid) 110 | 111 | expect(blockstore.has(cid)).to.be.false() 112 | }) 113 | 114 | it('gets all pairs from child', async () => { 115 | const block = Uint8Array.from([0, 1, 2, 3, 4]) 116 | const multihash = await sha256.digest(block) 117 | const cid = CID.createV1(raw.code, multihash) 118 | 119 | await child.put(cid, block) 120 | 121 | blockstore = new IdentityBlockstore(child) 122 | expect(blockstore.has(cid)).to.be.true() 123 | 124 | const result = await all(blockstore.getAll()) 125 | 126 | expect(result).to.have.lengthOf(1) 127 | expect(result[0].cid.toString()).to.equal(cid.toString()) 128 | }) 129 | }) 130 | -------------------------------------------------------------------------------- /packages/blockstore-core/test/memory.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { interfaceBlockstoreTests } from 'interface-blockstore-tests' 4 | import { MemoryBlockstore } from '../src/memory.js' 5 | 6 | describe('memory', () => { 7 | describe('interface-blockstore', () => { 8 | interfaceBlockstoreTests({ 9 | setup () { 10 | return new MemoryBlockstore() 11 | }, 12 | teardown () { } 13 | }) 14 | }) 15 | }) 16 | -------------------------------------------------------------------------------- /packages/blockstore-core/test/tiered.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { interfaceBlockstoreTests } from 'interface-blockstore-tests' 5 | import { CID } from 'multiformats/cid' 6 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 7 | import { MemoryBlockstore } from '../src/memory.js' 8 | import { TieredBlockstore } from '../src/tiered.js' 9 | import type { Blockstore } from 'interface-blockstore' 10 | 11 | describe('Tiered', () => { 12 | describe('all stores', () => { 13 | const ms: Blockstore[] = [] 14 | let store: TieredBlockstore 15 | beforeEach(() => { 16 | ms.push(new MemoryBlockstore()) 17 | ms.push(new MemoryBlockstore()) 18 | store = new TieredBlockstore(ms) 19 | }) 20 | 21 | it('put', async () => { 22 | const k = CID.parse('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') 23 | const v = uint8ArrayFromString('world') 24 | await store.put(k, v) 25 | const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) 26 | res.forEach((val) => { 27 | expect(val).to.be.eql(v) 28 | }) 29 | }) 30 | 31 | it('get and has, where available', async () => { 32 | const k = CID.parse('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') 33 | const v = uint8ArrayFromString('world') 34 | await ms[1].put(k, v) 35 | const val = await store.get(k) 36 | expect(val).to.be.eql(v) 37 | const exists = await store.has(k) 38 | expect(exists).to.be.eql(true) 39 | }) 40 | 41 | it('has - key not found', async () => { 42 | expect(await store.has(CID.parse('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnA'))).to.be.eql(false) 43 | }) 44 | 45 | it('has and delete', async () => { 46 | const k = CID.parse('QmTp9VkYvnHyrqKQuFPiuZkiX9gPcqj6x5LJ1rmWuSySnL') 47 | const v = uint8ArrayFromString('world') 48 | await store.put(k, v) 49 | let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) 50 | expect(res).to.be.eql([true, true]) 51 | await store.delete(k) 52 | res = await Promise.all([ms[0].has(k), ms[1].has(k)]) 53 | expect(res).to.be.eql([false, false]) 54 | }) 55 | }) 56 | 57 | describe('inteface-blockstore-single', () => { 58 | interfaceBlockstoreTests({ 59 | setup () { 60 | return new TieredBlockstore([ 61 | new MemoryBlockstore(), 62 | new MemoryBlockstore() 63 | ]) 64 | }, 65 | teardown () { } 66 | }) 67 | }) 68 | }) 69 | -------------------------------------------------------------------------------- /packages/blockstore-core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../interface-blockstore" 13 | }, 14 | { 15 | "path": "../interface-blockstore-tests" 16 | }, 17 | { 18 | "path": "../interface-store" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /packages/blockstore-core/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts", 5 | "./src/base.ts", 6 | "./src/black-hole.ts", 7 | "./src/errors.ts", 8 | "./src/identity.ts", 9 | "./src/memory.ts", 10 | "./src/tiered.ts" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /packages/blockstore-fs/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/blockstore-fs/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/blockstore-fs/README.md: -------------------------------------------------------------------------------- 1 | # blockstore-fs 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Blockstore implementation with file system backend 9 | 10 | # About 11 | 12 | 26 | 27 | A Blockstore implementation that stores blocks in the local filesystem. 28 | 29 | ## Example 30 | 31 | ```js 32 | import { FsBlockstore } from 'blockstore-fs' 33 | 34 | const store = new FsBlockstore('path/to/store') 35 | ``` 36 | 37 | # Install 38 | 39 | ```console 40 | $ npm i blockstore-fs 41 | ``` 42 | 43 | # API Docs 44 | 45 | - 46 | 47 | # License 48 | 49 | Licensed under either of 50 | 51 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-fs/LICENSE-APACHE) / ) 52 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-fs/LICENSE-MIT) / ) 53 | 54 | # Contribute 55 | 56 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 57 | 58 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 59 | 60 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 61 | 62 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 63 | 64 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 65 | -------------------------------------------------------------------------------- /packages/blockstore-fs/benchmarks/encoding/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmarks-encoding", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "private": true, 6 | "type": "module", 7 | "scripts": { 8 | "clean": "aegir clean", 9 | "build": "aegir build --bundle false", 10 | "lint": "aegir lint", 11 | "dep-check": "aegir dep-check", 12 | "start": "npm run build && node dist/src/index.js" 13 | }, 14 | "devDependencies": { 15 | "multiformats": "^11.0.1", 16 | "tinybench": "^2.4.0" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/blockstore-fs/benchmarks/encoding/src/README.md: -------------------------------------------------------------------------------- 1 | # Encoding Benchmark 2 | 3 | Multiformats ships a number of base encoding algorithms. This module has no strong opinion 4 | on which is best, as long as it is case insensitive, so benchmark them to choose the fastest. 5 | 6 | At the time of writing `base8` is the fastest, followed other alorithms using `rfc4648` encoding 7 | internally in `multiformats` (e.g. `base16`, `base32`), and finally anything using `baseX` encoding. 8 | 9 | We choose base32upper which uses `rfc4648` because it has a longer alphabet so will shard better. 10 | 11 | ## Usage 12 | 13 | ```console 14 | $ npm i 15 | $ npm start 16 | 17 | > benchmarks-gc@1.0.0 start 18 | > npm run build && node dist/src/index.js 19 | 20 | 21 | > benchmarks-gc@1.0.0 build 22 | > aegir build --bundle false 23 | 24 | [14:51:28] tsc [started] 25 | [14:51:33] tsc [completed] 26 | generating Ed25519 keypair... 27 | ┌─────────┬────────────────┬─────────┬───────────┬──────┐ 28 | │ (index) │ Implementation │ ops/s │ ms/op │ runs │ 29 | ├─────────┼────────────────┼─────────┼───────────┼──────┤ 30 | //... results here 31 | ``` 32 | -------------------------------------------------------------------------------- /packages/blockstore-fs/benchmarks/encoding/src/index.ts: -------------------------------------------------------------------------------- 1 | import { base10 } from 'multiformats/bases/base10' 2 | import { base16upper } from 'multiformats/bases/base16' 3 | import { base256emoji } from 'multiformats/bases/base256emoji' 4 | import { base32, base32upper, base32hexupper, base32z } from 'multiformats/bases/base32' 5 | import { base36, base36upper } from 'multiformats/bases/base36' 6 | import { base8 } from 'multiformats/bases/base8' 7 | import { CID } from 'multiformats/cid' 8 | import { Bench } from 'tinybench' 9 | 10 | const RESULT_PRECISION = 2 11 | 12 | const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') 13 | 14 | async function main (): Promise { 15 | const suite = new Bench() 16 | suite.add('base8', () => { 17 | base8.encode(cid.bytes) 18 | }) 19 | suite.add('base10', () => { 20 | base10.encode(cid.bytes) 21 | }) 22 | suite.add('base16upper', () => { 23 | base16upper.encode(cid.bytes) 24 | }) 25 | suite.add('base32', () => { 26 | base32.encode(cid.bytes) 27 | }) 28 | suite.add('base32upper', () => { 29 | base32upper.encode(cid.bytes) 30 | }) 31 | suite.add('base32hexupper', () => { 32 | base32hexupper.encode(cid.bytes) 33 | }) 34 | suite.add('base32z', () => { 35 | base32z.encode(cid.bytes) 36 | }) 37 | suite.add('base36', () => { 38 | base36.encode(cid.bytes) 39 | }) 40 | suite.add('base36upper', () => { 41 | base36upper.encode(cid.bytes) 42 | }) 43 | suite.add('base256emoji', () => { 44 | base256emoji.encode(cid.bytes) 45 | }) 46 | 47 | await suite.run() 48 | 49 | console.table(suite.tasks.sort((a, b) => { // eslint-disable-line no-console 50 | const resultA = a.result?.hz ?? 0 51 | const resultB = b.result?.hz ?? 0 52 | 53 | if (resultA === resultB) { 54 | return 0 55 | } 56 | 57 | if (resultA < resultB) { 58 | return 1 59 | } 60 | 61 | return -1 62 | }).map(({ name, result }) => ({ 63 | Implementation: name, 64 | 'ops/s': parseFloat(result?.hz.toFixed(RESULT_PRECISION) ?? '0'), 65 | 'ms/op': parseFloat(result?.period.toFixed(RESULT_PRECISION) ?? '0'), 66 | runs: result?.samples.length 67 | }))) 68 | } 69 | 70 | main().catch(err => { 71 | console.error(err) // eslint-disable-line no-console 72 | process.exit(1) 73 | }) 74 | -------------------------------------------------------------------------------- /packages/blockstore-fs/benchmarks/encoding/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "target": "ES2022", 6 | "module": "ES2022", 7 | "lib": ["ES2022", "DOM", "DOM.Iterable"] 8 | }, 9 | "include": [ 10 | "src", 11 | "test" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /packages/blockstore-fs/src/sharding.ts: -------------------------------------------------------------------------------- 1 | import path from 'node:path' 2 | import { base32upper } from 'multiformats/bases/base32' 3 | import { CID } from 'multiformats/cid' 4 | import type { MultibaseCodec } from 'multiformats/bases/interface' 5 | 6 | export interface ShardingStrategy { 7 | extension: string 8 | encode(cid: CID): { dir: string, file: string } 9 | decode(path: string): CID 10 | } 11 | 12 | export interface NextToLastInit { 13 | /** 14 | * The file extension to use. default: '.data' 15 | */ 16 | extension?: string 17 | 18 | /** 19 | * How many characters to take from the end of the CID. default: 2 20 | */ 21 | prefixLength?: number 22 | 23 | /** 24 | * The multibase codec to use - nb. should be case insensitive. 25 | * default: base32upper 26 | */ 27 | base?: MultibaseCodec 28 | } 29 | 30 | /** 31 | * A sharding strategy that takes the last few characters of a multibase encoded 32 | * CID and uses them as the directory to store the block in. This prevents 33 | * storing all blocks in a single directory which would overwhelm most 34 | * filesystems. 35 | */ 36 | export class NextToLast implements ShardingStrategy { 37 | public extension: string 38 | private readonly prefixLength: number 39 | private readonly base: MultibaseCodec 40 | 41 | constructor (init: NextToLastInit = {}) { 42 | this.extension = init.extension ?? '.data' 43 | this.prefixLength = init.prefixLength ?? 2 44 | this.base = init.base ?? base32upper 45 | } 46 | 47 | encode (cid: CID): { dir: string, file: string } { 48 | const str = this.base.encoder.encode(cid.multihash.bytes) 49 | const prefix = str.substring(str.length - this.prefixLength) 50 | 51 | return { 52 | dir: prefix, 53 | file: `${str}${this.extension}` 54 | } 55 | } 56 | 57 | decode (str: string): CID { 58 | let fileName = path.basename(str) 59 | 60 | if (fileName.endsWith(this.extension)) { 61 | fileName = fileName.substring(0, fileName.length - this.extension.length) 62 | } 63 | 64 | return CID.decode(this.base.decoder.decode(fileName)) 65 | } 66 | } 67 | 68 | export interface FlatDirectoryInit { 69 | /** 70 | * The file extension to use. default: '.data' 71 | */ 72 | extension?: string 73 | 74 | /** 75 | * How many characters to take from the end of the CID. default: 2 76 | */ 77 | prefixLength?: number 78 | 79 | /** 80 | * The multibase codec to use - nb. should be case insensitive. 81 | * default: base32padupper 82 | */ 83 | base?: MultibaseCodec 84 | } 85 | 86 | /** 87 | * A sharding strategy that does not do any sharding and stores all files 88 | * in one directory. Only for testing, do not use in production. 89 | */ 90 | export class FlatDirectory implements ShardingStrategy { 91 | public extension: string 92 | private readonly base: MultibaseCodec 93 | 94 | constructor (init: NextToLastInit = {}) { 95 | this.extension = init.extension ?? '.data' 96 | this.base = init.base ?? base32upper 97 | } 98 | 99 | encode (cid: CID): { dir: string, file: string } { 100 | const str = this.base.encoder.encode(cid.multihash.bytes) 101 | 102 | return { 103 | dir: '', 104 | file: `${str}${this.extension}` 105 | } 106 | } 107 | 108 | decode (str: string): CID { 109 | let fileName = path.basename(str) 110 | 111 | if (fileName.endsWith(this.extension)) { 112 | fileName = fileName.substring(0, fileName.length - this.extension.length) 113 | } 114 | 115 | return CID.decode(this.base.decoder.decode(fileName)) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /packages/blockstore-fs/test/fixtures/writer-worker.ts: -------------------------------------------------------------------------------- 1 | import { CID } from 'multiformats/cid' 2 | // @ts-expect-error types are broken: https://github.com/andywer/threads.js/pull/470 3 | import { expose } from 'threads/worker' 4 | import { FsBlockstore } from '../../src/index.js' 5 | 6 | let fs: FsBlockstore 7 | expose({ 8 | async isReady (path: string) { 9 | fs = new FsBlockstore(path) 10 | try { 11 | await fs.open() 12 | return true 13 | } catch (err) { 14 | // eslint-disable-next-line no-console 15 | console.error('Error opening blockstore', err) 16 | throw err 17 | } 18 | }, 19 | async put (cidString: string, value: Uint8Array) { 20 | const key = CID.parse(cidString) 21 | try { 22 | return await fs.put(key, value) 23 | } catch (err) { 24 | // eslint-disable-next-line no-console 25 | console.error('Error putting block', err) 26 | throw err 27 | } 28 | } 29 | }) 30 | -------------------------------------------------------------------------------- /packages/blockstore-fs/test/sharding.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | import { expect } from 'aegir/chai' 3 | import { base32upper } from 'multiformats/bases/base32' 4 | import { CID } from 'multiformats/cid' 5 | import { FlatDirectory, NextToLast } from '../src/sharding.js' 6 | 7 | describe('flat', () => { 8 | it('should encode', () => { 9 | const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') 10 | const strategy = new FlatDirectory() 11 | const { dir, file } = strategy.encode(cid) 12 | 13 | expect(dir).to.equal('') 14 | expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.data`) 15 | }) 16 | 17 | it('should encode with extension', () => { 18 | const cid = CID.parse('QmeimKZyjcBnuXmAD9zMnSjM9JodTbgGT3gutofkTqz9rE') 19 | const strategy = new FlatDirectory({ 20 | extension: '.file' 21 | }) 22 | const { dir, file } = strategy.encode(cid) 23 | 24 | expect(dir).to.equal('') 25 | expect(file).to.equal(`${base32upper.encode(cid.multihash.bytes)}.file`) 26 | }) 27 | 28 | it('should decode', () => { 29 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 30 | const strategy = new FlatDirectory() 31 | const cid = strategy.decode(`${mh}.data`) 32 | 33 | expect(cid).to.eql(CID.decode(base32upper.decode(mh))) 34 | }) 35 | 36 | it('should decode with extension', () => { 37 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 38 | const strategy = new FlatDirectory({ 39 | extension: '.file' 40 | }) 41 | const cid = strategy.decode(`${mh}.file`) 42 | 43 | expect(cid).to.eql(CID.decode(base32upper.decode(mh))) 44 | }) 45 | }) 46 | 47 | describe('next to last', () => { 48 | it('should encode', () => { 49 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 50 | const cid = CID.decode(base32upper.decode(mh)) 51 | const strategy = new NextToLast() 52 | const { dir, file } = strategy.encode(cid) 53 | 54 | expect(dir).to.equal('LY') 55 | expect(file).to.equal(`${mh}.data`) 56 | }) 57 | 58 | it('should encode with prefix length', () => { 59 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 60 | const cid = CID.decode(base32upper.decode(mh)) 61 | const strategy = new NextToLast({ 62 | prefixLength: 4 63 | }) 64 | const { dir, file } = strategy.encode(cid) 65 | 66 | expect(dir).to.equal('QYLY') 67 | expect(file).to.equal(`${mh}.data`) 68 | }) 69 | 70 | it('should encode with extension', () => { 71 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 72 | const cid = CID.decode(base32upper.decode(mh)) 73 | const strategy = new NextToLast({ 74 | extension: '.file' 75 | }) 76 | const { dir, file } = strategy.encode(cid) 77 | 78 | expect(dir).to.equal('LY') 79 | expect(file).to.equal(`${mh}.file`) 80 | }) 81 | 82 | it('should decode', () => { 83 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 84 | const strategy = new NextToLast() 85 | const cid = strategy.decode(`LY/${mh}.data`) 86 | 87 | expect(cid).to.eql(CID.decode(base32upper.decode(mh))) 88 | }) 89 | 90 | it('should decode with prefix length', () => { 91 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 92 | const strategy = new NextToLast({ 93 | prefixLength: 4 94 | }) 95 | const cid = strategy.decode(`QYLY/${mh}.data`) 96 | 97 | expect(cid).to.eql(CID.decode(base32upper.decode(mh))) 98 | }) 99 | 100 | it('should decode with extension', () => { 101 | const mh = 'BCIQPGZJ6QLZOFG3OP45NLMSJUWGJCO72QQKHLDTB6FXIB6BDSLRQYLY' 102 | const strategy = new NextToLast({ 103 | extension: '.file' 104 | }) 105 | const cid = strategy.decode(`LY/${mh}.file`) 106 | 107 | expect(cid).to.eql(CID.decode(base32upper.decode(mh))) 108 | }) 109 | }) 110 | -------------------------------------------------------------------------------- /packages/blockstore-fs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../interface-blockstore" 13 | }, 14 | { 15 | "path": "../interface-blockstore-tests" 16 | }, 17 | { 18 | "path": "../interface-store" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /packages/blockstore-fs/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts", 5 | "./src/sharding.ts" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /packages/blockstore-idb/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/blockstore-idb/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/blockstore-idb/README.md: -------------------------------------------------------------------------------- 1 | # blockstore-idb 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Blockstore implementation with IndexedDB backend 9 | 10 | # About 11 | 12 | 26 | 27 | A Blockstore implementation for browsers that stores blocks in [IndexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API). 28 | 29 | ## Example 30 | 31 | ```js 32 | import { IDBBlockstore } from 'blockstore-idb' 33 | 34 | const store = new IDBBlockstore('path/to/store') 35 | ``` 36 | 37 | # Install 38 | 39 | ```console 40 | $ npm i blockstore-idb 41 | ``` 42 | 43 | ## Browser ` 49 | ``` 50 | 51 | # API Docs 52 | 53 | - 54 | 55 | # License 56 | 57 | Licensed under either of 58 | 59 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-idb/LICENSE-APACHE) / ) 60 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-idb/LICENSE-MIT) / ) 61 | 62 | # Contribute 63 | 64 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 65 | 66 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 67 | 68 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 69 | 70 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 71 | 72 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 73 | -------------------------------------------------------------------------------- /packages/blockstore-idb/test/index.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { interfaceBlockstoreTests } from 'interface-blockstore-tests' 5 | import { CID } from 'multiformats/cid' 6 | import { IDBBlockstore } from '../src/index.js' 7 | 8 | describe('IndexedDB Blockstore', function () { 9 | describe('interface-blockstore (idb)', () => { 10 | interfaceBlockstoreTests({ 11 | async setup () { 12 | const store = new IDBBlockstore(`hello-${Math.random()}`) 13 | await store.open() 14 | return store 15 | }, 16 | async teardown (store) { 17 | await store.close() 18 | await store.destroy() 19 | } 20 | }) 21 | }) 22 | 23 | describe('concurrency', () => { 24 | let store: IDBBlockstore 25 | 26 | before(async () => { 27 | store = new IDBBlockstore('hello') 28 | await store.open() 29 | }) 30 | 31 | it('should not explode under unreasonable load', function (done) { 32 | this.timeout(10000) 33 | 34 | const updater = setInterval(async () => { 35 | try { 36 | const key = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') 37 | 38 | await store.put(key, Uint8Array.from([0, 1, 2, 3])) 39 | await store.has(key) 40 | await store.get(key) 41 | } catch (err) { 42 | clearInterval(updater) 43 | clearInterval(mutatorQuery) 44 | clearInterval(readOnlyQuery) 45 | done(err) 46 | } 47 | }, 0) 48 | 49 | const mutatorQuery = setInterval(async () => { 50 | try { 51 | for await (const { cid } of store.getAll()) { 52 | await store.get(cid) 53 | 54 | const otherKey = CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F') 55 | const otherValue = Uint8Array.from([0, 1, 2, 3]) 56 | await store.put(otherKey, otherValue) 57 | const res = await store.get(otherKey) 58 | expect(res).to.deep.equal(otherValue) 59 | } 60 | } catch (err) { 61 | clearInterval(updater) 62 | clearInterval(mutatorQuery) 63 | clearInterval(readOnlyQuery) 64 | done(err) 65 | } 66 | }, 0) 67 | 68 | const readOnlyQuery = setInterval(async () => { 69 | try { 70 | for await (const { cid } of store.getAll()) { 71 | await store.has(cid) 72 | } 73 | } catch (err) { 74 | clearInterval(updater) 75 | clearInterval(mutatorQuery) 76 | clearInterval(readOnlyQuery) 77 | done(err) 78 | } 79 | }, 0) 80 | 81 | setTimeout(() => { 82 | clearInterval(updater) 83 | clearInterval(mutatorQuery) 84 | clearInterval(readOnlyQuery) 85 | done() 86 | }, 5000) 87 | }) 88 | }) 89 | }) 90 | -------------------------------------------------------------------------------- /packages/blockstore-idb/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../blockstore-core" 13 | }, 14 | { 15 | "path": "../interface-blockstore" 16 | }, 17 | { 18 | "path": "../interface-blockstore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/blockstore-idb/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/blockstore-level/.aegir.js: -------------------------------------------------------------------------------- 1 | /** @type {import('aegir').PartialOptions} */ 2 | export default { 3 | build: { 4 | bundlesizeMax: '27KB' 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/blockstore-level/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/blockstore-level/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/blockstore-level/README.md: -------------------------------------------------------------------------------- 1 | # blockstore-level 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Blockstore implementation with level(up|down) backend 9 | 10 | # About 11 | 12 | 26 | 27 | A Blockstore implementation that uses a flavour of [Level](https://leveljs.org/) as a backend. 28 | 29 | N.b. this is here largely for the sake of completeness, in node you should probably use FSDatastore, in browsers you should probably use IDBDatastore. 30 | 31 | ## Example 32 | 33 | ```js 34 | import { LevelBlockstore } from 'blockstore-level' 35 | 36 | const store = new LevelBlockstore('path/to/store') 37 | ``` 38 | 39 | # Install 40 | 41 | ```console 42 | $ npm i blockstore-level 43 | ``` 44 | 45 | ## Browser ` 51 | ``` 52 | 53 | # API Docs 54 | 55 | - 56 | 57 | # License 58 | 59 | Licensed under either of 60 | 61 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-level/LICENSE-APACHE) / ) 62 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-level/LICENSE-MIT) / ) 63 | 64 | # Contribute 65 | 66 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 67 | 68 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 69 | 70 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 71 | 72 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 73 | 74 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 75 | -------------------------------------------------------------------------------- /packages/blockstore-level/test/fixtures/test-level-iterator-destroy.ts: -------------------------------------------------------------------------------- 1 | import tempdir from 'ipfs-utils/src/temp-dir.js' 2 | import { CID } from 'multiformats/cid' 3 | import { LevelBlockstore } from '../../src/index.js' 4 | 5 | async function testLevelIteratorDestroy (): Promise { 6 | const store = new LevelBlockstore(tempdir()) 7 | await store.open() 8 | await store.put(CID.parse('QmaQwYWpchozXhFv8nvxprECWBSCEppN9dfd2VQiJfRo3F'), new TextEncoder().encode(`TESTDATA${Date.now()}`)) 9 | for await (const d of store.getAll()) { 10 | console.log(d) // eslint-disable-line no-console 11 | } 12 | } 13 | 14 | // Will exit with: 15 | // > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. 16 | // If iterator gets destroyed (in c++ land) and .end() was not called on it. 17 | void testLevelIteratorDestroy() 18 | -------------------------------------------------------------------------------- /packages/blockstore-level/test/index.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { interfaceBlockstoreTests } from 'interface-blockstore-tests' 5 | import tempdir from 'ipfs-utils/src/temp-dir.js' 6 | import { Level } from 'level' 7 | import { MemoryLevel } from 'memory-level' 8 | import { LevelBlockstore } from '../src/index.js' 9 | 10 | describe('LevelBlockstore', () => { 11 | describe('initialization', () => { 12 | it('should default to a leveldown database', async () => { 13 | const levelStore = new LevelBlockstore(`${tempdir()}/init-default-${Date.now()}`) 14 | await levelStore.open() 15 | 16 | expect(levelStore.db).to.be.an.instanceOf(Level) 17 | }) 18 | 19 | it('should be able to override the database', async () => { 20 | const levelStore = new LevelBlockstore( 21 | // @ts-expect-error MemoryLevel does not implement the same interface as Level 22 | new MemoryLevel({ 23 | keyEncoding: 'utf8', 24 | valueEncoding: 'view' 25 | }) 26 | ) 27 | 28 | await levelStore.open() 29 | 30 | expect(levelStore.db).to.be.an.instanceOf(MemoryLevel) 31 | }) 32 | }) 33 | 34 | describe('interface-blockstore MemoryLevel', () => { 35 | interfaceBlockstoreTests({ 36 | async setup () { 37 | const store = new LevelBlockstore( 38 | // @ts-expect-error MemoryLevel does not implement the same interface as Level 39 | new MemoryLevel({ 40 | keyEncoding: 'utf8', 41 | valueEncoding: 'view' 42 | }) 43 | ) 44 | await store.open() 45 | 46 | return store 47 | }, 48 | teardown () {} 49 | }) 50 | }) 51 | 52 | describe('interface-blockstore Level', () => { 53 | interfaceBlockstoreTests({ 54 | async setup () { 55 | const store = new LevelBlockstore(tempdir()) 56 | await store.open() 57 | 58 | return store 59 | }, 60 | teardown () {} 61 | }) 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /packages/blockstore-level/test/node.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import childProcess from 'child_process' 4 | import path from 'path' 5 | import { expect } from 'aegir/chai' 6 | import { interfaceBlockstoreTests } from 'interface-blockstore-tests' 7 | import tempdir from 'ipfs-utils/src/temp-dir.js' 8 | import { LevelBlockstore } from '../src/index.js' 9 | 10 | describe('LevelBlockstore', () => { 11 | describe('interface-blockstore (leveldown)', () => { 12 | interfaceBlockstoreTests({ 13 | async setup () { 14 | const store = new LevelBlockstore(tempdir()) 15 | await store.open() 16 | 17 | return store 18 | }, 19 | teardown () {} 20 | }) 21 | }) 22 | 23 | // The `.end()` method MUST be called on LevelDB iterators or they remain open, 24 | // leaking memory. 25 | // 26 | // This test exposes this problem by causing an error to be thrown on process 27 | // exit when an iterator is open AND leveldb is not closed. 28 | // 29 | // Normally when leveldb is closed it'll automatically clean up open iterators 30 | // but if you don't close the store this error will occur: 31 | // 32 | // > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. 33 | // 34 | // This is thrown by a destructor function for iterator objects that asserts 35 | // the iterator has ended before cleanup. 36 | // 37 | // https://github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc#L545 38 | it('should not leave iterators open and leak memory', (done) => { 39 | const cp = childProcess.fork(path.join(process.cwd(), '/dist/test/fixtures/test-level-iterator-destroy'), { stdio: 'pipe' }) 40 | 41 | let out = '' 42 | const { stdout, stderr } = cp 43 | stdout?.on('data', d => { out = `${out}${d}` }) 44 | stderr?.on('data', d => { out = `${out}${d}` }) 45 | 46 | cp.on('exit', code => { 47 | expect(code).to.equal(0) 48 | expect(out).to.not.include('Assertion failed: (ended_)') 49 | done() 50 | }) 51 | }) 52 | }) 53 | -------------------------------------------------------------------------------- /packages/blockstore-level/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "test", 8 | "src" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../blockstore-core" 13 | }, 14 | { 15 | "path": "../interface-blockstore" 16 | }, 17 | { 18 | "path": "../interface-blockstore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/blockstore-level/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/blockstore-s3/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/blockstore-s3/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/blockstore-s3/README.md: -------------------------------------------------------------------------------- 1 | # blockstore-s3 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > IPFS blockstore implementation backed by s3 9 | 10 | # About 11 | 12 | 26 | 27 | A Blockstore implementation that stores blocks on Amazon S3. 28 | 29 | ## Example - Quickstart 30 | 31 | If the flag `createIfMissing` is not set or is false, then the bucket must be created prior to using blockstore-s3. Please see the AWS docs for information on how to configure the S3 instance. A bucket name is required to be set at the s3 instance level, see the below example. 32 | 33 | ```js 34 | import { S3 } from '@aws-sdk/client-s3' 35 | import { S3Blockstore } from 'blockstore-s3' 36 | 37 | const s3 = new S3({ 38 | region: 'region', 39 | credentials: { 40 | accessKeyId: 'myaccesskey', 41 | secretAccessKey: 'mysecretkey' 42 | } 43 | }) 44 | 45 | const store = new S3Blockstore( 46 | s3, 47 | 'my-bucket', 48 | { createIfMissing: false } 49 | ) 50 | ``` 51 | 52 | ## Example 53 | 54 | ```ts 55 | Using with Helia 56 | 57 | See [examples/helia](./examples/helia) for a full example of how to use Helia with an S3 backed blockstore. 58 | ``` 59 | 60 | # Install 61 | 62 | ```console 63 | $ npm i blockstore-s3 64 | ``` 65 | 66 | ## Browser ` 72 | ``` 73 | 74 | # API Docs 75 | 76 | - 77 | 78 | # License 79 | 80 | Licensed under either of 81 | 82 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-s3/LICENSE-APACHE) / ) 83 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-s3/LICENSE-MIT) / ) 84 | 85 | # Contribute 86 | 87 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 88 | 89 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 90 | 91 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 92 | 93 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 94 | 95 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 96 | -------------------------------------------------------------------------------- /packages/blockstore-s3/examples/helia/README.md: -------------------------------------------------------------------------------- 1 | Use with Helia 2 | ====== 3 | 4 | This example uses a Blockstore S3 instance to serve as the entire backend for Helia. 5 | 6 | ## Running 7 | The S3 parameters must be updated with an existing Bucket and credentials with access to it: 8 | ```js 9 | // Configure S3 as normal 10 | const s3 = new S3({ 11 | region: 'region', 12 | credentials: { 13 | accessKeyId: 'myaccesskey', 14 | secretAccessKey: 'mysecretkey' 15 | } 16 | }) 17 | 18 | const blockstore = new BlockstoreS3(s3, 'my-bucket') 19 | ``` 20 | 21 | Once the S3 instance has its needed data, you can run the example: 22 | ``` 23 | npm install 24 | node index.js 25 | ``` 26 | -------------------------------------------------------------------------------- /packages/blockstore-s3/examples/helia/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | 3 | import { S3 } from '@aws-sdk/client-s3' 4 | import { unixfs } from '@helia/unixfs' 5 | import { BlockstoreS3 } from 'blockstore-s3' 6 | import { createHelia } from 'helia' 7 | import toBuffer from 'it-to-buffer' 8 | 9 | async function main () { 10 | // Configure S3 as normal 11 | const s3 = new S3({ 12 | region: 'region', 13 | credentials: { 14 | accessKeyId: 'myaccesskey', 15 | secretAccessKey: 'mysecretkey' 16 | } 17 | }) 18 | 19 | const blockstore = new BlockstoreS3(s3, 'my-bucket') 20 | 21 | // Create a new Helia node with our S3 backed Repo 22 | console.log('Start Helia') 23 | const helia = await createHelia({ 24 | blockstore 25 | }) 26 | 27 | // Test out the repo by sending and fetching some data 28 | console.log('Helia is ready') 29 | 30 | try { 31 | const fs = unixfs(helia) 32 | 33 | // Let's add a file to Helia 34 | const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) 35 | 36 | console.log('\nAdded file:', cid) 37 | 38 | // Log out the added files metadata and cat the file from IPFS 39 | const data = await toBuffer(fs.cat(cid)) 40 | 41 | // Print out the files contents to console 42 | console.log(`\nFetched file content containing ${data.byteLength} bytes`) 43 | } catch (err) { 44 | // Log out the error 45 | console.log('File Processing Error:', err) 46 | } 47 | // After everything is done, shut the node down 48 | // We don't need to worry about catching errors here 49 | console.log('\n\nStopping the node') 50 | await helia.stop() 51 | } 52 | 53 | main() 54 | .catch(err => { 55 | console.error(err) 56 | process.exit(1) 57 | }) 58 | -------------------------------------------------------------------------------- /packages/blockstore-s3/examples/helia/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "helia", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "@aws-sdk/client-s3": "^3.297.0", 14 | "@helia/unixfs": "^1.2.0", 15 | "blockstore-s3": "../../", 16 | "helia": "^1.0.0", 17 | "it-to-buffer": "^3.0.1" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /packages/blockstore-s3/src/sharding.ts: -------------------------------------------------------------------------------- 1 | import { base32upper } from 'multiformats/bases/base32' 2 | import { CID } from 'multiformats/cid' 3 | import type { MultibaseCodec } from 'multiformats/bases/interface' 4 | 5 | export interface ShardingStrategy { 6 | extension: string 7 | encode(cid: CID): string 8 | decode(path: string): CID 9 | } 10 | 11 | export interface NextToLastInit { 12 | /** 13 | * The file extension to use. default: '.data' 14 | */ 15 | extension?: string 16 | 17 | /** 18 | * How many characters to take from the end of the CID. default: 2 19 | */ 20 | prefixLength?: number 21 | 22 | /** 23 | * The multibase codec to use - nb. should be case insensitive. 24 | * default: base32upper 25 | */ 26 | base?: MultibaseCodec 27 | } 28 | 29 | /** 30 | * A sharding strategy that takes the last few characters of a multibase encoded 31 | * CID and uses them as the directory to store the block in. This prevents 32 | * storing all blocks in a single directory which would overwhelm most 33 | * filesystems. 34 | */ 35 | export class NextToLast implements ShardingStrategy { 36 | public extension: string 37 | private readonly prefixLength: number 38 | private readonly base: MultibaseCodec 39 | 40 | constructor (init: NextToLastInit = {}) { 41 | this.extension = init.extension ?? '.data' 42 | this.prefixLength = init.prefixLength ?? 2 43 | this.base = init.base ?? base32upper 44 | } 45 | 46 | encode (cid: CID): string { 47 | const str = this.base.encoder.encode(cid.multihash.bytes) 48 | const prefix = str.substring(str.length - this.prefixLength) 49 | 50 | return `${prefix}/${str}${this.extension}` 51 | } 52 | 53 | decode (str: string): CID { 54 | let fileName = str.split('/').pop() 55 | 56 | if (fileName == null) { 57 | throw new Error('Invalid path') 58 | } 59 | 60 | if (fileName.endsWith(this.extension)) { 61 | fileName = fileName.substring(0, fileName.length - this.extension.length) 62 | } 63 | 64 | return CID.decode(this.base.decoder.decode(fileName)) 65 | } 66 | } 67 | 68 | export interface FlatDirectoryInit { 69 | /** 70 | * The file extension to use. default: '.data' 71 | */ 72 | extension?: string 73 | 74 | /** 75 | * How many characters to take from the end of the CID. default: 2 76 | */ 77 | prefixLength?: number 78 | 79 | /** 80 | * The multibase codec to use - nb. should be case insensitive. 81 | * default: base32padupper 82 | */ 83 | base?: MultibaseCodec 84 | } 85 | 86 | /** 87 | * A sharding strategy that does not do any sharding and stores all files 88 | * in one directory. Only for testing, do not use in production. 89 | */ 90 | export class FlatDirectory implements ShardingStrategy { 91 | public extension: string 92 | private readonly base: MultibaseCodec 93 | 94 | constructor (init: NextToLastInit = {}) { 95 | this.extension = init.extension ?? '.data' 96 | this.base = init.base ?? base32upper 97 | } 98 | 99 | encode (cid: CID): string { 100 | const str = this.base.encoder.encode(cid.multihash.bytes) 101 | 102 | return `${str}${this.extension}` 103 | } 104 | 105 | decode (str: string): CID { 106 | let fileName = str.split('/').pop() 107 | 108 | if (fileName == null) { 109 | throw new Error('Invalid path') 110 | } 111 | 112 | if (fileName.endsWith(this.extension)) { 113 | fileName = fileName.substring(0, fileName.length - this.extension.length) 114 | } 115 | 116 | return CID.decode(this.base.decoder.decode(fileName)) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /packages/blockstore-s3/test/utils/s3-mock.ts: -------------------------------------------------------------------------------- 1 | import sinon from 'sinon' 2 | import type { S3 } from '@aws-sdk/client-s3' 3 | 4 | export class S3Error extends Error { 5 | public code: string 6 | public statusCode?: number 7 | public $metadata?: { httpStatusCode: number } 8 | 9 | constructor (message: string, code?: number) { 10 | super(message) 11 | this.code = message 12 | this.statusCode = code 13 | 14 | this.$metadata = { 15 | httpStatusCode: code ?? 200 16 | } 17 | } 18 | } 19 | 20 | export function s3Resolve (res?: any): any { 21 | return Promise.resolve(res) 22 | } 23 | 24 | export function s3Reject (err: Error): any { 25 | return Promise.reject(err) 26 | } 27 | 28 | /** 29 | * Mocks out the s3 calls made by blockstore-s3 30 | */ 31 | export function s3Mock (s3: S3): S3 { 32 | const mocks: any = {} 33 | const storage = new Map() 34 | 35 | mocks.send = sinon.replace(s3, 'send', (command: any) => { 36 | const commandName = command.constructor.name 37 | const input: any = command.input 38 | 39 | if (commandName.includes('PutObjectCommand') === true) { 40 | storage.set(input.Key, input.Body) 41 | return s3Resolve({}) 42 | } 43 | 44 | if (commandName.includes('HeadObjectCommand') === true) { 45 | if (storage.has(input.Key)) { 46 | return s3Resolve({}) 47 | } 48 | 49 | return s3Reject(new S3Error('NotFound', 404)) 50 | } 51 | 52 | if (commandName.includes('GetObjectCommand') === true) { 53 | if (!storage.has(input.Key)) { 54 | return s3Reject(new S3Error('NotFound', 404)) 55 | } 56 | 57 | return s3Resolve({ 58 | Body: storage.get(input.Key) 59 | }) 60 | } 61 | 62 | if (commandName.includes('DeleteObjectCommand') === true) { 63 | storage.delete(input.Key) 64 | return s3Resolve({}) 65 | } 66 | 67 | if (commandName.includes('ListObjectsV2Command') === true) { 68 | const results: { Contents: Array<{ Key: string }> } = { 69 | Contents: [] 70 | } 71 | 72 | for (const k of storage.keys()) { 73 | if (k.startsWith(`${input.Prefix ?? ''}`)) { 74 | results.Contents.push({ 75 | Key: k 76 | }) 77 | } 78 | } 79 | 80 | return s3Resolve(results) 81 | } 82 | 83 | return s3Reject(new S3Error('UnknownCommand', 400)) 84 | }) 85 | 86 | return s3 87 | } 88 | -------------------------------------------------------------------------------- /packages/blockstore-s3/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../blockstore-core" 13 | }, 14 | { 15 | "path": "../interface-blockstore" 16 | }, 17 | { 18 | "path": "../interface-blockstore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/blockstore-s3/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/datastore-core/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/datastore-core/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/datastore-core/README.md: -------------------------------------------------------------------------------- 1 | # datastore-core 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Wrapper implementation for interface-datastore 9 | 10 | # About 11 | 12 | 26 | 27 | Various Datastore implementations are available. 28 | 29 | ## Implementations 30 | 31 | - Mount: [`src/mount`](./src/mount.ts) 32 | - Keytransform: [`src/keytransform`](./src/keytransform.ts) 33 | - Sharding: [`src/sharding`](./src/sharding.ts) 34 | - Tiered: [`src/tiered`](./src/tirered.ts) 35 | - Namespace: [`src/namespace`](./src/namespace.ts) 36 | - BlackHole: [`src/black-hole`](./src/black-hole.ts) 37 | 38 | ## Example - BaseDatastore 39 | 40 | An base store is made available to make implementing your own datastore easier: 41 | 42 | ```javascript 43 | import { BaseDatastore } from 'datastore-core' 44 | 45 | class MyDatastore extends BaseDatastore { 46 | constructor () { 47 | super() 48 | } 49 | 50 | async put (key, val) { 51 | // your implementation here 52 | } 53 | 54 | async get (key) { 55 | // your implementation here 56 | } 57 | 58 | // etc... 59 | } 60 | ``` 61 | 62 | See the [MemoryDatastore](./src/memory.js) for an example of how it is used. 63 | 64 | ## Example - Wrapping Stores 65 | 66 | ```js 67 | import { Key } from 'interface-datastore' 68 | import { 69 | MemoryStore, 70 | MountStore 71 | } from 'datastore-core' 72 | 73 | const store = new MountStore({prefix: new Key('/a'), datastore: new MemoryStore()}) 74 | ``` 75 | 76 | ## Example - BlackHoleDatastore 77 | 78 | A datastore that does not store any data. 79 | 80 | ```js 81 | import { BlackHoleDatastore } from 'datastore-core/black-hole' 82 | 83 | const store = new BlackHoleDatastore() 84 | ``` 85 | 86 | # Install 87 | 88 | ```console 89 | $ npm i datastore-core 90 | ``` 91 | 92 | ## Browser ` 98 | ``` 99 | 100 | # API Docs 101 | 102 | - 103 | 104 | # License 105 | 106 | Licensed under either of 107 | 108 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/datastore-core/LICENSE-APACHE) / ) 109 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/datastore-core/LICENSE-MIT) / ) 110 | 111 | # Contribute 112 | 113 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 114 | 115 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 116 | 117 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 118 | 119 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 120 | 121 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 122 | -------------------------------------------------------------------------------- /packages/datastore-core/src/base.ts: -------------------------------------------------------------------------------- 1 | import drain from 'it-drain' 2 | import filter from 'it-filter' 3 | import sort from 'it-sort' 4 | import take from 'it-take' 5 | import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' 6 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 7 | 8 | export class BaseDatastore implements Datastore { 9 | put (key: Key, val: Uint8Array, options?: AbortOptions): Await { 10 | return Promise.reject(new Error('.put is not implemented')) 11 | } 12 | 13 | get (key: Key, options?: AbortOptions): Await { 14 | return Promise.reject(new Error('.get is not implemented')) 15 | } 16 | 17 | has (key: Key, options?: AbortOptions): Await { 18 | return Promise.reject(new Error('.has is not implemented')) 19 | } 20 | 21 | delete (key: Key, options?: AbortOptions): Await { 22 | return Promise.reject(new Error('.delete is not implemented')) 23 | } 24 | 25 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { 26 | for await (const { key, value } of source) { 27 | await this.put(key, value, options) 28 | yield key 29 | } 30 | } 31 | 32 | async * getMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { 33 | for await (const key of source) { 34 | yield { 35 | key, 36 | value: await this.get(key, options) 37 | } 38 | } 39 | } 40 | 41 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { 42 | for await (const key of source) { 43 | await this.delete(key, options) 44 | yield key 45 | } 46 | } 47 | 48 | batch (): Batch { 49 | let puts: Pair[] = [] 50 | let dels: Key[] = [] 51 | 52 | return { 53 | put (key, value) { 54 | puts.push({ key, value }) 55 | }, 56 | 57 | delete (key) { 58 | dels.push(key) 59 | }, 60 | commit: async (options) => { 61 | await drain(this.putMany(puts, options)) 62 | puts = [] 63 | await drain(this.deleteMany(dels, options)) 64 | dels = [] 65 | } 66 | } 67 | } 68 | 69 | /** 70 | * Extending classes should override `query` or implement this method 71 | */ 72 | // eslint-disable-next-line require-yield 73 | async * _all (q: Query, options?: AbortOptions): AwaitIterable { 74 | throw new Error('._all is not implemented') 75 | } 76 | 77 | /** 78 | * Extending classes should override `queryKeys` or implement this method 79 | */ 80 | // eslint-disable-next-line require-yield 81 | async * _allKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 82 | throw new Error('._allKeys is not implemented') 83 | } 84 | 85 | query (q: Query, options?: AbortOptions): AwaitIterable { 86 | let it = this._all(q, options) 87 | 88 | if (q.prefix != null) { 89 | const prefix = q.prefix 90 | it = filter(it, (e) => e.key.toString().startsWith(prefix)) 91 | } 92 | 93 | if (Array.isArray(q.filters)) { 94 | it = q.filters.reduce((it, f) => filter(it, f), it) 95 | } 96 | 97 | if (Array.isArray(q.orders)) { 98 | it = q.orders.reduce((it, f) => sort(it, f), it) 99 | } 100 | 101 | if (q.offset != null) { 102 | let i = 0 103 | const offset = q.offset 104 | it = filter(it, () => i++ >= offset) 105 | } 106 | 107 | if (q.limit != null) { 108 | it = take(it, q.limit) 109 | } 110 | 111 | return it 112 | } 113 | 114 | queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 115 | let it = this._allKeys(q, options) 116 | 117 | if (q.prefix != null) { 118 | const prefix = q.prefix 119 | it = filter(it, (key) => 120 | key.toString().startsWith(prefix) 121 | ) 122 | } 123 | 124 | if (Array.isArray(q.filters)) { 125 | it = q.filters.reduce((it, f) => filter(it, f), it) 126 | } 127 | 128 | if (Array.isArray(q.orders)) { 129 | it = q.orders.reduce((it, f) => sort(it, f), it) 130 | } 131 | 132 | if (q.offset != null) { 133 | const offset = q.offset 134 | let i = 0 135 | it = filter(it, () => i++ >= offset) 136 | } 137 | 138 | if (q.limit != null) { 139 | it = take(it, q.limit) 140 | } 141 | 142 | return it 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /packages/datastore-core/src/black-hole.ts: -------------------------------------------------------------------------------- 1 | import { NotFoundError } from 'interface-store' 2 | import { BaseDatastore } from './base.js' 3 | import type { Pair, Query, KeyQuery } from 'interface-datastore' 4 | import type { Key } from 'interface-datastore/key' 5 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 6 | 7 | export class BlackHoleDatastore extends BaseDatastore { 8 | put (key: Key, value: Uint8Array, options?: AbortOptions): Await { 9 | options?.signal?.throwIfAborted() 10 | return key 11 | } 12 | 13 | get (key: Key, options?: AbortOptions): Await { 14 | options?.signal?.throwIfAborted() 15 | throw new NotFoundError() 16 | } 17 | 18 | has (key: Key, options?: AbortOptions): Await { 19 | options?.signal?.throwIfAborted() 20 | return false 21 | } 22 | 23 | delete (key: Key, options?: AbortOptions): Await { 24 | options?.signal?.throwIfAborted() 25 | } 26 | 27 | // eslint-disable-next-line require-yield 28 | * _all (q: Query, options?: AbortOptions): AwaitIterable { 29 | options?.signal?.throwIfAborted() 30 | } 31 | 32 | // eslint-disable-next-line require-yield 33 | * _allKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 34 | options?.signal?.throwIfAborted() 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /packages/datastore-core/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @packageDocumentation 3 | * 4 | * Various Datastore implementations are available. 5 | * 6 | * ## Implementations 7 | * 8 | * - Mount: [`src/mount`](./src/mount.ts) 9 | * - Keytransform: [`src/keytransform`](./src/keytransform.ts) 10 | * - Sharding: [`src/sharding`](./src/sharding.ts) 11 | * - Tiered: [`src/tiered`](./src/tirered.ts) 12 | * - Namespace: [`src/namespace`](./src/namespace.ts) 13 | * - BlackHole: [`src/black-hole`](./src/black-hole.ts) 14 | * 15 | * @example BaseDatastore 16 | * 17 | * An base store is made available to make implementing your own datastore easier: 18 | * 19 | * ```javascript 20 | * import { BaseDatastore } from 'datastore-core' 21 | * 22 | * class MyDatastore extends BaseDatastore { 23 | * constructor () { 24 | * super() 25 | * } 26 | * 27 | * async put (key, val) { 28 | * // your implementation here 29 | * } 30 | * 31 | * async get (key) { 32 | * // your implementation here 33 | * } 34 | * 35 | * // etc... 36 | * } 37 | * ``` 38 | * 39 | * See the [MemoryDatastore](./src/memory.js) for an example of how it is used. 40 | * 41 | * @example Wrapping Stores 42 | * 43 | * ```js 44 | * import { Key } from 'interface-datastore' 45 | * import { 46 | * MemoryStore, 47 | * MountStore 48 | * } from 'datastore-core' 49 | * 50 | * const store = new MountStore({prefix: new Key('/a'), datastore: new MemoryStore()}) 51 | * ``` 52 | * 53 | * @example BlackHoleDatastore 54 | * 55 | * A datastore that does not store any data. 56 | * 57 | * ```js 58 | * import { BlackHoleDatastore } from 'datastore-core/black-hole' 59 | * 60 | * const store = new BlackHoleDatastore() 61 | * ``` 62 | */ 63 | 64 | import * as shard from './shard.js' 65 | import type { Key } from 'interface-datastore' 66 | 67 | export { BaseDatastore } from './base.js' 68 | export { MemoryDatastore } from './memory.js' 69 | export { KeyTransformDatastore } from './keytransform.js' 70 | export { ShardingDatastore } from './sharding.js' 71 | export { MountDatastore } from './mount.js' 72 | export { TieredDatastore } from './tiered.js' 73 | export { NamespaceDatastore } from './namespace.js' 74 | 75 | export { shard } 76 | 77 | export interface Shard { 78 | name: string 79 | param: number 80 | readonly _padding: string 81 | fun(s: string): string 82 | toString(): string 83 | } 84 | 85 | export interface KeyTransform { 86 | convert(key: Key): Key 87 | invert(key: Key): Key 88 | } 89 | -------------------------------------------------------------------------------- /packages/datastore-core/src/memory.ts: -------------------------------------------------------------------------------- 1 | import { Key } from 'interface-datastore/key' 2 | import { NotFoundError } from 'interface-store' 3 | import { BaseDatastore } from './base.js' 4 | import type { KeyQuery, Pair, Query } from 'interface-datastore' 5 | import type { AbortOptions, Await, AwaitIterable } from 'interface-store' 6 | 7 | export class MemoryDatastore extends BaseDatastore { 8 | private readonly data: Map 9 | 10 | constructor () { 11 | super() 12 | 13 | this.data = new Map() 14 | } 15 | 16 | put (key: Key, val: Uint8Array, options?: AbortOptions): Await { 17 | options?.signal?.throwIfAborted() 18 | 19 | this.data.set(key.toString(), val) 20 | 21 | return key 22 | } 23 | 24 | get (key: Key, options?: AbortOptions): Await { 25 | options?.signal?.throwIfAborted() 26 | 27 | const result = this.data.get(key.toString()) 28 | 29 | if (result == null) { 30 | throw new NotFoundError() 31 | } 32 | 33 | return result 34 | } 35 | 36 | has (key: Key, options?: AbortOptions): Await { 37 | options?.signal?.throwIfAborted() 38 | return this.data.has(key.toString()) 39 | } 40 | 41 | delete (key: Key, options?: AbortOptions): Await { 42 | options?.signal?.throwIfAborted() 43 | this.data.delete(key.toString()) 44 | } 45 | 46 | * _all (q: Query, options?: AbortOptions): AwaitIterable { 47 | options?.signal?.throwIfAborted() 48 | for (const [key, value] of this.data.entries()) { 49 | yield { key: new Key(key), value } 50 | options?.signal?.throwIfAborted() 51 | } 52 | } 53 | 54 | * _allKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 55 | options?.signal?.throwIfAborted() 56 | for (const key of this.data.keys()) { 57 | yield new Key(key) 58 | options?.signal?.throwIfAborted() 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /packages/datastore-core/src/namespace.ts: -------------------------------------------------------------------------------- 1 | import { Key } from 'interface-datastore' 2 | import map from 'it-map' 3 | import { KeyTransformDatastore } from './keytransform.js' 4 | import type { Datastore, Query, Pair, KeyQuery } from 'interface-datastore' 5 | import type { AbortOptions } from 'interface-store' 6 | 7 | /** 8 | * Wraps a given datastore into a keytransform which 9 | * makes a given prefix transparent. 10 | * 11 | * For example, if the prefix is `new Key(/hello)` a call 12 | * to `store.put(new Key('/world'), mydata)` would store the data under 13 | * `/hello/world`. 14 | */ 15 | export class NamespaceDatastore extends KeyTransformDatastore { 16 | private readonly iChild: Datastore 17 | private readonly iKey: Key 18 | 19 | constructor (child: Datastore, prefix: Key) { 20 | super(child, { 21 | convert (key) { 22 | return prefix.child(key) 23 | }, 24 | invert (key) { 25 | if (prefix.toString() === '/') { 26 | return key 27 | } 28 | 29 | if (!prefix.isAncestorOf(key)) { 30 | throw new Error(`Expected prefix: (${prefix.toString()}) in key: ${key.toString()}`) 31 | } 32 | 33 | return new Key(key.toString().slice(prefix.toString().length), false) 34 | } 35 | }) 36 | 37 | this.iChild = child 38 | this.iKey = prefix 39 | } 40 | 41 | query (q: Query, options?: AbortOptions): AsyncIterable { 42 | const query: Query = { 43 | ...q 44 | } 45 | 46 | query.filters = (query.filters ?? []).map(filter => { 47 | return ({ key, value }) => filter({ key: this.transform.invert(key), value }) 48 | }) 49 | 50 | const { prefix } = q 51 | if (prefix != null && prefix !== '/') { 52 | delete query.prefix 53 | query.filters.push(({ key }) => { 54 | return this.transform.invert(key).toString().startsWith(prefix) 55 | }) 56 | } 57 | 58 | if (query.orders != null) { 59 | query.orders = query.orders.map(order => { 60 | return (a, b) => order( 61 | { key: this.transform.invert(a.key), value: a.value }, 62 | { key: this.transform.invert(b.key), value: b.value } 63 | ) 64 | }) 65 | } 66 | 67 | query.filters.unshift(({ key }) => this.iKey.isAncestorOf(key)) 68 | 69 | return map(this.iChild.query(query, options), ({ key, value }) => { 70 | return { 71 | key: this.transform.invert(key), 72 | value 73 | } 74 | }) 75 | } 76 | 77 | queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { 78 | const query = { 79 | ...q 80 | } 81 | 82 | query.filters = (query.filters ?? []).map(filter => { 83 | return (key) => filter(this.transform.invert(key)) 84 | }) 85 | 86 | const { prefix } = q 87 | if (prefix != null && prefix !== '/') { 88 | delete query.prefix 89 | query.filters.push((key) => { 90 | return this.transform.invert(key).toString().startsWith(prefix) 91 | }) 92 | } 93 | 94 | if (query.orders != null) { 95 | query.orders = query.orders.map(order => { 96 | return (a, b) => order( 97 | this.transform.invert(a), 98 | this.transform.invert(b) 99 | ) 100 | }) 101 | } 102 | 103 | query.filters.unshift(key => this.iKey.isAncestorOf(key)) 104 | 105 | return map(this.iChild.queryKeys(query, options), key => { 106 | return this.transform.invert(key) 107 | }) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /packages/datastore-core/src/shard.ts: -------------------------------------------------------------------------------- 1 | import { Key } from 'interface-datastore/key' 2 | import type { Shard } from './index.js' 3 | import type { Datastore } from 'interface-datastore' 4 | 5 | export const PREFIX = '/repo/flatfs/shard/' 6 | export const SHARDING_FN = 'SHARDING' 7 | 8 | export class ShardBase implements Shard { 9 | public param: number 10 | public name: string 11 | public _padding: string 12 | 13 | constructor (param: number) { 14 | this.param = param 15 | this.name = 'base' 16 | this._padding = '' 17 | } 18 | 19 | fun (s: string): string { 20 | return 'implement me' 21 | } 22 | 23 | toString (): string { 24 | return `${PREFIX}v1/${this.name}/${this.param}` 25 | } 26 | } 27 | 28 | export class Prefix extends ShardBase { 29 | constructor (prefixLen: number) { 30 | super(prefixLen) 31 | this._padding = ''.padStart(prefixLen, '_') 32 | this.name = 'prefix' 33 | } 34 | 35 | fun (noslash: string): string { 36 | return (noslash + this._padding).slice(0, this.param) 37 | } 38 | } 39 | 40 | export class Suffix extends ShardBase { 41 | constructor (suffixLen: number) { 42 | super(suffixLen) 43 | 44 | this._padding = ''.padStart(suffixLen, '_') 45 | this.name = 'suffix' 46 | } 47 | 48 | fun (noslash: string): string { 49 | const s = this._padding + noslash 50 | return s.slice(s.length - this.param) 51 | } 52 | } 53 | 54 | export class NextToLast extends ShardBase { 55 | constructor (suffixLen: number) { 56 | super(suffixLen) 57 | this._padding = ''.padStart(suffixLen + 1, '_') 58 | this.name = 'next-to-last' 59 | } 60 | 61 | fun (noslash: string): string { 62 | const s = this._padding + noslash 63 | const offset = s.length - this.param - 1 64 | return s.slice(offset, offset + this.param) 65 | } 66 | } 67 | 68 | /** 69 | * Convert a given string to the matching sharding function 70 | */ 71 | export function parseShardFun (str: string): Shard { 72 | str = str.trim() 73 | 74 | if (str.length === 0) { 75 | throw new Error('empty shard string') 76 | } 77 | 78 | if (!str.startsWith(PREFIX)) { 79 | throw new Error(`invalid or no path prefix: ${str}`) 80 | } 81 | 82 | const parts = str.slice(PREFIX.length).split('/') 83 | const version = parts[0] 84 | 85 | if (version !== 'v1') { 86 | throw new Error(`expect 'v1' version, got '${version}'`) 87 | } 88 | 89 | const name = parts[1] 90 | 91 | if (parts[2] == null || parts[2] === '') { 92 | throw new Error('missing param') 93 | } 94 | 95 | const param = parseInt(parts[2], 10) 96 | 97 | switch (name) { 98 | case 'prefix': 99 | return new Prefix(param) 100 | case 'suffix': 101 | return new Suffix(param) 102 | case 'next-to-last': 103 | return new NextToLast(param) 104 | default: 105 | throw new Error(`unkown sharding function: ${name}`) 106 | } 107 | } 108 | 109 | export const readShardFun = async (path: string | Uint8Array, store: Datastore): Promise => { 110 | const key = new Key(path).child(new Key(SHARDING_FN)) 111 | // @ts-expect-error not all stores have this 112 | const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store) 113 | const res = await get(key) 114 | return parseShardFun(new TextDecoder().decode(res ?? '').trim()) 115 | } 116 | -------------------------------------------------------------------------------- /packages/datastore-core/src/tiered.ts: -------------------------------------------------------------------------------- 1 | import { logger } from '@libp2p/logger' 2 | import { NotFoundError } from 'interface-store' 3 | import { BaseDatastore } from './base.js' 4 | import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' 5 | import type { AbortOptions, AwaitIterable } from 'interface-store' 6 | 7 | const log = logger('datastore:core:tiered') 8 | 9 | /** 10 | * A datastore that can combine multiple stores. Puts and deletes 11 | * will write through to all datastores. Has and get will 12 | * try each store sequentially. Query will always try the 13 | * last one first. 14 | * 15 | */ 16 | export class TieredDatastore extends BaseDatastore { 17 | private readonly stores: Datastore[] 18 | 19 | constructor (stores: Datastore[]) { 20 | super() 21 | 22 | this.stores = stores.slice() 23 | } 24 | 25 | async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { 26 | await Promise.all( 27 | this.stores.map(async store => { 28 | await store.put(key, value, options) 29 | }) 30 | ) 31 | 32 | return key 33 | } 34 | 35 | async get (key: Key, options?: AbortOptions): Promise { 36 | let error: Error | undefined 37 | 38 | for (const store of this.stores) { 39 | try { 40 | const res = await store.get(key, options) 41 | 42 | if (res != null) { 43 | return res 44 | } 45 | } catch (err: any) { 46 | error = err 47 | log.error(err) 48 | } 49 | } 50 | 51 | throw error ?? new NotFoundError() 52 | } 53 | 54 | async has (key: Key, options?: AbortOptions): Promise { 55 | for (const s of this.stores) { 56 | if (await s.has(key, options)) { 57 | return true 58 | } 59 | } 60 | 61 | return false 62 | } 63 | 64 | async delete (key: Key, options?: AbortOptions): Promise { 65 | await Promise.all( 66 | this.stores.map(async store => { 67 | await store.delete(key, options) 68 | }) 69 | ) 70 | } 71 | 72 | async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 73 | for await (const pair of source) { 74 | await this.put(pair.key, pair.value, options) 75 | yield pair.key 76 | } 77 | } 78 | 79 | async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { 80 | for await (const key of source) { 81 | await this.delete(key, options) 82 | yield key 83 | } 84 | } 85 | 86 | batch (): Batch { 87 | const batches = this.stores.map(store => store.batch()) 88 | 89 | return { 90 | put: (key, value) => { 91 | batches.forEach(b => { b.put(key, value) }) 92 | }, 93 | delete: (key) => { 94 | batches.forEach(b => { b.delete(key) }) 95 | }, 96 | commit: async (options) => { 97 | for (const batch of batches) { 98 | await batch.commit(options) 99 | } 100 | } 101 | } 102 | } 103 | 104 | query (q: Query, options?: AbortOptions): AwaitIterable { 105 | return this.stores[this.stores.length - 1].query(q, options) 106 | } 107 | 108 | queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { 109 | return this.stores[this.stores.length - 1].queryKeys(q, options) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /packages/datastore-core/src/utils.ts: -------------------------------------------------------------------------------- 1 | export const replaceStartWith = (s: string, r: string): string => { 2 | const matcher = new RegExp('^' + r) 3 | return s.replace(matcher, '') 4 | } 5 | -------------------------------------------------------------------------------- /packages/datastore-core/test/keytransform.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { Key } from 'interface-datastore/key' 5 | import all from 'it-all' 6 | import { KeyTransformDatastore } from '../src/keytransform.js' 7 | import { MemoryDatastore } from '../src/memory.js' 8 | 9 | describe('KeyTransformDatastore', () => { 10 | it('basic', async () => { 11 | const mStore = new MemoryDatastore() 12 | const transform = { 13 | convert (key: Key): Key { 14 | return new Key('/abc').child(key) 15 | }, 16 | invert (key: Key): Key { 17 | const l = key.list() 18 | if (l[0] !== 'abc') { 19 | throw new Error('missing prefix, convert failed?') 20 | } 21 | return Key.withNamespaces(l.slice(1)) 22 | } 23 | } 24 | 25 | const kStore = new KeyTransformDatastore(mStore, transform) 26 | 27 | const keys = [ 28 | 'foo', 29 | 'foo/bar', 30 | 'foo/bar/baz', 31 | 'foo/barb', 32 | 'foo/bar/bazb', 33 | 'foo/bar/baz/barb' 34 | ].map((s) => new Key(s)) 35 | await Promise.all(keys.map(async (key) => { await kStore.put(key, key.uint8Array()) })) 36 | const kResults = Promise.all(keys.map(async (key) => kStore.get(key))) 37 | const mResults = Promise.all(keys.map(async (key) => mStore.get(new Key('abc').child(key)))) 38 | const results = await Promise.all([kResults, mResults]) 39 | expect(results[0]).to.eql(results[1]) 40 | 41 | const mRes = await all(mStore.query({})) 42 | const kRes = await all(kStore.query({})) 43 | expect(kRes).to.have.length(mRes.length) 44 | 45 | mRes.forEach((a, i) => { 46 | const kA = a.key 47 | const kB = kRes[i].key 48 | expect(transform.invert(kA)).to.eql(kB) 49 | expect(kA).to.eql(transform.convert(kB)) 50 | }) 51 | }) 52 | }) 53 | -------------------------------------------------------------------------------- /packages/datastore-core/test/memory.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 4 | import { MemoryDatastore } from '../src/memory.js' 5 | 6 | describe('Memory', () => { 7 | describe('interface-datastore', () => { 8 | interfaceDatastoreTests({ 9 | setup () { 10 | return new MemoryDatastore() 11 | }, 12 | teardown () {} 13 | }) 14 | }) 15 | }) 16 | -------------------------------------------------------------------------------- /packages/datastore-core/test/shard.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { 5 | Prefix, 6 | Suffix, 7 | NextToLast, 8 | parseShardFun 9 | } from '../src/shard.js' 10 | 11 | describe('shard', () => { 12 | it('prefix', () => { 13 | expect( 14 | new Prefix(2).fun('hello') 15 | ).to.eql( 16 | 'he' 17 | ) 18 | expect( 19 | new Prefix(2).fun('h') 20 | ).to.eql( 21 | 'h_' 22 | ) 23 | 24 | expect( 25 | new Prefix(2).toString() 26 | ).to.eql( 27 | '/repo/flatfs/shard/v1/prefix/2' 28 | ) 29 | }) 30 | 31 | it('suffix', () => { 32 | expect( 33 | new Suffix(2).fun('hello') 34 | ).to.eql( 35 | 'lo' 36 | ) 37 | expect( 38 | new Suffix(2).fun('h') 39 | ).to.eql( 40 | '_h' 41 | ) 42 | 43 | expect( 44 | new Suffix(2).toString() 45 | ).to.eql( 46 | '/repo/flatfs/shard/v1/suffix/2' 47 | ) 48 | }) 49 | 50 | it('next-to-last', () => { 51 | expect( 52 | new NextToLast(2).fun('hello') 53 | ).to.eql( 54 | 'll' 55 | ) 56 | expect( 57 | new NextToLast(3).fun('he') 58 | ).to.eql( 59 | '__h' 60 | ) 61 | 62 | expect( 63 | new NextToLast(2).toString() 64 | ).to.eql( 65 | '/repo/flatfs/shard/v1/next-to-last/2' 66 | ) 67 | }) 68 | }) 69 | 70 | describe('parsesShardFun', () => { 71 | it('errors', () => { 72 | const errors = [ 73 | '', 74 | 'shard/v1/next-to-last/2', 75 | '/repo/flatfs/shard/v2/next-to-last/2', 76 | '/repo/flatfs/shard/v1/other/2', 77 | '/repo/flatfs/shard/v1/next-to-last/' 78 | ] 79 | 80 | errors.forEach((input) => { 81 | expect( 82 | () => parseShardFun(input) 83 | ).to.throw() 84 | }) 85 | }) 86 | 87 | it('success', () => { 88 | const success = [ 89 | 'prefix', 90 | 'suffix', 91 | 'next-to-last' 92 | ] 93 | 94 | success.forEach((name) => { 95 | const n = Math.floor(Math.random() * 100) 96 | expect( 97 | parseShardFun( 98 | `/repo/flatfs/shard/v1/${name}/${n}` 99 | ).name 100 | ).to.eql(name) 101 | }) 102 | }) 103 | }) 104 | -------------------------------------------------------------------------------- /packages/datastore-core/test/sharding.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { Key } from 'interface-datastore/key' 5 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 6 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 7 | import { toString as uint8ArrayToString } from 'uint8arrays/to-string' 8 | import { MemoryDatastore } from '../src/memory.js' 9 | import { 10 | NextToLast, 11 | SHARDING_FN 12 | } from '../src/shard.js' 13 | import { 14 | ShardingDatastore 15 | } from '../src/sharding.js' 16 | 17 | describe('ShardingDatastore', () => { 18 | it('create', async () => { 19 | const ms = new MemoryDatastore() 20 | const shard = new NextToLast(2) 21 | const store = new ShardingDatastore(ms, shard) 22 | await store.open() 23 | const res = await Promise.all([ 24 | store.get(new Key(SHARDING_FN)) 25 | ]) 26 | expect(uint8ArrayToString(res[0])).to.eql(shard.toString() + '\n') 27 | }) 28 | 29 | it('open - empty', () => { 30 | const ms = new MemoryDatastore() 31 | // @ts-expect-error shard is missing 32 | const store = new ShardingDatastore(ms) 33 | return expect(store.open()) 34 | .to.eventually.be.rejected() 35 | .with.property('name', 'OpenFailedError') 36 | }) 37 | 38 | it('open - existing', () => { 39 | const ms = new MemoryDatastore() 40 | const shard = new NextToLast(2) 41 | const store = new ShardingDatastore(ms, shard) 42 | 43 | return expect(store.open()).to.eventually.be.fulfilled() 44 | }) 45 | 46 | it('basics', async () => { 47 | const ms = new MemoryDatastore() 48 | const shard = new NextToLast(2) 49 | const store = new ShardingDatastore(ms, shard) 50 | await store.open() 51 | await store.put(new Key('hello'), uint8ArrayFromString('test')) 52 | const res = await ms.get(new Key('ll').child(new Key('hello'))) 53 | expect(res).to.eql(uint8ArrayFromString('test')) 54 | }) 55 | 56 | describe('interface-datastore', () => { 57 | interfaceDatastoreTests({ 58 | setup () { 59 | const shard = new NextToLast(2) 60 | return new ShardingDatastore(new MemoryDatastore(), shard) 61 | }, 62 | teardown () { } 63 | }) 64 | }) 65 | }) 66 | -------------------------------------------------------------------------------- /packages/datastore-core/test/tiered.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { Key } from 'interface-datastore/key' 5 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 6 | import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' 7 | import { MemoryDatastore } from '../src/memory.js' 8 | import { TieredDatastore } from '../src/tiered.js' 9 | import type { Datastore } from 'interface-datastore' 10 | 11 | /** 12 | * @typedef {import('interface-datastore').Datastore} Datastore 13 | */ 14 | 15 | describe('Tiered', () => { 16 | describe('all stores', () => { 17 | const ms: Datastore[] = [] 18 | let store: TieredDatastore 19 | beforeEach(() => { 20 | ms.push(new MemoryDatastore()) 21 | ms.push(new MemoryDatastore()) 22 | store = new TieredDatastore(ms) 23 | }) 24 | 25 | it('put', async () => { 26 | const k = new Key('hello') 27 | const v = uint8ArrayFromString('world') 28 | await store.put(k, v) 29 | const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) 30 | res.forEach((val) => { 31 | expect(val).to.be.eql(v) 32 | }) 33 | }) 34 | 35 | it('get and has, where available', async () => { 36 | const k = new Key('hello') 37 | const v = uint8ArrayFromString('world') 38 | await ms[1].put(k, v) 39 | const val = await store.get(k) 40 | expect(val).to.be.eql(v) 41 | const exists = await store.has(k) 42 | expect(exists).to.be.eql(true) 43 | }) 44 | 45 | it('has - key not found', async () => { 46 | expect(await store.has(new Key('hello1'))).to.be.eql(false) 47 | }) 48 | 49 | it('has and delete', async () => { 50 | const k = new Key('hello') 51 | const v = uint8ArrayFromString('world') 52 | await store.put(k, v) 53 | let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) 54 | expect(res).to.be.eql([true, true]) 55 | await store.delete(k) 56 | res = await Promise.all([ms[0].has(k), ms[1].has(k)]) 57 | expect(res).to.be.eql([false, false]) 58 | }) 59 | }) 60 | 61 | describe('inteface-datastore-single', () => { 62 | interfaceDatastoreTests({ 63 | setup () { 64 | return new TieredDatastore([ 65 | new MemoryDatastore(), 66 | new MemoryDatastore() 67 | ]) 68 | }, 69 | teardown () { } 70 | }) 71 | }) 72 | }) 73 | -------------------------------------------------------------------------------- /packages/datastore-core/test/utils.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import * as utils from '../src/utils.js' 5 | 6 | describe('utils', () => { 7 | it('replaceStartWith', () => { 8 | expect( 9 | utils.replaceStartWith('helloworld', 'hello') 10 | ).to.eql( 11 | 'world' 12 | ) 13 | 14 | expect( 15 | utils.replaceStartWith('helloworld', 'world') 16 | ).to.eql( 17 | 'helloworld' 18 | ) 19 | }) 20 | }) 21 | -------------------------------------------------------------------------------- /packages/datastore-core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../interface-datastore" 13 | }, 14 | { 15 | "path": "../interface-datastore-tests" 16 | }, 17 | { 18 | "path": "../interface-store" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /packages/datastore-core/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts", 5 | "./src/base.ts", 6 | "./src/black-hole.ts", 7 | "./src/errors.ts", 8 | "./src/keytransform.ts", 9 | "./src/memory.ts", 10 | "./src/mount.ts", 11 | "./src/namespace.ts", 12 | "./src/shard.ts", 13 | "./src/sharding.ts", 14 | "./src/tiered.ts" 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /packages/datastore-fs/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/datastore-fs/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/datastore-fs/README.md: -------------------------------------------------------------------------------- 1 | # datastore-fs 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Datastore implementation with file system backend 9 | 10 | # About 11 | 12 | 26 | 27 | A Datastore implementation with a file system backend. 28 | 29 | ## Example 30 | 31 | ```js 32 | import { FsDatastore } from 'datastore-fs' 33 | 34 | const store = new FsDatastore('path/to/store') 35 | ``` 36 | 37 | # Install 38 | 39 | ```console 40 | $ npm i datastore-fs 41 | ``` 42 | 43 | # API Docs 44 | 45 | - 46 | 47 | # License 48 | 49 | Licensed under either of 50 | 51 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/datastore-fs/LICENSE-APACHE) / ) 52 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/datastore-fs/LICENSE-MIT) / ) 53 | 54 | # Contribute 55 | 56 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 57 | 58 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 59 | 60 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 61 | 62 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 63 | 64 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 65 | -------------------------------------------------------------------------------- /packages/datastore-fs/test/fixtures/writer-worker.ts: -------------------------------------------------------------------------------- 1 | import { Key } from 'interface-datastore' 2 | // @ts-expect-error types are broken: https://github.com/andywer/threads.js/pull/470 3 | import { expose } from 'threads/worker' 4 | import { FsDatastore } from '../../src/index.js' 5 | 6 | let fs: FsDatastore 7 | expose({ 8 | async isReady (path: string) { 9 | fs = new FsDatastore(path) 10 | try { 11 | await fs.open() 12 | return true 13 | } catch (err) { 14 | // eslint-disable-next-line no-console 15 | console.error('Error opening blockstore', err) 16 | throw err 17 | } 18 | }, 19 | async put (keyString: string, value: Uint8Array) { 20 | const key = new Key(keyString) 21 | try { 22 | return await fs.put(key, value) 23 | } catch (err) { 24 | // eslint-disable-next-line no-console 25 | console.error('Error putting block', err) 26 | throw err 27 | } 28 | } 29 | }) 30 | -------------------------------------------------------------------------------- /packages/datastore-fs/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../datastore-core" 13 | }, 14 | { 15 | "path": "../interface-datastore" 16 | }, 17 | { 18 | "path": "../interface-datastore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/datastore-fs/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/datastore-idb/.aegir.js: -------------------------------------------------------------------------------- 1 | export default { 2 | bundlesize: { maxSize: '16kB' } 3 | } 4 | -------------------------------------------------------------------------------- /packages/datastore-idb/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/datastore-idb/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/datastore-idb/README.md: -------------------------------------------------------------------------------- 1 | # datastore-idb 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Datastore implementation with IndexedDB backend. 9 | 10 | # About 11 | 12 | 26 | 27 | A Datastore implementation for browsers that stores data in [IndexedDB](https://developer.mozilla.org/en-US/docs/Web/API/IndexedDB_API). 28 | 29 | ## Example 30 | 31 | ```js 32 | import { IDBDatastore } from 'datastore-idb' 33 | 34 | const store = new IDBDatastore('path/to/store') 35 | ``` 36 | 37 | # Install 38 | 39 | ```console 40 | $ npm i datastore-idb 41 | ``` 42 | 43 | ## Browser ` 49 | ``` 50 | 51 | # API Docs 52 | 53 | - 54 | 55 | # License 56 | 57 | Licensed under either of 58 | 59 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/datastore-idb/LICENSE-APACHE) / ) 60 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/datastore-idb/LICENSE-MIT) / ) 61 | 62 | # Contribute 63 | 64 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 65 | 66 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 67 | 68 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 69 | 70 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 71 | 72 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 73 | -------------------------------------------------------------------------------- /packages/datastore-idb/benchmark/datastore-level/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmarks-datastore-level", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "private": true, 6 | "type": "module", 7 | "scripts": { 8 | "clean": "aegir clean", 9 | "build": "aegir build --bundle false", 10 | "lint": "aegir lint", 11 | "dep-check": "aegir dep-check", 12 | "start": "npm run build && playwright-test dist/src/index.js --runner benchmark" 13 | }, 14 | "devDependencies": { 15 | "datastore-level": "^10.0.1", 16 | "datastore-idb": "file:../../", 17 | "multiformats": "^11.0.1", 18 | "playwright-test": "^8.2.0", 19 | "tinybench": "^2.4.0" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /packages/datastore-idb/benchmark/datastore-level/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist", 5 | "target": "ES2022", 6 | "module": "ES2022", 7 | "lib": ["ES2022", "DOM", "DOM.Iterable"] 8 | }, 9 | "include": [ 10 | "src", 11 | "test" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /packages/datastore-idb/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datastore-idb", 3 | "version": "3.0.3", 4 | "description": "Datastore implementation with IndexedDB backend.", 5 | "author": "Hugo Dias ", 6 | "license": "Apache-2.0 OR MIT", 7 | "homepage": "https://github.com/ipfs/js-stores/tree/main/packages/datastore-idb#readme", 8 | "repository": { 9 | "type": "git", 10 | "url": "git+https://github.com/ipfs/js-stores.git" 11 | }, 12 | "bugs": { 13 | "url": "https://github.com/ipfs/js-stores/issues" 14 | }, 15 | "publishConfig": { 16 | "access": "public", 17 | "provenance": true 18 | }, 19 | "keywords": [ 20 | "browser", 21 | "datastore", 22 | "idb", 23 | "interface", 24 | "ipfs", 25 | "key-value" 26 | ], 27 | "type": "module", 28 | "types": "./dist/src/index.d.ts", 29 | "files": [ 30 | "src", 31 | "dist", 32 | "!dist/test", 33 | "!**/*.tsbuildinfo" 34 | ], 35 | "exports": { 36 | ".": { 37 | "types": "./dist/src/index.d.ts", 38 | "import": "./dist/src/index.js" 39 | } 40 | }, 41 | "release": { 42 | "branches": [ 43 | "main" 44 | ], 45 | "plugins": [ 46 | [ 47 | "@semantic-release/commit-analyzer", 48 | { 49 | "preset": "conventionalcommits", 50 | "releaseRules": [ 51 | { 52 | "breaking": true, 53 | "release": "major" 54 | }, 55 | { 56 | "revert": true, 57 | "release": "patch" 58 | }, 59 | { 60 | "type": "feat", 61 | "release": "minor" 62 | }, 63 | { 64 | "type": "fix", 65 | "release": "patch" 66 | }, 67 | { 68 | "type": "docs", 69 | "release": "patch" 70 | }, 71 | { 72 | "type": "test", 73 | "release": "patch" 74 | }, 75 | { 76 | "type": "deps", 77 | "release": "patch" 78 | }, 79 | { 80 | "scope": "no-release", 81 | "release": false 82 | } 83 | ] 84 | } 85 | ], 86 | [ 87 | "@semantic-release/release-notes-generator", 88 | { 89 | "preset": "conventionalcommits", 90 | "presetConfig": { 91 | "types": [ 92 | { 93 | "type": "feat", 94 | "section": "Features" 95 | }, 96 | { 97 | "type": "fix", 98 | "section": "Bug Fixes" 99 | }, 100 | { 101 | "type": "chore", 102 | "section": "Trivial Changes" 103 | }, 104 | { 105 | "type": "docs", 106 | "section": "Documentation" 107 | }, 108 | { 109 | "type": "deps", 110 | "section": "Dependencies" 111 | }, 112 | { 113 | "type": "test", 114 | "section": "Tests" 115 | } 116 | ] 117 | } 118 | } 119 | ], 120 | "@semantic-release/changelog", 121 | "@semantic-release/npm", 122 | "@semantic-release/github", 123 | [ 124 | "@semantic-release/git", 125 | { 126 | "assets": [ 127 | "CHANGELOG.md", 128 | "package.json" 129 | ] 130 | } 131 | ] 132 | ] 133 | }, 134 | "scripts": { 135 | "test": "aegir test -t browser -t webworker", 136 | "test:browser": "aegir test -t browser", 137 | "test:webworker": "aegir test -t webworker", 138 | "build": "aegir build", 139 | "lint": "aegir lint", 140 | "dep-check": "aegir dep-check", 141 | "release": "aegir release" 142 | }, 143 | "dependencies": { 144 | "datastore-core": "^10.0.0", 145 | "idb": "^8.0.3", 146 | "interface-datastore": "^8.0.0", 147 | "interface-store": "^6.0.0", 148 | "it-filter": "^3.1.3", 149 | "it-sort": "^3.0.8", 150 | "race-signal": "^1.1.3" 151 | }, 152 | "devDependencies": { 153 | "aegir": "^47.0.16", 154 | "interface-datastore-tests": "^6.0.0" 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /packages/datastore-idb/test/index.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { MountDatastore } from 'datastore-core' 5 | import { Key } from 'interface-datastore' 6 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 7 | import { IDBDatastore } from '../src/index.js' 8 | 9 | describe('IndexedDB Datastore', function () { 10 | describe('interface-datastore (idb)', () => { 11 | interfaceDatastoreTests({ 12 | async setup () { 13 | const store = new IDBDatastore(`hello-${Date.now()}`) 14 | await store.open() 15 | return store 16 | }, 17 | async teardown (store) { 18 | await store.close() 19 | await store.destroy() 20 | } 21 | }) 22 | }) 23 | 24 | describe('interface-datastore (mount(idb, idb, idb))', () => { 25 | interfaceDatastoreTests({ 26 | async setup () { 27 | const one = new IDBDatastore(`one-${Date.now()}`) 28 | const two = new IDBDatastore(`two-${Date.now()}`) 29 | const three = new IDBDatastore(`three-${Date.now()}`) 30 | 31 | await one.open() 32 | await two.open() 33 | await three.open() 34 | 35 | const d = new MountDatastore([ 36 | { 37 | prefix: new Key('/a'), 38 | datastore: one 39 | }, 40 | { 41 | prefix: new Key('/q'), 42 | datastore: two 43 | }, 44 | { 45 | prefix: new Key('/z'), 46 | datastore: three 47 | } 48 | ]) 49 | 50 | return d 51 | }, 52 | teardown () { 53 | 54 | } 55 | }) 56 | }) 57 | 58 | describe('concurrency', () => { 59 | let store: IDBDatastore 60 | 61 | beforeEach(async () => { 62 | store = new IDBDatastore('hello') 63 | await store.open() 64 | }) 65 | 66 | it('should not explode under unreasonable load', function (done) { 67 | this.timeout(10000) 68 | 69 | const updater = setInterval(async () => { 70 | try { 71 | const key = new Key(`/a-${Date.now()}`) 72 | 73 | await store.put(key, Uint8Array.from([0, 1, 2, 3])) 74 | await store.has(key) 75 | await store.get(key) 76 | } catch (err) { 77 | clearInterval(updater) 78 | clearInterval(mutatorQuery) 79 | clearInterval(readOnlyQuery) 80 | done(err) 81 | } 82 | }, 0) 83 | 84 | const mutatorQuery = setInterval(async () => { 85 | try { 86 | for await (const { key } of store.query({})) { 87 | await store.get(key) 88 | 89 | const otherKey = new Key(`/b-${Date.now()}`) 90 | const otherValue = Uint8Array.from([0, 1, 2, 3]) 91 | await store.put(otherKey, otherValue) 92 | const res = await store.get(otherKey) 93 | expect(res).to.deep.equal(otherValue) 94 | } 95 | } catch (err) { 96 | clearInterval(updater) 97 | clearInterval(mutatorQuery) 98 | clearInterval(readOnlyQuery) 99 | done(err) 100 | } 101 | }, 0) 102 | 103 | const readOnlyQuery = setInterval(async () => { 104 | try { 105 | for await (const { key } of store.query({})) { 106 | await store.has(key) 107 | } 108 | } catch (err) { 109 | clearInterval(updater) 110 | clearInterval(mutatorQuery) 111 | clearInterval(readOnlyQuery) 112 | done(err) 113 | } 114 | }, 0) 115 | 116 | setTimeout(() => { 117 | clearInterval(updater) 118 | clearInterval(mutatorQuery) 119 | clearInterval(readOnlyQuery) 120 | done() 121 | }, 5000) 122 | }) 123 | }) 124 | }) 125 | -------------------------------------------------------------------------------- /packages/datastore-idb/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../datastore-core" 13 | }, 14 | { 15 | "path": "../interface-datastore" 16 | }, 17 | { 18 | "path": "../interface-datastore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/datastore-idb/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/datastore-level/.aegir.js: -------------------------------------------------------------------------------- 1 | /** @type {import('aegir').PartialOptions} */ 2 | export default { 3 | build: { 4 | bundlesizeMax: '67KB' 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/datastore-level/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/datastore-level/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/datastore-level/test/browser.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { MountDatastore } from 'datastore-core' 4 | import { Key } from 'interface-datastore/key' 5 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 6 | import { LevelDatastore } from '../src/index.js' 7 | 8 | describe('LevelDatastore', () => { 9 | describe('interface-datastore (leveljs)', () => { 10 | interfaceDatastoreTests({ 11 | setup: () => new LevelDatastore(`hello-${Math.random()}`), 12 | teardown: () => {} 13 | }) 14 | }) 15 | 16 | describe('interface-datastore (mount(leveljs, leveljs, leveljs))', () => { 17 | interfaceDatastoreTests({ 18 | setup () { 19 | return new MountDatastore([{ 20 | prefix: new Key('/a'), 21 | datastore: new LevelDatastore(`one-${Math.random()}`) 22 | }, { 23 | prefix: new Key('/q'), 24 | datastore: new LevelDatastore(`two-${Math.random()}`) 25 | }, { 26 | prefix: new Key('/z'), 27 | datastore: new LevelDatastore(`three-${Math.random()}`) 28 | }]) 29 | }, 30 | teardown () {} 31 | }) 32 | }) 33 | }) 34 | -------------------------------------------------------------------------------- /packages/datastore-level/test/fixtures/test-level-iterator-destroy.ts: -------------------------------------------------------------------------------- 1 | import { Key } from 'interface-datastore/key' 2 | import tempdir from 'ipfs-utils/src/temp-dir.js' 3 | import { LevelDatastore } from '../../src/index.js' 4 | 5 | async function testLevelIteratorDestroy (): Promise { 6 | const store = new LevelDatastore(tempdir()) 7 | await store.open() 8 | await store.put(new Key(`/test/key${Date.now()}`), new TextEncoder().encode(`TESTDATA${Date.now()}`)) 9 | for await (const d of store.query({})) { 10 | console.log(d) // eslint-disable-line no-console 11 | } 12 | } 13 | 14 | // Will exit with: 15 | // > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. 16 | // If iterator gets destroyed (in c++ land) and .end() was not called on it. 17 | void testLevelIteratorDestroy() 18 | -------------------------------------------------------------------------------- /packages/datastore-level/test/index.spec.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import { expect } from 'aegir/chai' 4 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 5 | import tempdir from 'ipfs-utils/src/temp-dir.js' 6 | import { Level } from 'level' 7 | import { MemoryLevel } from 'memory-level' 8 | import { LevelDatastore } from '../src/index.js' 9 | 10 | describe('LevelDatastore', () => { 11 | describe('initialization', () => { 12 | it('should default to a leveldown database', async () => { 13 | const levelStore = new LevelDatastore(`${tempdir()}/init-default-${Date.now()}`) 14 | await levelStore.open() 15 | 16 | expect(levelStore.db).to.be.an.instanceOf(Level) 17 | }) 18 | 19 | it('should be able to override the database', async () => { 20 | const levelStore = new LevelDatastore( 21 | // @ts-expect-error MemoryLevel does not implement the same interface as Level 22 | new MemoryLevel({ 23 | keyEncoding: 'utf8', 24 | valueEncoding: 'view' 25 | }) 26 | ) 27 | 28 | await levelStore.open() 29 | 30 | expect(levelStore.db).to.be.an.instanceOf(MemoryLevel) 31 | }) 32 | }) 33 | 34 | describe('interface-datastore MemoryLevel', () => { 35 | interfaceDatastoreTests({ 36 | async setup () { 37 | const store = new LevelDatastore( 38 | // @ts-expect-error MemoryLevel does not implement the same interface as Level 39 | new MemoryLevel({ 40 | keyEncoding: 'utf8', 41 | valueEncoding: 'view' 42 | }) 43 | ) 44 | await store.open() 45 | 46 | return store 47 | }, 48 | teardown () {} 49 | }) 50 | }) 51 | 52 | describe('interface-datastore Level', () => { 53 | interfaceDatastoreTests({ 54 | async setup () { 55 | const store = new LevelDatastore(tempdir()) 56 | await store.open() 57 | 58 | return store 59 | }, 60 | teardown () {} 61 | }) 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /packages/datastore-level/test/node.ts: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import childProcess from 'child_process' 4 | import path from 'path' 5 | import { expect } from 'aegir/chai' 6 | import { MountDatastore } from 'datastore-core' 7 | import { Key } from 'interface-datastore/key' 8 | import { interfaceDatastoreTests } from 'interface-datastore-tests' 9 | import tempdir from 'ipfs-utils/src/temp-dir.js' 10 | import { LevelDatastore } from '../src/index.js' 11 | 12 | describe('LevelDatastore', () => { 13 | describe('interface-datastore (leveldown)', () => { 14 | interfaceDatastoreTests({ 15 | async setup () { 16 | const store = new LevelDatastore(tempdir()) 17 | await store.open() 18 | 19 | return store 20 | }, 21 | teardown () {} 22 | }) 23 | }) 24 | 25 | describe('interface-datastore (mount(leveldown, leveldown, leveldown))', () => { 26 | interfaceDatastoreTests({ 27 | async setup () { 28 | return new MountDatastore( 29 | await Promise.all( 30 | ['/a', '/q', '/z'].map(async prefix => { 31 | const datastore = new LevelDatastore(tempdir()) 32 | await datastore.open() 33 | 34 | return { 35 | prefix: new Key(prefix), 36 | datastore 37 | } 38 | }) 39 | ) 40 | ) 41 | }, 42 | async teardown () { 43 | 44 | } 45 | }) 46 | }) 47 | 48 | // The `.end()` method MUST be called on LevelDB iterators or they remain open, 49 | // leaking memory. 50 | // 51 | // This test exposes this problem by causing an error to be thrown on process 52 | // exit when an iterator is open AND leveldb is not closed. 53 | // 54 | // Normally when leveldb is closed it'll automatically clean up open iterators 55 | // but if you don't close the store this error will occur: 56 | // 57 | // > Assertion failed: (ended_), function ~Iterator, file ../binding.cc, line 546. 58 | // 59 | // This is thrown by a destructor function for iterator objects that asserts 60 | // the iterator has ended before cleanup. 61 | // 62 | // https://github.com/Level/leveldown/blob/d3453fbde4d2a8aa04d9091101c25c999649069b/binding.cc#L545 63 | it('should not leave iterators open and leak memory', (done) => { 64 | const cp = childProcess.fork(path.join(process.cwd(), '/dist/test/fixtures/test-level-iterator-destroy'), { stdio: 'pipe' }) 65 | 66 | let out = '' 67 | const { stdout, stderr } = cp 68 | stdout?.on('data', d => { out = `${out}${d}` }) 69 | stderr?.on('data', d => { out = `${out}${d}` }) 70 | 71 | cp.on('exit', code => { 72 | expect(code).to.equal(0) 73 | expect(out).to.not.include('Assertion failed: (ended_)') 74 | done() 75 | }) 76 | }) 77 | }) 78 | -------------------------------------------------------------------------------- /packages/datastore-level/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "test", 8 | "src" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../datastore-core" 13 | }, 14 | { 15 | "path": "../interface-datastore" 16 | }, 17 | { 18 | "path": "../interface-datastore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/datastore-level/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/datastore-s3/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/datastore-s3/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/datastore-s3/README.md: -------------------------------------------------------------------------------- 1 | # datastore-s3 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > IPFS datastore implementation backed by s3 9 | 10 | # About 11 | 12 | 26 | 27 | A Datastore implementation that stores data on Amazon S3. 28 | 29 | ## Example - Quickstart 30 | 31 | If the flag `createIfMissing` is not set or is false, then the bucket must be created prior to using datastore-s3. Please see the AWS docs for information on how to configure the S3 instance. A bucket name is required to be set at the s3 instance level, see the below example. 32 | 33 | ```js 34 | import { S3 } from '@aws-sdk/client-s3' 35 | import { S3Datastore } from 'datastore-s3' 36 | 37 | const s3 = new S3({ 38 | region: 'region', 39 | credentials: { 40 | accessKeyId: 'myaccesskey', 41 | secretAccessKey: 'mysecretkey' 42 | } 43 | }) 44 | 45 | const store = new S3Datastore( 46 | s3, 47 | 'my-bucket', 48 | { path: '.ipfs/datastore', createIfMissing: false } 49 | ) 50 | ``` 51 | 52 | ## Example 53 | 54 | ```ts 55 | Using with Helia 56 | 57 | See [examples/helia](./examples/helia) for a full example of how to use Helia with an S3 backed datastore. 58 | ``` 59 | 60 | # Install 61 | 62 | ```console 63 | $ npm i datastore-s3 64 | ``` 65 | 66 | ## Browser ` 72 | ``` 73 | 74 | # API Docs 75 | 76 | - 77 | 78 | # License 79 | 80 | Licensed under either of 81 | 82 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/datastore-s3/LICENSE-APACHE) / ) 83 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/datastore-s3/LICENSE-MIT) / ) 84 | 85 | # Contribute 86 | 87 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 88 | 89 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 90 | 91 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 92 | 93 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 94 | 95 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 96 | -------------------------------------------------------------------------------- /packages/datastore-s3/examples/helia/README.md: -------------------------------------------------------------------------------- 1 | Use with Helia 2 | ====== 3 | 4 | This example uses a Datastore S3 instance to serve as the entire backend for Helia. 5 | 6 | ## Running 7 | The S3 parameters must be updated with an existing Bucket and credentials with access to it: 8 | ```js 9 | // Configure S3 as normal 10 | const s3 = new S3({ 11 | region: 'region', 12 | credentials: { 13 | accessKeyId: 'myaccesskey', 14 | secretAccessKey: 'mysecretkey' 15 | } 16 | }) 17 | 18 | const datastore = new DatastoreS3(s3, 'my-bucket') 19 | ``` 20 | 21 | Once the S3 instance has its needed data, you can run the example: 22 | ``` 23 | npm install 24 | node index.js 25 | ``` 26 | -------------------------------------------------------------------------------- /packages/datastore-s3/examples/helia/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | 3 | import { S3 } from '@aws-sdk/client-s3' 4 | import { unixfs } from '@helia/unixfs' 5 | import { DatastoreS3 } from 'datastore-s3' 6 | import { createHelia } from 'helia' 7 | import toBuffer from 'it-to-buffer' 8 | 9 | async function main () { 10 | // Configure S3 as normal 11 | const s3 = new S3({ 12 | region: 'region', 13 | credentials: { 14 | accessKeyId: 'myaccesskey', 15 | secretAccessKey: 'mysecretkey' 16 | } 17 | }) 18 | 19 | const datastore = new DatastoreS3(s3, 'my-bucket') 20 | 21 | // Create a new Helia node with our S3 backed Repo 22 | console.log('Start Helia') 23 | const helia = await createHelia({ 24 | datastore 25 | }) 26 | 27 | // Test out the repo by sending and fetching some data 28 | console.log('Helia is ready') 29 | 30 | try { 31 | const fs = unixfs(helia) 32 | 33 | // Let's add a file to Helia 34 | const cid = await fs.addBytes(Uint8Array.from([0, 1, 2, 3])) 35 | 36 | console.log('\nAdded file:', cid) 37 | 38 | // Log out the added files metadata and cat the file from IPFS 39 | const data = await toBuffer(fs.cat(cid)) 40 | 41 | // Print out the files contents to console 42 | console.log(`\nFetched file content containing ${data.byteLength} bytes`) 43 | } catch (err) { 44 | // Log out the error 45 | console.log('File Processing Error:', err) 46 | } 47 | // After everything is done, shut the node down 48 | // We don't need to worry about catching errors here 49 | console.log('\n\nStopping the node') 50 | await helia.stop() 51 | } 52 | 53 | main() 54 | .catch(err => { 55 | console.error(err) 56 | process.exit(1) 57 | }) 58 | -------------------------------------------------------------------------------- /packages/datastore-s3/examples/helia/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "helia", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "@aws-sdk/client-s3": "^3.297.0", 14 | "@helia/unixfs": "^1.2.0", 15 | "datastore-s3": "../../", 16 | "helia": "^1.0.0", 17 | "it-to-buffer": "^3.0.1" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /packages/datastore-s3/test/utils/s3-mock.ts: -------------------------------------------------------------------------------- 1 | import sinon from 'sinon' 2 | import type { S3 } from '@aws-sdk/client-s3' 3 | 4 | export class S3Error extends Error { 5 | public code: string 6 | public statusCode?: number 7 | public $metadata?: { httpStatusCode: number } 8 | 9 | constructor (message: string, code?: number) { 10 | super(message) 11 | this.code = message 12 | this.statusCode = code 13 | 14 | this.$metadata = { 15 | httpStatusCode: code ?? 200 16 | } 17 | } 18 | } 19 | 20 | export function s3Resolve (res?: any): any { 21 | return Promise.resolve(res) 22 | } 23 | 24 | export function s3Reject (err: Error): any { 25 | return Promise.reject(err) 26 | } 27 | 28 | /** 29 | * Mocks out the s3 calls made by datastore-s3 30 | */ 31 | export function s3Mock (s3: S3): S3 { 32 | const mocks: any = {} 33 | const storage = new Map() 34 | 35 | mocks.send = sinon.replace(s3, 'send', (command: any) => { 36 | const commandName = command.constructor.name 37 | const input: any = command.input 38 | 39 | if (commandName.includes('PutObjectCommand') === true) { 40 | storage.set(input.Key, input.Body) 41 | return s3Resolve({}) 42 | } 43 | 44 | if (commandName.includes('HeadObjectCommand') === true) { 45 | if (storage.has(input.Key)) { 46 | return s3Resolve({}) 47 | } 48 | 49 | return s3Reject(new S3Error('NotFound', 404)) 50 | } 51 | 52 | if (commandName.includes('GetObjectCommand') === true) { 53 | if (!storage.has(input.Key)) { 54 | return s3Reject(new S3Error('NotFound', 404)) 55 | } 56 | 57 | return s3Resolve({ 58 | Body: storage.get(input.Key) 59 | }) 60 | } 61 | 62 | if (commandName.includes('DeleteObjectCommand') === true) { 63 | storage.delete(input.Key) 64 | return s3Resolve({}) 65 | } 66 | 67 | if (commandName.includes('ListObjectsV2Command') === true) { 68 | const results: { Contents: Array<{ Key: string }> } = { 69 | Contents: [] 70 | } 71 | 72 | for (const k of storage.keys()) { 73 | if (k.startsWith(`${input.Prefix ?? ''}`)) { 74 | results.Contents.push({ 75 | Key: k 76 | }) 77 | } 78 | } 79 | 80 | return s3Resolve(results) 81 | } 82 | 83 | return s3Reject(new S3Error('UnknownCommand', 400)) 84 | }) 85 | 86 | return s3 87 | } 88 | -------------------------------------------------------------------------------- /packages/datastore-s3/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../datastore-core" 13 | }, 14 | { 15 | "path": "../interface-datastore" 16 | }, 17 | { 18 | "path": "../interface-datastore-tests" 19 | }, 20 | { 21 | "path": "../interface-store" 22 | } 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /packages/datastore-s3/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/interface-blockstore-tests/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/interface-blockstore-tests/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/interface-blockstore-tests/README.md: -------------------------------------------------------------------------------- 1 | # interface-blockstore-tests 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Compliance tests for the blockstore interface 9 | 10 | # About 11 | 12 | 26 | 27 | A test suite that ensures a given implementation implements the Blockstore interface properly. 28 | 29 | ## Example 30 | 31 | ```js 32 | const MyBlockstore from './path/to/my-blockstore') 33 | const suite from 'interface-blockstore-tests') 34 | 35 | describe('MyBlockstore', () => { 36 | describe('interface-blockstore compliance tests', () => { 37 | suite({ 38 | setup () { 39 | return new MyBlockstore() 40 | }, 41 | teardown () {} 42 | }) 43 | }) 44 | }) 45 | ``` 46 | 47 | # Install 48 | 49 | ```console 50 | $ npm i interface-blockstore-tests 51 | ``` 52 | 53 | # API Docs 54 | 55 | - 56 | 57 | # License 58 | 59 | Licensed under either of 60 | 61 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/interface-blockstore-tests/LICENSE-APACHE) / ) 62 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/interface-blockstore-tests/LICENSE-MIT) / ) 63 | 64 | # Contribute 65 | 66 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 67 | 68 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 69 | 70 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 71 | 72 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 73 | 74 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 75 | -------------------------------------------------------------------------------- /packages/interface-blockstore-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "interface-blockstore-tests", 3 | "version": "7.0.3", 4 | "description": "Compliance tests for the blockstore interface", 5 | "license": "Apache-2.0 OR MIT", 6 | "homepage": "https://github.com/ipfs/js-stores/tree/main/packages/interface-blockstore-tests#readme", 7 | "repository": { 8 | "type": "git", 9 | "url": "git+https://github.com/ipfs/js-stores.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/ipfs/js-stores/issues" 13 | }, 14 | "publishConfig": { 15 | "access": "public", 16 | "provenance": true 17 | }, 18 | "keywords": [ 19 | "blockstore", 20 | "interface", 21 | "ipfs", 22 | "key-value" 23 | ], 24 | "type": "module", 25 | "types": "./dist/src/index.d.ts", 26 | "files": [ 27 | "src", 28 | "dist", 29 | "!dist/test", 30 | "!**/*.tsbuildinfo" 31 | ], 32 | "exports": { 33 | ".": { 34 | "types": "./dist/src/index.d.ts", 35 | "import": "./dist/src/index.js" 36 | } 37 | }, 38 | "release": { 39 | "branches": [ 40 | "main" 41 | ], 42 | "plugins": [ 43 | [ 44 | "@semantic-release/commit-analyzer", 45 | { 46 | "preset": "conventionalcommits", 47 | "releaseRules": [ 48 | { 49 | "breaking": true, 50 | "release": "major" 51 | }, 52 | { 53 | "revert": true, 54 | "release": "patch" 55 | }, 56 | { 57 | "type": "feat", 58 | "release": "minor" 59 | }, 60 | { 61 | "type": "fix", 62 | "release": "patch" 63 | }, 64 | { 65 | "type": "docs", 66 | "release": "patch" 67 | }, 68 | { 69 | "type": "test", 70 | "release": "patch" 71 | }, 72 | { 73 | "type": "deps", 74 | "release": "patch" 75 | }, 76 | { 77 | "scope": "no-release", 78 | "release": false 79 | } 80 | ] 81 | } 82 | ], 83 | [ 84 | "@semantic-release/release-notes-generator", 85 | { 86 | "preset": "conventionalcommits", 87 | "presetConfig": { 88 | "types": [ 89 | { 90 | "type": "feat", 91 | "section": "Features" 92 | }, 93 | { 94 | "type": "fix", 95 | "section": "Bug Fixes" 96 | }, 97 | { 98 | "type": "chore", 99 | "section": "Trivial Changes" 100 | }, 101 | { 102 | "type": "docs", 103 | "section": "Documentation" 104 | }, 105 | { 106 | "type": "deps", 107 | "section": "Dependencies" 108 | }, 109 | { 110 | "type": "test", 111 | "section": "Tests" 112 | } 113 | ] 114 | } 115 | } 116 | ], 117 | "@semantic-release/changelog", 118 | "@semantic-release/npm", 119 | "@semantic-release/github", 120 | [ 121 | "@semantic-release/git", 122 | { 123 | "assets": [ 124 | "CHANGELOG.md", 125 | "package.json" 126 | ] 127 | } 128 | ] 129 | ] 130 | }, 131 | "scripts": { 132 | "build": "aegir build", 133 | "lint": "aegir lint", 134 | "dep-check": "aegir dep-check", 135 | "clean": "aegir clean", 136 | "release": "aegir release" 137 | }, 138 | "dependencies": { 139 | "interface-blockstore": "^5.0.0", 140 | "it-all": "^3.0.8", 141 | "it-drain": "^3.0.9", 142 | "multiformats": "^13.3.6", 143 | "uint8arrays": "^5.1.0" 144 | }, 145 | "devDependencies": { 146 | "aegir": "^47.0.16" 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /packages/interface-blockstore-tests/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src" 8 | ], 9 | "references": [ 10 | { 11 | "path": "../interface-blockstore" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /packages/interface-blockstore-tests/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/interface-blockstore/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/interface-blockstore/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/interface-blockstore/README.md: -------------------------------------------------------------------------------- 1 | # interface-blockstore 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > An interface for storing and retrieving blocks 9 | 10 | # About 11 | 12 | 26 | 27 | A Blockstore is a key/value database that lets use CIDs to store/retrieve binary blobs. 28 | 29 | It is used by IPFS to store/retrieve the block that a given CID resolves to. 30 | 31 | ## Implementations 32 | 33 | - File System: [`blockstore-fs`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-fs) 34 | - IndexedDB: [`blockstore-idb`](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-idb) 35 | - level: [`blockstore-level`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-level) (supports any levelup compatible backend) 36 | - Memory: [`blockstore-core/memory`](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-core/src/memory.ts) 37 | - S3: [`blockstore-s3`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-s3) 38 | 39 | # Install 40 | 41 | ```console 42 | $ npm i interface-blockstore 43 | ``` 44 | 45 | # API Docs 46 | 47 | - 48 | 49 | # License 50 | 51 | Licensed under either of 52 | 53 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/interface-blockstore/LICENSE-APACHE) / ) 54 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/interface-blockstore/LICENSE-MIT) / ) 55 | 56 | # Contribute 57 | 58 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 59 | 60 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 61 | 62 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 63 | 64 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 65 | 66 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 67 | -------------------------------------------------------------------------------- /packages/interface-blockstore/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "interface-blockstore", 3 | "version": "5.3.2", 4 | "description": "An interface for storing and retrieving blocks", 5 | "license": "Apache-2.0 OR MIT", 6 | "homepage": "https://github.com/ipfs/js-stores/tree/main/packages/interface-blockstore#readme", 7 | "repository": { 8 | "type": "git", 9 | "url": "git+https://github.com/ipfs/js-stores.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/ipfs/js-stores/issues" 13 | }, 14 | "publishConfig": { 15 | "access": "public", 16 | "provenance": true 17 | }, 18 | "type": "module", 19 | "types": "./dist/src/index.d.ts", 20 | "files": [ 21 | "src", 22 | "dist", 23 | "!dist/test", 24 | "!**/*.tsbuildinfo" 25 | ], 26 | "exports": { 27 | ".": { 28 | "types": "./dist/src/index.d.ts", 29 | "import": "./dist/src/index.js" 30 | } 31 | }, 32 | "release": { 33 | "branches": [ 34 | "main" 35 | ], 36 | "plugins": [ 37 | [ 38 | "@semantic-release/commit-analyzer", 39 | { 40 | "preset": "conventionalcommits", 41 | "releaseRules": [ 42 | { 43 | "breaking": true, 44 | "release": "major" 45 | }, 46 | { 47 | "revert": true, 48 | "release": "patch" 49 | }, 50 | { 51 | "type": "feat", 52 | "release": "minor" 53 | }, 54 | { 55 | "type": "fix", 56 | "release": "patch" 57 | }, 58 | { 59 | "type": "docs", 60 | "release": "patch" 61 | }, 62 | { 63 | "type": "test", 64 | "release": "patch" 65 | }, 66 | { 67 | "type": "deps", 68 | "release": "patch" 69 | }, 70 | { 71 | "scope": "no-release", 72 | "release": false 73 | } 74 | ] 75 | } 76 | ], 77 | [ 78 | "@semantic-release/release-notes-generator", 79 | { 80 | "preset": "conventionalcommits", 81 | "presetConfig": { 82 | "types": [ 83 | { 84 | "type": "feat", 85 | "section": "Features" 86 | }, 87 | { 88 | "type": "fix", 89 | "section": "Bug Fixes" 90 | }, 91 | { 92 | "type": "chore", 93 | "section": "Trivial Changes" 94 | }, 95 | { 96 | "type": "docs", 97 | "section": "Documentation" 98 | }, 99 | { 100 | "type": "deps", 101 | "section": "Dependencies" 102 | }, 103 | { 104 | "type": "test", 105 | "section": "Tests" 106 | } 107 | ] 108 | } 109 | } 110 | ], 111 | "@semantic-release/changelog", 112 | "@semantic-release/npm", 113 | "@semantic-release/github", 114 | [ 115 | "@semantic-release/git", 116 | { 117 | "assets": [ 118 | "CHANGELOG.md", 119 | "package.json" 120 | ] 121 | } 122 | ] 123 | ] 124 | }, 125 | "scripts": { 126 | "build": "aegir build", 127 | "lint": "aegir lint", 128 | "clean": "aegir clean", 129 | "release": "aegir release" 130 | }, 131 | "dependencies": { 132 | "interface-store": "^6.0.0", 133 | "multiformats": "^13.3.6" 134 | }, 135 | "devDependencies": { 136 | "aegir": "^47.0.16" 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /packages/interface-blockstore/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @packageDocumentation 3 | * 4 | * A Blockstore is a key/value database that lets use CIDs to store/retrieve binary blobs. 5 | * 6 | * It is used by IPFS to store/retrieve the block that a given CID resolves to. 7 | * 8 | * ## Implementations 9 | * 10 | * - File System: [`blockstore-fs`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-fs) 11 | * - IndexedDB: [`blockstore-idb`](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-idb) 12 | * - level: [`blockstore-level`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-level) (supports any levelup compatible backend) 13 | * - Memory: [`blockstore-core/memory`](https://github.com/ipfs/js-stores/blob/main/packages/blockstore-core/src/memory.ts) 14 | * - S3: [`blockstore-s3`](https://github.com/ipfs/js-stores/tree/main/packages/blockstore-s3) 15 | */ 16 | 17 | import type { 18 | AbortOptions, 19 | AwaitIterable, 20 | Store 21 | } from 'interface-store' 22 | import type { CID } from 'multiformats/cid' 23 | 24 | export interface Pair { 25 | cid: CID 26 | block: Uint8Array 27 | } 28 | 29 | export interface Blockstore extends Store { 36 | /** 37 | * Retrieve all cid/block pairs from the blockstore as an unordered iterable 38 | * 39 | * @example 40 | * ```js 41 | * for await (const { multihash, block } of store.getAll()) { 42 | * console.log('got:', multihash, block) 43 | * // => got MultihashDigest('Qmfoo') Uint8Array[...] 44 | * } 45 | * ``` 46 | */ 47 | getAll(options?: AbortOptions & GetAllOptionsExtension): AwaitIterable 48 | } 49 | -------------------------------------------------------------------------------- /packages/interface-blockstore/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src" 8 | ], 9 | "references": [ 10 | { 11 | "path": "../interface-store" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /packages/interface-blockstore/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/interface-datastore-tests/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/interface-datastore-tests/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/interface-datastore-tests/README.md: -------------------------------------------------------------------------------- 1 | # interface-datastore-tests 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > Compliance tests for the datastore interface 9 | 10 | # About 11 | 12 | 26 | 27 | A test suite that ensures a given implementation implements the Datastore interface properly. 28 | 29 | ## Example 30 | 31 | ```js 32 | const MyDatastore from './path/to/my-datastore') 33 | const suite from 'interface-datastore-tests') 34 | 35 | describe('MyDatastore', () => { 36 | describe('interface-datastore compliance tests', () => { 37 | suite({ 38 | setup () { 39 | return new MyDatastore() 40 | }, 41 | teardown () {} 42 | }) 43 | }) 44 | }) 45 | ``` 46 | 47 | # Install 48 | 49 | ```console 50 | $ npm i interface-datastore-tests 51 | ``` 52 | 53 | # API Docs 54 | 55 | - 56 | 57 | # License 58 | 59 | Licensed under either of 60 | 61 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/interface-datastore-tests/LICENSE-APACHE) / ) 62 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/interface-datastore-tests/LICENSE-MIT) / ) 63 | 64 | # Contribute 65 | 66 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 67 | 68 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 69 | 70 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 71 | 72 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 73 | 74 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 75 | -------------------------------------------------------------------------------- /packages/interface-datastore-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "interface-datastore-tests", 3 | "version": "6.0.3", 4 | "description": "Compliance tests for the datastore interface", 5 | "license": "Apache-2.0 OR MIT", 6 | "homepage": "https://github.com/ipfs/js-stores/tree/main/packages/interface-datastore-tests#readme", 7 | "repository": { 8 | "type": "git", 9 | "url": "git+https://github.com/ipfs/js-stores.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/ipfs/js-stores/issues" 13 | }, 14 | "publishConfig": { 15 | "access": "public", 16 | "provenance": true 17 | }, 18 | "keywords": [ 19 | "datastore", 20 | "interface", 21 | "ipfs", 22 | "key-value" 23 | ], 24 | "type": "module", 25 | "types": "./dist/src/index.d.ts", 26 | "files": [ 27 | "src", 28 | "dist", 29 | "!dist/test", 30 | "!**/*.tsbuildinfo" 31 | ], 32 | "exports": { 33 | ".": { 34 | "types": "./dist/src/index.d.ts", 35 | "import": "./dist/src/index.js" 36 | } 37 | }, 38 | "release": { 39 | "branches": [ 40 | "main" 41 | ], 42 | "plugins": [ 43 | [ 44 | "@semantic-release/commit-analyzer", 45 | { 46 | "preset": "conventionalcommits", 47 | "releaseRules": [ 48 | { 49 | "breaking": true, 50 | "release": "major" 51 | }, 52 | { 53 | "revert": true, 54 | "release": "patch" 55 | }, 56 | { 57 | "type": "feat", 58 | "release": "minor" 59 | }, 60 | { 61 | "type": "fix", 62 | "release": "patch" 63 | }, 64 | { 65 | "type": "docs", 66 | "release": "patch" 67 | }, 68 | { 69 | "type": "test", 70 | "release": "patch" 71 | }, 72 | { 73 | "type": "deps", 74 | "release": "patch" 75 | }, 76 | { 77 | "scope": "no-release", 78 | "release": false 79 | } 80 | ] 81 | } 82 | ], 83 | [ 84 | "@semantic-release/release-notes-generator", 85 | { 86 | "preset": "conventionalcommits", 87 | "presetConfig": { 88 | "types": [ 89 | { 90 | "type": "feat", 91 | "section": "Features" 92 | }, 93 | { 94 | "type": "fix", 95 | "section": "Bug Fixes" 96 | }, 97 | { 98 | "type": "chore", 99 | "section": "Trivial Changes" 100 | }, 101 | { 102 | "type": "docs", 103 | "section": "Documentation" 104 | }, 105 | { 106 | "type": "deps", 107 | "section": "Dependencies" 108 | }, 109 | { 110 | "type": "test", 111 | "section": "Tests" 112 | } 113 | ] 114 | } 115 | } 116 | ], 117 | "@semantic-release/changelog", 118 | "@semantic-release/npm", 119 | "@semantic-release/github", 120 | [ 121 | "@semantic-release/git", 122 | { 123 | "assets": [ 124 | "CHANGELOG.md", 125 | "package.json" 126 | ] 127 | } 128 | ] 129 | ] 130 | }, 131 | "scripts": { 132 | "build": "aegir build", 133 | "lint": "aegir lint", 134 | "dep-check": "aegir dep-check", 135 | "clean": "aegir clean", 136 | "release": "aegir release" 137 | }, 138 | "dependencies": { 139 | "interface-datastore": "^8.0.0", 140 | "iso-random-stream": "^2.0.2", 141 | "it-all": "^3.0.8", 142 | "it-drain": "^3.0.9", 143 | "it-length": "^3.0.8", 144 | "uint8arrays": "^5.1.0" 145 | }, 146 | "devDependencies": { 147 | "aegir": "^47.0.16" 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /packages/interface-datastore-tests/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src" 8 | ], 9 | "references": [ 10 | { 11 | "path": "../interface-datastore" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /packages/interface-datastore-tests/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /packages/interface-datastore/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/interface-datastore/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/interface-datastore/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src", 8 | "test" 9 | ], 10 | "references": [ 11 | { 12 | "path": "../interface-store" 13 | } 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /packages/interface-datastore/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts", 5 | "./src/key.ts" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /packages/interface-store/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | This project follows the [`IPFS Community Code of Conduct`](https://github.com/ipfs/community/blob/master/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /packages/interface-store/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /packages/interface-store/README.md: -------------------------------------------------------------------------------- 1 | # interface-store 2 | 3 | [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) 4 | [![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) 5 | [![codecov](https://img.shields.io/codecov/c/github/ipfs/js-stores.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-stores) 6 | [![CI](https://img.shields.io/github/actions/workflow/status/ipfs/js-stores/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/js-stores/actions/workflows/js-test-and-release.yml?query=branch%3Amain) 7 | 8 | > A generic interface for storing and retrieving data 9 | 10 | # About 11 | 12 | 26 | 27 | An abstraction of the Datastore/Blockstore codebases. 28 | 29 | # Install 30 | 31 | ```console 32 | $ npm i interface-store 33 | ``` 34 | 35 | # API Docs 36 | 37 | - 38 | 39 | # License 40 | 41 | Licensed under either of 42 | 43 | - Apache 2.0, ([LICENSE-APACHE](https://github.com/ipfs/js-stores/blob/main/packages/interface-store/LICENSE-APACHE) / ) 44 | - MIT ([LICENSE-MIT](https://github.com/ipfs/js-stores/blob/main/packages/interface-store/LICENSE-MIT) / ) 45 | 46 | # Contribute 47 | 48 | Contributions welcome! Please check out [the issues](https://github.com/ipfs/js-stores/issues). 49 | 50 | Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. 51 | 52 | Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). 53 | 54 | Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. 55 | 56 | [![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) 57 | -------------------------------------------------------------------------------- /packages/interface-store/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "interface-store", 3 | "version": "6.0.3", 4 | "description": "A generic interface for storing and retrieving data", 5 | "license": "Apache-2.0 OR MIT", 6 | "homepage": "https://github.com/ipfs/js-stores/tree/main/packages/interface-store#readme", 7 | "repository": { 8 | "type": "git", 9 | "url": "git+https://github.com/ipfs/js-stores.git" 10 | }, 11 | "bugs": { 12 | "url": "https://github.com/ipfs/js-stores/issues" 13 | }, 14 | "publishConfig": { 15 | "access": "public", 16 | "provenance": true 17 | }, 18 | "type": "module", 19 | "types": "./dist/src/index.d.ts", 20 | "files": [ 21 | "src", 22 | "dist", 23 | "!dist/test", 24 | "!**/*.tsbuildinfo" 25 | ], 26 | "exports": { 27 | ".": { 28 | "types": "./dist/src/index.d.ts", 29 | "import": "./dist/src/index.js" 30 | } 31 | }, 32 | "release": { 33 | "branches": [ 34 | "main" 35 | ], 36 | "plugins": [ 37 | [ 38 | "@semantic-release/commit-analyzer", 39 | { 40 | "preset": "conventionalcommits", 41 | "releaseRules": [ 42 | { 43 | "breaking": true, 44 | "release": "major" 45 | }, 46 | { 47 | "revert": true, 48 | "release": "patch" 49 | }, 50 | { 51 | "type": "feat", 52 | "release": "minor" 53 | }, 54 | { 55 | "type": "fix", 56 | "release": "patch" 57 | }, 58 | { 59 | "type": "docs", 60 | "release": "patch" 61 | }, 62 | { 63 | "type": "test", 64 | "release": "patch" 65 | }, 66 | { 67 | "type": "deps", 68 | "release": "patch" 69 | }, 70 | { 71 | "scope": "no-release", 72 | "release": false 73 | } 74 | ] 75 | } 76 | ], 77 | [ 78 | "@semantic-release/release-notes-generator", 79 | { 80 | "preset": "conventionalcommits", 81 | "presetConfig": { 82 | "types": [ 83 | { 84 | "type": "feat", 85 | "section": "Features" 86 | }, 87 | { 88 | "type": "fix", 89 | "section": "Bug Fixes" 90 | }, 91 | { 92 | "type": "chore", 93 | "section": "Trivial Changes" 94 | }, 95 | { 96 | "type": "docs", 97 | "section": "Documentation" 98 | }, 99 | { 100 | "type": "deps", 101 | "section": "Dependencies" 102 | }, 103 | { 104 | "type": "test", 105 | "section": "Tests" 106 | } 107 | ] 108 | } 109 | } 110 | ], 111 | "@semantic-release/changelog", 112 | "@semantic-release/npm", 113 | "@semantic-release/github", 114 | [ 115 | "@semantic-release/git", 116 | { 117 | "assets": [ 118 | "CHANGELOG.md", 119 | "package.json" 120 | ] 121 | } 122 | ] 123 | ] 124 | }, 125 | "scripts": { 126 | "build": "aegir build", 127 | "lint": "aegir lint", 128 | "clean": "aegir clean", 129 | "dep-check": "aegir dep-check", 130 | "release": "aegir release" 131 | }, 132 | "devDependencies": { 133 | "aegir": "^47.0.16" 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /packages/interface-store/src/errors.ts: -------------------------------------------------------------------------------- 1 | export class OpenFailedError extends Error { 2 | static name = 'OpenFailedError' 3 | static code = 'ERR_OPEN_FAILED' 4 | name = OpenFailedError.name 5 | code = OpenFailedError.code 6 | 7 | constructor (message = 'Open failed') { 8 | super(message) 9 | } 10 | } 11 | 12 | export class CloseFailedError extends Error { 13 | static name = 'CloseFailedError' 14 | static code = 'ERR_CLOSE_FAILED' 15 | name = CloseFailedError.name 16 | code = CloseFailedError.code 17 | 18 | constructor (message = 'Close failed') { 19 | super(message) 20 | } 21 | } 22 | 23 | export class PutFailedError extends Error { 24 | static name = 'PutFailedError' 25 | static code = 'ERR_PUT_FAILED' 26 | name = PutFailedError.name 27 | code = PutFailedError.code 28 | 29 | constructor (message = 'Put failed') { 30 | super(message) 31 | } 32 | } 33 | 34 | export class GetFailedError extends Error { 35 | static name = 'GetFailedError' 36 | static code = 'ERR_GET_FAILED' 37 | name = GetFailedError.name 38 | code = GetFailedError.code 39 | 40 | constructor (message = 'Get failed') { 41 | super(message) 42 | } 43 | } 44 | 45 | export class DeleteFailedError extends Error { 46 | static name = 'DeleteFailedError' 47 | static code = 'ERR_DELETE_FAILED' 48 | name = DeleteFailedError.name 49 | code = DeleteFailedError.code 50 | 51 | constructor (message = 'Delete failed') { 52 | super(message) 53 | } 54 | } 55 | 56 | export class HasFailedError extends Error { 57 | static name = 'HasFailedError' 58 | static code = 'ERR_HAS_FAILED' 59 | name = HasFailedError.name 60 | code = HasFailedError.code 61 | 62 | constructor (message = 'Has failed') { 63 | super(message) 64 | } 65 | } 66 | 67 | export class NotFoundError extends Error { 68 | static name = 'NotFoundError' 69 | static code = 'ERR_NOT_FOUND' 70 | name = NotFoundError.name 71 | code = NotFoundError.code 72 | 73 | constructor (message = 'Not Found') { 74 | super(message) 75 | } 76 | } 77 | 78 | export class AbortError extends Error { 79 | static name = 'AbortError' 80 | static code = 'ERR_ABORTED' 81 | name = AbortError.name 82 | code = AbortError.code 83 | 84 | constructor (message = 'Aborted') { 85 | super(message) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /packages/interface-store/src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @packageDocumentation 3 | * 4 | * An abstraction of the Datastore/Blockstore codebases. 5 | */ 6 | 7 | /** 8 | * An iterable or async iterable of values 9 | */ 10 | export type AwaitIterable = Iterable | AsyncIterable 11 | 12 | /** 13 | * A value or a promise of a value 14 | */ 15 | export type Await = Promise | T 16 | 17 | /** 18 | * Options for async operations. 19 | */ 20 | export interface AbortOptions { 21 | signal?: AbortSignal 22 | } 23 | 24 | export interface Store { 28 | /** 29 | * Check for the existence of a value for the passed key 30 | * 31 | * @example 32 | * ```js 33 | *const exists = await store.has(new Key('awesome')) 34 | * 35 | *if (exists) { 36 | * console.log('it is there') 37 | *} else { 38 | * console.log('it is not there') 39 | *} 40 | *``` 41 | */ 42 | has(key: Key, options?: AbortOptions & HasOptionsExtension): Await 43 | 44 | /** 45 | * Store the passed value under the passed key 46 | * 47 | * @example 48 | * 49 | * ```js 50 | * await store.put([{ key: new Key('awesome'), value: new Uint8Array([0, 1, 2, 3]) }]) 51 | * ``` 52 | */ 53 | put(key: Key, val: Value, options?: AbortOptions & PutOptionsExtension): Await 54 | 55 | /** 56 | * Store the given key/value pairs 57 | * 58 | * @example 59 | * ```js 60 | * const source = [{ key: new Key('awesome'), value: new Uint8Array([0, 1, 2, 3]) }] 61 | * 62 | * for await (const { key, value } of store.putMany(source)) { 63 | * console.info(`put content for key ${key}`) 64 | * } 65 | * ``` 66 | */ 67 | putMany( 68 | source: AwaitIterable, 69 | options?: AbortOptions & PutManyOptionsExtension 70 | ): AwaitIterable 71 | 72 | /** 73 | * Retrieve the value stored under the given key 74 | * 75 | * @example 76 | * ```js 77 | * const value = await store.get(new Key('awesome')) 78 | * console.log('got content: %s', value.toString('utf8')) 79 | * // => got content: datastore 80 | * ``` 81 | */ 82 | get(key: Key, options?: AbortOptions & GetOptionsExtension): Await 83 | 84 | /** 85 | * Retrieve values for the passed keys 86 | * 87 | * @example 88 | * ```js 89 | * for await (const { key, value } of store.getMany([new Key('awesome')])) { 90 | * console.log(`got "${key}" = "${new TextDecoder('utf8').decode(value)}"`') 91 | * // => got "/awesome" = "datastore" 92 | * } 93 | * ``` 94 | */ 95 | getMany( 96 | source: AwaitIterable, 97 | options?: AbortOptions & GetManyOptionsExtension 98 | ): AwaitIterable 99 | 100 | /** 101 | * Remove the record for the passed key 102 | * 103 | * @example 104 | * 105 | * ```js 106 | * await store.delete(new Key('awesome')) 107 | * console.log('deleted awesome content :(') 108 | * ``` 109 | */ 110 | delete(key: Key, options?: AbortOptions & DeleteOptionsExtension): Await 111 | 112 | /** 113 | * Remove values for the passed keys 114 | * 115 | * @example 116 | * 117 | * ```js 118 | * const source = [new Key('awesome')] 119 | * 120 | * for await (const key of store.deleteMany(source)) { 121 | * console.log(`deleted content with key ${key}`) 122 | * } 123 | * ``` 124 | */ 125 | deleteMany( 126 | source: AwaitIterable, 127 | options?: AbortOptions & DeleteManyOptionsExtension 128 | ): AwaitIterable 129 | } 130 | 131 | export * from './errors.js' 132 | -------------------------------------------------------------------------------- /packages/interface-store/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "aegir/src/config/tsconfig.aegir.json", 3 | "compilerOptions": { 4 | "outDir": "dist" 5 | }, 6 | "include": [ 7 | "src" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /packages/interface-store/typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "readme": "none", 3 | "entryPoints": [ 4 | "./src/index.ts" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /typedoc.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://typedoc.org/schema.json", 3 | "name": "stores", 4 | "readme": "./README.md", 5 | "headings": { 6 | "readme": false, 7 | "document": false 8 | } 9 | } 10 | --------------------------------------------------------------------------------