├── .github └── workflows │ └── test-node.yml ├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── index.js ├── lib ├── monitor.js ├── prefetcher.js └── streams.js ├── package.json └── test └── all.js /.github/workflows/test-node.yml: -------------------------------------------------------------------------------- 1 | name: Build Status 2 | on: 3 | push: 4 | branches: 5 | - main 6 | tags: # To trigger the canary 7 | - '*' 8 | pull_request: 9 | branches: 10 | - main 11 | 12 | jobs: 13 | build: 14 | if: ${{ !startsWith(github.ref, 'refs/tags/')}} # Already runs for the push of the commit, no need to run again for the tag 15 | strategy: 16 | matrix: 17 | node-version: [lts/*] 18 | os: [ubuntu-latest, macos-latest, windows-latest] 19 | runs-on: ${{ matrix.os }} 20 | steps: 21 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 https://github.com/actions/checkout/releases/tag/v4.1.1 22 | - name: Use Node.js ${{ matrix.node-version }} 23 | uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2 https://github.com/actions/setup-node/releases/tag/v3.8.2 24 | with: 25 | node-version: ${{ matrix.node-version }} 26 | - run: npm install 27 | - run: npm test 28 | 29 | trigger_canary: 30 | if: startsWith(github.ref, 'refs/tags/') # Only run when a new package is published (detects when a new tag is pushed) 31 | runs-on: ubuntu-latest 32 | steps: 33 | - name: trigger canary 34 | run: | 35 | curl -L -X POST \ 36 | -H "Accept: application/vnd.github+json" \ 37 | -H "Authorization: Bearer ${{ secrets.CANARY_DISPATCH_PAT }}" \ 38 | -H "X-GitHub-Api-Version: 2022-11-28" \ 39 | https://api.github.com/repos/holepunchto/canary-tests/dispatches \ 40 | -d '{"event_type":"triggered-by-${{ github.event.repository.name }}-${{ github.ref_name }}"}' 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | package-lock.json 2 | 3 | # Logs 4 | logs 5 | *.log 6 | npm-debug.log* 7 | yarn-debug.log* 8 | yarn-error.log* 9 | lerna-debug.log* 10 | 11 | # Diagnostic reports (https://nodejs.org/api/report.html) 12 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 13 | 14 | # Runtime data 15 | pids 16 | *.pid 17 | *.seed 18 | *.pid.lock 19 | 20 | # Directory for instrumented libs generated by jscoverage/JSCover 21 | lib-cov 22 | 23 | # Coverage directory used by tools like istanbul 24 | coverage 25 | *.lcov 26 | 27 | # nyc test coverage 28 | .nyc_output 29 | 30 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 31 | .grunt 32 | 33 | # Bower dependency directory (https://bower.io/) 34 | bower_components 35 | 36 | # node-waf configuration 37 | .lock-wscript 38 | 39 | # Compiled binary addons (https://nodejs.org/api/addons.html) 40 | build/Release 41 | 42 | # Dependency directories 43 | node_modules/ 44 | jspm_packages/ 45 | 46 | # TypeScript v1 declaration files 47 | typings/ 48 | 49 | # TypeScript cache 50 | *.tsbuildinfo 51 | 52 | # Optional npm cache directory 53 | .npm 54 | 55 | # Optional eslint cache 56 | .eslintcache 57 | 58 | # Microbundle cache 59 | .rpt2_cache/ 60 | .rts2_cache_cjs/ 61 | .rts2_cache_es/ 62 | .rts2_cache_umd/ 63 | 64 | # Optional REPL history 65 | .node_repl_history 66 | 67 | # Output of 'npm pack' 68 | *.tgz 69 | 70 | # Yarn Integrity file 71 | .yarn-integrity 72 | 73 | # dotenv environment variables file 74 | .env 75 | .env.test 76 | 77 | # parcel-bundler cache (https://parceljs.org/) 78 | .cache 79 | 80 | # Next.js build output 81 | .next 82 | 83 | # Nuxt.js build / generate output 84 | .nuxt 85 | dist 86 | 87 | # Gatsby files 88 | .cache/ 89 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 90 | # https://nextjs.org/blog/next-9-1#public-directory-support 91 | # public 92 | 93 | # vuepress build output 94 | .vuepress/dist 95 | 96 | # Serverless directories 97 | .serverless/ 98 | 99 | # FuseBox cache 100 | .fusebox/ 101 | 102 | # DynamoDB Local files 103 | .dynamodb/ 104 | 105 | # TernJS port file 106 | .tern-port 107 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2023 Contributors 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hyperblobs 2 | 3 | A simple blob store for Hypercore. 4 | 5 | Each blob is identified by its unique bounds within the Hypercore, e.g. `{ byteOffset: 0, blockOffset: 0, blockLength: 5, byteLength: 327680 }`, which makes them easy to save and retrieve: 6 | ```js 7 | const blobs = new Hyperblobs(core) 8 | // ID is { byteOffset: 0, blockOffset: 0, blockLength: 1, byteLength: 11 } 9 | const id = await blobs.put(Buffer.from('hello world', 'utf-8')) 10 | await blobs.get(id) // Buffer.from('hello world', 'utf-8') 11 | ``` 12 | 13 | You can also get from start/end bounds within a single blob: 14 | ```js 15 | const blobs = new Hyperblobs(core) 16 | // ID is { byteOffset: 0, blockOffset: 0, blockLength: 1, byteLength: 11 } 17 | const id = await blobs.put(Buffer.from('hello world', 'utf-8')) 18 | await blobs.get(id, { start: 1, length: 2 }) // Buffer.from('el', 'utf-8') 19 | ``` 20 | 21 | If the blob is large, there's a Streams interface (`createReadStream` and `createWriteStream`) too. 22 | 23 | ## Installation 24 | ``` 25 | npm i hyperblobs 26 | ``` 27 | 28 | ## API 29 | ```js 30 | const Hyperblobs = require('hyperblobs') 31 | ``` 32 | 33 | #### `const blobs = new Hyperblobs(core, opts)` 34 | Create a new blob store wrapping a single Hypercore. 35 | 36 | Options can include: 37 | ```js 38 | { 39 | blockSize: 64KB // The block size that will be used when storing large blobs. 40 | } 41 | ``` 42 | 43 | #### `const id = await blobs.put(blob, opts)` 44 | Store a new blob. If the blob is large, it will be chunked according to `opts.blockSize` (default 64KB). 45 | 46 | Options can include: 47 | ```js 48 | { 49 | blockSize: 64KB, // The block size that will be used when storing large blobs. 50 | start: 0, // Relative offset to start within the blob 51 | end: blob.length - 1, // End offset within the blob (inclusive) 52 | length: blob.length, // Number of bytes to read. 53 | core // A custom core to write (overrides the default core) 54 | } 55 | ``` 56 | 57 | #### `const content = await blobs.get(id, opts)` 58 | Return a complete blob as a `Buffer`. 59 | 60 | `id` is the value returned by `put` 61 | 62 | Options can include: 63 | ```js 64 | { 65 | core, // A custom core to read from (overrides the default core) 66 | wait: true, // Wait for block to be downloaded 67 | timeout: 0 // Wait at max some milliseconds (0 means no timeout) 68 | } 69 | ``` 70 | 71 | #### `await blobs.clear(id, opts)` 72 | Remove a blob from the core. 73 | 74 | `opts` are the same as `Hypercore.clear` method. 75 | 76 | #### `const stream = blobs.createReadStream(id, opts)` 77 | Create a Readable stream that will yield the `id` blob. 78 | 79 | Options match the `get` options. 80 | 81 | #### `const stream = blobs.createWriteStream(opts)` 82 | Create a Writable stream that will save a blob. 83 | 84 | The corresponding ID will be set on the stream at `stream.id`. 85 | 86 | ## License 87 | Apache-2.0 88 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const mutexify = require('mutexify') 2 | const b4a = require('b4a') 3 | 4 | const { BlobReadStream, BlobWriteStream } = require('./lib/streams') 5 | const Monitor = require('./lib/monitor') 6 | 7 | const DEFAULT_BLOCK_SIZE = 2 ** 16 8 | 9 | class HyperBlobsBatch { 10 | constructor (blobs) { 11 | this.blobs = blobs 12 | this.blocks = [] 13 | this.bytes = 0 14 | } 15 | 16 | ready () { 17 | return this.blobs.ready() 18 | } 19 | 20 | async put (buffer) { 21 | if (!this.blobs.core.opened) await this.blobs.core.ready() 22 | 23 | const blockSize = this.blobs.blockSize 24 | const result = { 25 | blockOffset: this.blobs.core.length + this.blocks.length, 26 | blockLength: 0, 27 | byteOffset: this.blobs.core.byteLength + this.bytes, 28 | byteLength: 0 29 | } 30 | 31 | let offset = 0 32 | while (offset < buffer.byteLength) { 33 | const blk = buffer.subarray(offset, offset + blockSize) 34 | offset += blockSize 35 | 36 | result.blockLength++ 37 | result.byteLength += blk.byteLength 38 | this.bytes += blk.byteLength 39 | this.blocks.push(blk) 40 | } 41 | 42 | return result 43 | } 44 | 45 | async get (id) { 46 | if (id.blockOffset < this.blobs.core.length) { 47 | return this.blobs.get(id) 48 | } 49 | 50 | const bufs = [] 51 | 52 | for (let i = id.blockOffset - this.blobs.core.length; i < id.blockOffset + id.blockLength; i++) { 53 | if (i >= this.blocks.length) return null 54 | bufs.push(this.blocks[i]) 55 | } 56 | 57 | return bufs.length === 1 ? bufs[0] : b4a.concat(bufs) 58 | } 59 | 60 | async flush () { 61 | await this.blobs.core.append(this.blocks) 62 | this.blocks = [] 63 | this.bytes = 0 64 | } 65 | 66 | close () { 67 | // noop, atm nothing to unlink 68 | } 69 | } 70 | 71 | class Hyperblobs { 72 | constructor (core, opts = {}) { 73 | this.core = core 74 | this.blockSize = opts.blockSize || DEFAULT_BLOCK_SIZE 75 | 76 | this._lock = mutexify() 77 | this._monitors = new Set() 78 | 79 | this._boundUpdatePeers = this._updatePeers.bind(this) 80 | this._boundOnUpload = this._onUpload.bind(this) 81 | this._boundOnDownload = this._onDownload.bind(this) 82 | } 83 | 84 | get key () { 85 | return this.core.key 86 | } 87 | 88 | get discoveryKey () { 89 | return this.core.discoveryKey 90 | } 91 | 92 | get feed () { 93 | return this.core 94 | } 95 | 96 | get locked () { 97 | return this._lock.locked 98 | } 99 | 100 | replicate (isInitiator, opts) { 101 | return this.core.replicate(isInitiator, opts) 102 | } 103 | 104 | ready () { 105 | return this.core.ready() 106 | } 107 | 108 | close () { 109 | return this.core.close() 110 | } 111 | 112 | batch () { 113 | return new HyperBlobsBatch(this) 114 | } 115 | 116 | snapshot () { 117 | return new Hyperblobs(this.core.snapshot()) 118 | } 119 | 120 | async put (blob, opts) { 121 | if (!b4a.isBuffer(blob)) blob = b4a.from(blob) 122 | const blockSize = (opts && opts.blockSize) || this.blockSize 123 | 124 | const stream = this.createWriteStream(opts) 125 | for (let i = 0; i < blob.length; i += blockSize) { 126 | stream.write(blob.subarray(i, i + blockSize)) 127 | } 128 | stream.end() 129 | 130 | return new Promise((resolve, reject) => { 131 | stream.once('error', reject) 132 | stream.once('close', () => resolve(stream.id)) 133 | }) 134 | } 135 | 136 | async _getAll (id, opts) { 137 | if (id.blockLength === 1) return this.core.get(id.blockOffset, opts) 138 | 139 | const promises = new Array(id.blockLength) 140 | for (let i = 0; i < id.blockLength; i++) { 141 | promises[i] = this.core.get(id.blockOffset + i, opts) 142 | } 143 | 144 | const blocks = await Promise.all(promises) 145 | for (let i = 0; i < id.blockLength; i++) { 146 | if (blocks[i] === null) return null 147 | } 148 | return b4a.concat(blocks) 149 | } 150 | 151 | async get (id, opts) { 152 | const all = !opts || (!opts.start && opts.length === undefined && opts.end === undefined && !opts.core) 153 | if (all) return this._getAll(id, opts) 154 | 155 | const res = [] 156 | try { 157 | for await (const block of this.createReadStream(id, opts)) { 158 | res.push(block) 159 | } 160 | } catch (error) { 161 | if (error.code === 'BLOCK_NOT_AVAILABLE') return null 162 | throw error 163 | } 164 | 165 | if (res.length === 1) return res[0] 166 | return b4a.concat(res) 167 | } 168 | 169 | async clear (id, opts) { 170 | return this.core.clear(id.blockOffset, id.blockOffset + id.blockLength, opts) 171 | } 172 | 173 | createReadStream (id, opts) { 174 | const core = (opts && opts.core) ? opts.core : this.core 175 | return new BlobReadStream(core, id, opts) 176 | } 177 | 178 | createWriteStream (opts) { 179 | const core = (opts && opts.core) ? opts.core : this.core 180 | return new BlobWriteStream(core, this._lock, opts) 181 | } 182 | 183 | monitor (id) { 184 | const monitor = new Monitor(this, id) 185 | if (this._monitors.size === 0) this._startListening() 186 | this._monitors.add(monitor) 187 | return monitor 188 | } 189 | 190 | _removeMonitor (mon) { 191 | this._monitors.delete(mon) 192 | if (this._monitors.size === 0) this._stopListening() 193 | } 194 | 195 | _updatePeers () { 196 | for (const m of this._monitors) m._updatePeers() 197 | } 198 | 199 | _onUpload (index, bytes, from) { 200 | for (const m of this._monitors) m._onUpload(index, bytes, from) 201 | } 202 | 203 | _onDownload (index, bytes, from) { 204 | for (const m of this._monitors) m._onDownload(index, bytes, from) 205 | } 206 | 207 | _startListening () { 208 | this.core.on('peer-add', this._boundUpdatePeers) 209 | this.core.on('peer-remove', this._boundUpdatePeers) 210 | this.core.on('upload', this._boundOnUpload) 211 | this.core.on('download', this._boundOnDownload) 212 | } 213 | 214 | _stopListening () { 215 | this.core.off('peer-add', this._boundUpdatePeers) 216 | this.core.off('peer-remove', this._boundUpdatePeers) 217 | this.core.off('upload', this._boundOnUpload) 218 | this.core.off('download', this._boundOnDownload) 219 | } 220 | } 221 | 222 | module.exports = Hyperblobs 223 | -------------------------------------------------------------------------------- /lib/monitor.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events') 2 | const speedometer = require('speedometer') 3 | 4 | module.exports = class Monitor extends EventEmitter { 5 | constructor (blobs, id) { 6 | super() 7 | 8 | if (!id) throw new Error('id is required') 9 | 10 | this.blobs = blobs 11 | this.id = id 12 | this.peers = 0 13 | this.uploadSpeedometer = null 14 | this.downloadSpeedometer = null 15 | 16 | const stats = { 17 | startTime: 0, 18 | percentage: 0, 19 | peers: 0, 20 | speed: 0, 21 | blocks: 0, 22 | totalBytes: 0, // local + bytes loaded during monitoring 23 | monitoringBytes: 0, // bytes loaded during monitoring 24 | targetBytes: 0, 25 | targetBlocks: 0 26 | } 27 | 28 | this.uploadStats = { ...stats } 29 | this.downloadStats = { ...stats } 30 | this.uploadStats.targetBytes = this.downloadStats.targetBytes = this.id.byteLength 31 | this.uploadStats.targetBlocks = this.downloadStats.targetBlocks = this.id.blockLength 32 | this.uploadStats.peers = this.downloadStats.peers = this.peers = this.blobs.core.peers.length 33 | 34 | this.uploadSpeedometer = speedometer() 35 | this.downloadSpeedometer = speedometer() 36 | 37 | // Handlers 38 | } 39 | 40 | // just an alias 41 | destroy () { 42 | return this.close() 43 | } 44 | 45 | close () { 46 | this.blobs._removeMonitor(this) 47 | } 48 | 49 | _onUpload (index, bytes, from) { 50 | this._updateStats(this.uploadSpeedometer, this.uploadStats, index, bytes, from) 51 | } 52 | 53 | _onDownload (index, bytes, from) { 54 | this._updateStats(this.downloadSpeedometer, this.downloadStats, index, bytes, from) 55 | } 56 | 57 | _updatePeers () { 58 | this.uploadStats.peers = this.downloadStats.peers = this.peers = this.blobs.core.peers.length 59 | this.emit('update') 60 | } 61 | 62 | _updateStats (speed, stats, index, bytes) { 63 | if (this.closing) return 64 | if (!isWithinRange(index, this.id)) return 65 | 66 | if (!stats.startTime) stats.startTime = Date.now() 67 | 68 | stats.speed = speed(bytes) 69 | stats.blocks++ 70 | stats.totalBytes += bytes 71 | stats.monitoringBytes += bytes 72 | stats.percentage = toFixed(stats.blocks / stats.targetBlocks * 100) 73 | 74 | this.emit('update') 75 | } 76 | 77 | downloadSpeed () { 78 | return this.downloadSpeedometer ? this.downloadSpeedometer() : 0 79 | } 80 | 81 | uploadSpeed () { 82 | return this.uploadSpeedometer ? this.uploadSpeedometer() : 0 83 | } 84 | } 85 | 86 | function isWithinRange (index, { blockOffset, blockLength }) { 87 | return index >= blockOffset && index < blockOffset + blockLength 88 | } 89 | 90 | function toFixed (n) { 91 | return Math.round(n * 100) / 100 92 | } 93 | -------------------------------------------------------------------------------- /lib/prefetcher.js: -------------------------------------------------------------------------------- 1 | // should move to hypercore itself 2 | 3 | const MAX_READAHEAD_TARGET = 0.05 // aim to buffer 5% always 4 | 5 | module.exports = class Prefetcher { 6 | constructor (core, { max = 64, start = 0, end = core.length, linear = true } = {}) { 7 | this.core = core 8 | this.max = max 9 | this.range = null 10 | this.startBound = start 11 | this.endBound = end 12 | this.maxReadAhead = Math.max(max * 2, Math.floor((end - start) * MAX_READAHEAD_TARGET)) 13 | 14 | this.start = start 15 | this.end = start 16 | this.linear = linear 17 | this.missing = 0 18 | 19 | this._ondownloadBound = this._ondownload.bind(this) 20 | this.core.on('download', this._ondownloadBound) 21 | } 22 | 23 | _ondownload (index) { 24 | if (this.range && index < this.end && this.start <= index) { 25 | this.missing-- 26 | this._update() 27 | } 28 | } 29 | 30 | destroy () { 31 | this.core.off('download', this._ondownloadBound) 32 | if (this.range) this.range.destroy() 33 | this.range = null 34 | this.max = 0 35 | } 36 | 37 | update (position) { 38 | this.start = position 39 | if (!this.range) this._update() 40 | } 41 | 42 | _update () { 43 | if (this.missing >= this.max) return 44 | if (this.range) this.range.destroy() 45 | 46 | let end = this.end 47 | 48 | while (end < this.endBound && this.missing < this.max) { 49 | end = this.core.core.bitfield.firstUnset(end) + 1 50 | if (end >= this.endBound) break 51 | this.missing++ 52 | } 53 | 54 | if (end > this.start + this.maxReadAhead) end = this.start + this.maxReadAhead 55 | if (end >= this.endBound) end = this.endBound 56 | 57 | this.end = end 58 | 59 | if (this.start >= this.end) return 60 | 61 | this.range = this.core.download({ 62 | start: this.start, 63 | end: this.end, 64 | linear: this.linear 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /lib/streams.js: -------------------------------------------------------------------------------- 1 | const { Readable, Writable } = require('streamx') 2 | const { BLOCK_NOT_AVAILABLE } = require('hypercore-errors') 3 | const Prefetcher = require('./prefetcher') 4 | 5 | class BlobWriteStream extends Writable { 6 | constructor (core, lock, opts) { 7 | super(opts) 8 | this.id = {} 9 | this.core = core 10 | this._lock = lock 11 | this._release = null 12 | this._batch = [] 13 | } 14 | 15 | _open (cb) { 16 | this.core.ready().then(() => { 17 | this._lock(release => { 18 | this._release = release 19 | this.id.byteOffset = this.core.byteLength 20 | this.id.blockOffset = this.core.length 21 | return cb(null) 22 | }) 23 | }, err => cb(err)) 24 | } 25 | 26 | _final (cb) { 27 | this._append(err => { 28 | if (err) return cb(err) 29 | this.id.blockLength = this.core.length - this.id.blockOffset 30 | this.id.byteLength = this.core.byteLength - this.id.byteOffset 31 | return cb(null) 32 | }) 33 | } 34 | 35 | _destroy (cb) { 36 | if (this._release) this._release() 37 | cb(null) 38 | } 39 | 40 | _append (cb) { 41 | if (!this._batch.length) return cb(null) 42 | return this.core.append(this._batch).then(() => { 43 | this._batch = [] 44 | return cb(null) 45 | }, err => { 46 | this._batch = [] 47 | return cb(err) 48 | }) 49 | } 50 | 51 | _write (data, cb) { 52 | this._batch.push(data) 53 | if (this._batch.length >= 16) return this._append(cb) 54 | return cb(null) 55 | } 56 | } 57 | 58 | class BlobReadStream extends Readable { 59 | constructor (core, id, opts = {}) { 60 | super(opts) 61 | this.id = id 62 | this.core = core.session({ wait: opts.wait, timeout: opts.timeout }) 63 | 64 | const start = id.blockOffset 65 | const end = id.blockOffset + id.blockLength 66 | const noPrefetch = opts.wait === false || opts.prefetch === false || !core.core 67 | 68 | this._prefetch = noPrefetch ? null : new Prefetcher(this.core, { max: opts.prefetch, start, end }) 69 | this._lastPrefetch = null 70 | 71 | this._pos = opts.start !== undefined ? id.byteOffset + opts.start : id.byteOffset 72 | 73 | if (opts.length !== undefined) this._end = this._pos + opts.length 74 | else if (opts.end !== undefined) this._end = id.byteOffset + opts.end + 1 75 | else this._end = id.byteOffset + id.byteLength 76 | 77 | this._index = 0 78 | this._relativeOffset = 0 79 | this._bytesRead = 0 80 | } 81 | 82 | _open (cb) { 83 | if (this._pos === this.id.byteOffset) { 84 | this._index = this.id.blockOffset 85 | this._relativeOffset = 0 86 | return cb(null) 87 | } 88 | 89 | this.core.seek(this._pos, { 90 | start: this.id.blockOffset, 91 | end: this.id.blockOffset + this.id.blockLength 92 | }).then(result => { 93 | if (!result) return cb(BLOCK_NOT_AVAILABLE()) 94 | 95 | this._index = result[0] 96 | this._relativeOffset = result[1] 97 | return cb(null) 98 | }, err => cb(err)) 99 | } 100 | 101 | _predestroy () { 102 | if (this._prefetch) this._prefetch.destroy() 103 | this.core.close().then(noop, noop) 104 | } 105 | 106 | _destroy (cb) { 107 | if (this._prefetch) this._prefetch.destroy() 108 | this.core.close().then(cb, cb) 109 | } 110 | 111 | _read (cb) { 112 | if (this._pos >= this._end) { 113 | this.push(null) 114 | return cb(null) 115 | } 116 | 117 | if (this._prefetch) this._prefetch.update(this._index) 118 | 119 | this.core.get(this._index).then(block => { 120 | if (!block) return cb(BLOCK_NOT_AVAILABLE()) 121 | 122 | const remainder = this._end - this._pos 123 | if (this._relativeOffset || (remainder < block.length)) { 124 | block = block.subarray(this._relativeOffset, this._relativeOffset + remainder) 125 | } 126 | 127 | this._index++ 128 | this._relativeOffset = 0 129 | this._pos += block.length 130 | this._bytesRead += block.length 131 | 132 | this.push(block) 133 | return cb(null) 134 | }, err => cb(err)) 135 | } 136 | } 137 | 138 | module.exports = { 139 | BlobReadStream, 140 | BlobWriteStream 141 | } 142 | 143 | function noop () {} 144 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hyperblobs", 3 | "version": "2.8.0", 4 | "description": "A blob store for Hypercore", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "standard && brittle test/*.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/holepunchto/hyperblobs.git" 12 | }, 13 | "keywords": [ 14 | "hypercore", 15 | "blob", 16 | "store" 17 | ], 18 | "author": "Andrew Osheroff ", 19 | "license": "Apache-2.0", 20 | "bugs": { 21 | "url": "https://github.com/holepunchto/hyperblobs/issues" 22 | }, 23 | "homepage": "https://github.com/holepunchto/hyperblobs#readme", 24 | "files": [ 25 | "index.js", 26 | "lib/**.js" 27 | ], 28 | "imports": { 29 | "events": { 30 | "bare": "bare-events", 31 | "default": "events" 32 | } 33 | }, 34 | "dependencies": { 35 | "b4a": "^1.6.1", 36 | "bare-events": "^2.5.0", 37 | "hypercore-errors": "^1.1.1", 38 | "mutexify": "^1.4.0", 39 | "speedometer": "^1.1.0", 40 | "streamx": "^2.13.2" 41 | }, 42 | "devDependencies": { 43 | "brittle": "^3.1.0", 44 | "hypercore": "^10.18.0", 45 | "random-access-memory": "^6.0.0", 46 | "standard": "^17.0.0" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test/all.js: -------------------------------------------------------------------------------- 1 | const test = require('brittle') 2 | const b4a = require('b4a') 3 | const Hypercore = require('hypercore') 4 | const RAM = require('random-access-memory') 5 | 6 | const Hyperblobs = require('..') 7 | 8 | test('can get/put a large blob', async t => { 9 | const core = new Hypercore(RAM) 10 | const blobs = new Hyperblobs(core) 11 | 12 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 13 | const id = await blobs.put(buf) 14 | const result = await blobs.get(id) 15 | 16 | t.alike(result, buf) 17 | }) 18 | 19 | test('can put/get two blobs in one core', async t => { 20 | const core = new Hypercore(RAM) 21 | const blobs = new Hyperblobs(core) 22 | 23 | { 24 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 25 | const id = await blobs.put(buf) 26 | const res = await blobs.get(id) 27 | 28 | t.alike(res, buf) 29 | } 30 | 31 | { 32 | const buf = b4a.alloc(5 * blobs.blockSize, 'hijklmn') 33 | const id = await blobs.put(buf) 34 | const res = await blobs.get(id) 35 | 36 | t.alike(res, buf) 37 | } 38 | }) 39 | 40 | test('can seek to start/length within one blob, one block', async t => { 41 | const core = new Hypercore(RAM) 42 | const blobs = new Hyperblobs(core) 43 | 44 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 45 | const id = await blobs.put(buf) 46 | const result = await blobs.get(id, { start: 2, length: 2 }) 47 | 48 | t.alike(b4a.toString(result, 'utf-8'), 'cd') 49 | }) 50 | 51 | test('can seek to start/length within one blob, multiple blocks', async t => { 52 | const core = new Hypercore(RAM) 53 | const blobs = new Hyperblobs(core, { blockSize: 10 }) 54 | 55 | const buf = b4a.concat([b4a.alloc(10, 'a'), b4a.alloc(10, 'b')]) 56 | const id = await blobs.put(buf) 57 | const result = await blobs.get(id, { start: 8, length: 4 }) 58 | 59 | t.is(b4a.toString(result, 'utf-8'), 'aabb') 60 | }) 61 | 62 | test('can seek to start/length within one blob, multiple blocks, multiple blobs', async t => { 63 | const core = new Hypercore(RAM) 64 | const blobs = new Hyperblobs(core, { blockSize: 10 }) 65 | 66 | { 67 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 68 | const id = await blobs.put(buf) 69 | const res = await blobs.get(id) 70 | 71 | t.alike(res, buf) 72 | } 73 | 74 | const buf = b4a.concat([b4a.alloc(10, 'a'), b4a.alloc(10, 'b')]) 75 | const id = await blobs.put(buf) 76 | const result = await blobs.get(id, { start: 8, length: 4 }) 77 | 78 | t.is(b4a.toString(result, 'utf-8'), 'aabb') 79 | }) 80 | 81 | test('can seek to start/end within one blob', async t => { 82 | const core = new Hypercore(RAM) 83 | const blobs = new Hyperblobs(core) 84 | 85 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 86 | const id = await blobs.put(buf) 87 | const result = await blobs.get(id, { start: 2, end: 4 }) // inclusive 88 | 89 | t.is(b4a.toString(result, 'utf-8'), 'cde') 90 | }) 91 | 92 | test('basic seek', async t => { 93 | const core = new Hypercore(RAM) 94 | const blobs = new Hyperblobs(core) 95 | 96 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 97 | const id = await blobs.put(buf) 98 | const start = blobs.blockSize + 424 99 | const result = await blobs.get(id, { start }) 100 | 101 | t.alike(result, buf.subarray(start)) 102 | }) 103 | 104 | test('can pass in a custom core', async t => { 105 | const core1 = new Hypercore(RAM) 106 | const core2 = new Hypercore(RAM) 107 | const blobs = new Hyperblobs(core1) 108 | await core1.ready() 109 | 110 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 111 | const id = await blobs.put(buf, { core: core2 }) 112 | const result = await blobs.get(id, { core: core2 }) 113 | 114 | t.alike(result, buf) 115 | t.is(core1.length, 0) 116 | }) 117 | 118 | test('two write streams does not deadlock', async t => { 119 | t.plan(2) 120 | 121 | const core = new Hypercore(RAM) 122 | const blobs = new Hyperblobs(core) 123 | await core.ready() 124 | 125 | const ws = blobs.createWriteStream() 126 | 127 | ws.on('open', () => ws.destroy()) 128 | ws.on('drain', () => t.comment('ws drained')) 129 | ws.on('close', () => t.pass('ws closed')) 130 | 131 | ws.on('close', function () { 132 | const ws2 = blobs.createWriteStream() 133 | ws2.write(b4a.from('hello')) 134 | ws2.end() 135 | ws2.on('close', () => t.pass('ws2 closed')) 136 | }) 137 | }) 138 | 139 | test('append error does not deadlock', async t => { 140 | t.plan(2) 141 | 142 | const core = new Hypercore(RAM) 143 | const blobs = new Hyperblobs(core) 144 | await core.ready() 145 | 146 | const ws = blobs.createWriteStream() 147 | 148 | ws.on('open', async function () { 149 | await core.close() 150 | 151 | ws.write(b4a.from('hello')) 152 | ws.end() 153 | }) 154 | 155 | ws.on('drain', () => t.comment('ws drained')) 156 | ws.on('error', (err) => t.comment('ws error: ' + err.message)) 157 | ws.on('close', () => t.pass('ws closed')) 158 | 159 | ws.on('close', function () { 160 | const core2 = new Hypercore(RAM) 161 | const ws2 = blobs.createWriteStream({ core: core2 }) 162 | ws2.write(b4a.from('hello')) 163 | ws2.end() 164 | ws2.on('close', () => t.pass('ws2 closed')) 165 | }) 166 | }) 167 | 168 | test('can put/get a blob and clear it', async t => { 169 | const core = new Hypercore(RAM) 170 | const blobs = new Hyperblobs(core) 171 | 172 | const buf = b4a.alloc(5 * blobs.blockSize, 'abcdefg') 173 | const id = await blobs.put(buf) 174 | 175 | t.alike(await blobs.get(id), buf) 176 | 177 | await blobs.clear(id) 178 | 179 | for (let i = 0; i < id.blockLength; i++) { 180 | const block = id.blockOffset + i 181 | t.absent(await core.has(block), `block ${block} cleared`) 182 | } 183 | }) 184 | 185 | test('get with timeout', async function (t) { 186 | t.plan(1) 187 | 188 | const [, b] = await createPair() 189 | const blobs = new Hyperblobs(b) 190 | 191 | try { 192 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 193 | await blobs.get(id, { timeout: 1 }) 194 | t.fail('should have failed') 195 | } catch (error) { 196 | t.is(error.code, 'REQUEST_TIMEOUT') 197 | } 198 | }) 199 | 200 | test('seek with timeout', async function (t) { 201 | t.plan(1) 202 | 203 | const [, b] = await createPair() 204 | const blobs = new Hyperblobs(b) 205 | 206 | try { 207 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 208 | await blobs.get(id, { start: 100, timeout: 1 }) 209 | t.fail('should have failed') 210 | } catch (error) { 211 | t.is(error.code, 'REQUEST_TIMEOUT') 212 | } 213 | }) 214 | 215 | test('get without waiting', async function (t) { 216 | t.plan(1) 217 | 218 | const [, b] = await createPair() 219 | const blobs = new Hyperblobs(b) 220 | 221 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 222 | const blob = await blobs.get(id, { wait: false }) 223 | t.is(blob, null) 224 | }) 225 | 226 | test('seek without waiting', async function (t) { 227 | t.plan(1) 228 | 229 | const [, b] = await createPair() 230 | const blobs = new Hyperblobs(b) 231 | 232 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 233 | const blob = await blobs.get(id, { start: 100, wait: false }) 234 | t.is(blob, null) 235 | }) 236 | 237 | test('read stream with timeout', async function (t) { 238 | t.plan(1) 239 | 240 | const [, b] = await createPair() 241 | const blobs = new Hyperblobs(b) 242 | 243 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 244 | 245 | try { 246 | for await (const block of blobs.createReadStream(id, { timeout: 1 })) { 247 | t.fail('should not get any block: ' + block.toString()) 248 | } 249 | } catch (error) { 250 | t.is(error.code, 'REQUEST_TIMEOUT') 251 | } 252 | }) 253 | 254 | test('read stream without waiting', async function (t) { 255 | t.plan(1) 256 | 257 | const [, b] = await createPair() 258 | const blobs = new Hyperblobs(b) 259 | 260 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 261 | 262 | try { 263 | for await (const block of blobs.createReadStream(id, { wait: false })) { 264 | t.fail('should not get any block: ' + block.toString()) 265 | } 266 | } catch (error) { 267 | t.is(error.code, 'BLOCK_NOT_AVAILABLE') 268 | } 269 | }) 270 | 271 | test('seek stream without waiting', async function (t) { 272 | t.plan(1) 273 | 274 | const [, b] = await createPair() 275 | const blobs = new Hyperblobs(b) 276 | 277 | const id = { byteOffset: 5, blockOffset: 1, blockLength: 1, byteLength: 5 } 278 | 279 | try { 280 | for await (const block of blobs.createReadStream(id, { start: 100, wait: false })) { 281 | t.fail('should not get any block: ' + block.toString()) 282 | } 283 | } catch (error) { 284 | t.is(error.code, 'BLOCK_NOT_AVAILABLE') 285 | } 286 | }) 287 | 288 | test('clear with diff option', async function (t) { 289 | t.plan(3) 290 | 291 | const core = new Hypercore(() => new RAM({ pageSize: 128 })) 292 | const blobs = new Hyperblobs(core) 293 | 294 | const buf = b4a.alloc(128) 295 | const id = await blobs.put(buf) 296 | const id2 = await blobs.put(buf) 297 | 298 | const cleared = await blobs.clear(id) 299 | t.is(cleared, null) 300 | 301 | const cleared2 = await blobs.clear(id2, { diff: true }) 302 | t.ok(cleared2.blocks > 0) 303 | 304 | const cleared3 = await blobs.clear(id2, { diff: true }) 305 | t.is(cleared3.blocks, 0) 306 | }) 307 | 308 | test('upload/download can be monitored', async (t) => { 309 | t.plan(30) 310 | 311 | const [a, b] = await createPair() 312 | const blobsA = new Hyperblobs(a) 313 | const blobsB = new Hyperblobs(b) 314 | 315 | const bytes = 1024 * 100 // big enough to trigger more than one update event 316 | const buf = Buffer.alloc(bytes, '0') 317 | const id = await blobsA.put(buf) 318 | 319 | // add another blob which should not be monitored 320 | const controlId = await blobsA.put(buf) 321 | 322 | { 323 | const expectedBlocks = [2, 1] 324 | const expectedBytes = [bytes, 65536] 325 | const expectedPercentage = [100, 50] 326 | 327 | // Start monitoring upload 328 | const monitor = blobsA.monitor(id) 329 | monitor.on('update', () => { 330 | t.is(monitor.uploadStats.blocks, expectedBlocks.pop()) 331 | t.is(monitor.uploadStats.monitoringBytes, expectedBytes.pop()) 332 | t.is(monitor.uploadStats.targetBlocks, 2) 333 | t.is(monitor.uploadStats.targetBytes, bytes) 334 | t.is(monitor.uploadSpeed(), monitor.uploadStats.speed) 335 | t.is(monitor.uploadStats.percentage, expectedPercentage.pop()) 336 | t.absent(monitor.downloadStats.blocks) 337 | }) 338 | } 339 | 340 | { 341 | // Start monitoring download 342 | const expectedBlocks = [2, 1] 343 | const expectedBytes = [bytes, 65536] 344 | const expectedPercentage = [100, 50] 345 | 346 | const monitor = blobsB.monitor(id) 347 | monitor.on('update', () => { 348 | t.is(monitor.downloadStats.blocks, expectedBlocks.pop()) 349 | t.is(monitor.downloadStats.monitoringBytes, expectedBytes.pop()) 350 | t.is(monitor.downloadStats.targetBlocks, 2) 351 | t.is(monitor.downloadStats.targetBytes, bytes) 352 | t.is(monitor.downloadSpeed(), monitor.downloadStats.speed) 353 | t.is(monitor.downloadStats.percentage, expectedPercentage.pop()) 354 | t.absent(monitor.uploadStats.blocks) 355 | }) 356 | } 357 | 358 | const res = await blobsB.get(id) 359 | t.alike(res, buf) 360 | 361 | // should not generate events 362 | const controRes = await blobsB.get(controlId) 363 | t.alike(controRes, buf) 364 | }) 365 | 366 | test('monitor is removed from the Set on close', async (t) => { 367 | const core = new Hypercore(RAM) 368 | const blobs = new Hyperblobs(core) 369 | 370 | const bytes = 1024 * 100 // big enough to trigger more than one update event 371 | const buf = Buffer.alloc(bytes, '0') 372 | const id = await blobs.put(buf) 373 | const monitor = blobs.monitor(id) 374 | t.is(blobs._monitors.size, 1) 375 | monitor.close() 376 | t.is(blobs._monitors.size, 0) 377 | }) 378 | 379 | test('basic batch', async (t) => { 380 | const core = new Hypercore(RAM) 381 | const blobs = new Hyperblobs(core) 382 | const batch = blobs.batch() 383 | 384 | { 385 | const id = await batch.put(Buffer.from('hello world')) 386 | const buf = await batch.get(id) 387 | t.alike(buf, Buffer.from('hello world')) 388 | } 389 | 390 | { 391 | const id = await batch.put(Buffer.from('hej verden')) 392 | const buf = await batch.get(id) 393 | t.alike(buf, Buffer.from('hej verden')) 394 | } 395 | 396 | await batch.flush() 397 | }) 398 | 399 | async function createPair () { 400 | const a = new Hypercore(RAM) 401 | await a.ready() 402 | 403 | const b = new Hypercore(RAM, a.key) 404 | await b.ready() 405 | 406 | replicate(a, b) 407 | 408 | return [a, b] 409 | } 410 | 411 | function replicate (a, b) { 412 | const s1 = a.replicate(true, { keepAlive: false }) 413 | const s2 = b.replicate(false, { keepAlive: false }) 414 | s1.pipe(s2).pipe(s1) 415 | return [s1, s2] 416 | } 417 | --------------------------------------------------------------------------------