├── .editorconfig ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── package-lock.json ├── package.json ├── src └── index.js └── tests └── index.test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "node" 4 | - "12" 5 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## 1.0.7 - 2022-06-21 [YANKED] 10 | 11 | ## 1.0.6 - 2021-07-16 [YANKED] 12 | 13 | ## 1.0.5 - 2021-02-24 [YANKED] 14 | 15 | ## 1.0.4 - 2021-02-24 [YANKED] 16 | 17 | ## 1.0.3 - 2021-02-09 [YANKED] 18 | 19 | ## 1.0.2 - 2021-02-09 [YANKED] 20 | 21 | ## 1.0.1 - 2021-01-27 [YANKED] 22 | 23 | ## 1.0.0 - 2021-01-27 [YANKED] 24 | [Unreleased]: https://github.com/geut/hyperbee-live-stream/compare/v1.0.7...HEAD 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to hyperbee-live-stream 2 | 3 | ## Issue Contributions 4 | 5 | When opening new issues or commenting on existing issues on this repository 6 | please make sure discussions are related to concrete technical issues. 7 | 8 | Try to be *friendly* (we are not animals :monkey: or bad people :rage4:) and explain correctly how we can reproduce your issue. 9 | 10 | ## Code Contributions 11 | 12 | This document will guide you through the contribution process. 13 | 14 | ### Step 1: Fork 15 | 16 | Fork the project [on GitHub](https://github.com/geut/hyperbee-live-stream) and check out your copy locally. 17 | 18 | ```bash 19 | $ git clone git@github.com:username/hyperbee-live-stream.git 20 | $ cd hyperbee-live-stream 21 | $ npm install 22 | $ git remote add upstream git://github.com/geut/hyperbee-live-stream.git 23 | ``` 24 | 25 | ### Step 2: Branch 26 | 27 | Create a feature branch and start hacking: 28 | 29 | ```bash 30 | $ git checkout -b my-feature-branch -t origin/main 31 | ``` 32 | 33 | ### Step 3: Test 34 | 35 | Bug fixes and features **should come with tests**. We use [jest](https://jestjs.io/) to do that. 36 | 37 | ```bash 38 | $ npm test 39 | ``` 40 | 41 | ### Step 4: Lint 42 | 43 | Make sure the linter is happy and that all tests pass. Please, do not submit 44 | patches that fail either check. 45 | 46 | We use [standard](https://standardjs.com/) 47 | 48 | ### Step 5: Commit 49 | 50 | Make sure git knows your name and email address: 51 | 52 | ```bash 53 | $ git config --global user.name "Bruce Wayne" 54 | $ git config --global user.email "bruce@batman.com" 55 | ``` 56 | 57 | Writing good commit logs is important. A commit log should describe what 58 | changed and why. 59 | 60 | ### Step 6: Changelog 61 | 62 | If your changes are really important for the project probably the users want to know about it. 63 | 64 | We use [chan](https://github.com/geut/chan/) to maintain a well readable changelog for our users. 65 | 66 | ### Step 7: Push 67 | 68 | ```bash 69 | $ git push origin my-feature-branch 70 | ``` 71 | 72 | ### Step 8: Make a pull request ;) 73 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 GEUT 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hyperbee-live-stream 2 | 3 | > Creates a ReadableStream but keep watching for changes in the range defined. 4 | 5 | [![Build Status](https://travis-ci.com/geut/hyperbee-live-stream.svg?branch=main)](https://travis-ci.com/geut/hyperbee-live-stream) 6 | [![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com) 7 | [![standard-readme compliant](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) 8 | 9 | [![Made by GEUT][geut-badge]][geut-url] 10 | 11 | ## Install 12 | 13 | $ npm install @geut/hyperbee-live-stream 14 | 15 | ## Usage 16 | 17 | ```js 18 | const { HyperbeeLiveStream } = require('@geut/hyperbee-live-stream') 19 | 20 | const stream = new HyperbeeLiveStream(db, { gte: 'a', lte: 'b' }) 21 | stream.on('data', data => console.log(data)) 22 | db.put('a') 23 | db.put('c') 24 | db.put('b') 25 | // will print a, b 26 | ``` 27 | 28 | ## API 29 | 30 | 31 | 32 | #### `hyperbeeLiveStream = new HyperbeeLiveStream(db, opts?)` 33 | 34 | * `db: Hyperbee` 35 | * `opts?: any = {}` 36 | * `old?: boolean = true` Iterate over the old items before start to watching 37 | * `gt?: Buffer | string` Only return keys > than this 38 | * `gte?: Buffer | string` Only return keys >= than this 39 | * `lt?: Buffer | string` Only return keys < than this 40 | * `lte?: Buffer | string` Only return keys <= than this 41 | * `reverse?: boolean = false` Set to true to get them in reverse order 42 | * `limit?: number = -1` Set to the max number of entries you want 43 | 44 | #### `hyperbeeLiveStream.version: number (R)` 45 | 46 | Returns the last matched version readed 47 | 48 | #### `hyperbeeLiveStream.on('synced', version) => void` 49 | 50 | Emitted when the stream is synced with the last version in the database 51 | 52 | * `version: number` 53 | 54 | ## Issues 55 | 56 | :bug: If you found an issue we encourage you to report it on [github](https://github.com/geut/hyperbee-live-stream/issues). Please specify your OS and the actions to reproduce it. 57 | 58 | ## Contributing 59 | 60 | :busts_in_silhouette: Ideas and contributions to the project are welcome. You must follow this [guideline](https://github.com/geut/hyperbee-live-stream/blob/main/CONTRIBUTING.md). 61 | 62 | ## License 63 | 64 | MIT © A [**GEUT**](http://geutstudio.com/) project 65 | 66 | [geut-url]: https://geutstudio.com 67 | 68 | [geut-badge]: https://img.shields.io/badge/Made%20By-GEUT-4f5186?style=for-the-badge&link=https://geutstudio.com&labelColor=white&logo=data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDABALDA4MChAODQ4SERATGCgaGBYWGDEjJR0oOjM9PDkzODdASFxOQERXRTc4UG1RV19iZ2hnPk1xeXBkeFxlZ2P/2wBDARESEhgVGC8aGi9jQjhCY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2P/wAARCABAAEADASIAAhEBAxEB/8QAGwAAAgIDAQAAAAAAAAAAAAAABAYDBQACBwH/xAA0EAACAQMBBAcGBgMAAAAAAAABAgMABBEFBhIhQRMiMVFhgcEUIzJxkbFCUmKh0fAkcuH/xAAYAQADAQEAAAAAAAAAAAAAAAABAwQCAP/EACARAAMAAwACAgMAAAAAAAAAAAABAgMRIRIxBEEiM1H/2gAMAwEAAhEDEQA/AOgVlau6xoXdgqqMkk8AKV9U2oYs0WngBRw6VhxPyFamXXoDeiz1PUbmzuujQIUKgjIqGLXnz72FSO9TikfVbi6uXWSSaWRuzixNBx3VzCepNIvgTw+hpjwv+iGr3tM6xa30F2PdP1uangRRNc70fUbi4JLIVaPskXgM/wA076Ze+2W+WwJF4MPWlNaemajI2/GvYbWVlZQHCptZqLNKLGJsKoDSY5nkKorKzlvrlYIRlm5nsA7zWX8pnv55SfikJ/emPZGDcs7m6CguTuL5DPrVf64Me2F2mzNhAg6ZTO/MsSB9BW15s1pt1GVEPRHvQ+hqbTNT9sZ0kCpIOIA5ij5ZEijaSRgqqMkmpVkb7sMuWtoV73S49L3I4B7kjq57c881BZ6vFpuoKjq7dIvYBw8PtUOqX1xcSxoJXw8mQuewVW3vX1eFR+Fcn96OLVvpFzz8kM020kp4QwIvixzVpot5Je2bSTEFw5HAY7qUKadnIymm7x/G5I+3pTskzM8G4rqq6JGpI8E1wi8HR2H0NT7P6rcRKUEzYR9/czgEf0VabV2JgvhdKPdzdvg399aVG37K4Esfw/3hTU1S2NpNrSHqax9q/wAzTm3lY5KA4ZTQl2mo9CWljncL+cnA+tVVhqeSGt5mik5qDg/9o+XVb6aFonuDusMHqjP2qavjbfGTPX3xgTstrm4uGDSEYVV+woWPMKy3dzwd+JHcOQrdkgtyZpXJb87nJ8qqr68a7cKgIjB4DmadGNQjohs9i1C66Xqtvbx+EjIp10jaOMLBaPasDwRTGc5PyNJ1rb9EN5/jP7U17KaaZJvbpV6icI88z3+VG0vH8ipJJ8Ga8tIr22eCYZVh5g94pC1TTJtPmMU67yH4XxwYV0So54IriIxzRrIh7QwzSIyOTbWzlElkCcxtjwNedHeKMCVseDmnq72UgkJa1maL9LDeH81XvspfA9WSBh/sR6U9XD+zDQp+yTSNmR/MnJomG3SLiBlu80zQ7JXTH31xEg/Tlj6Vb2OzljaEO6meQc5OweVc8koOmUGjaFLfuss4MdsOOewv8v5p0ijSGNY41CoowAOQrbsr2p7t0zSWj//Z 69 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@geut/hyperbee-live-stream", 3 | "version": "1.0.7", 4 | "description": "Hyperbee createReadStream live", 5 | "main": "src/index.js", 6 | "files": [ 7 | "lib", 8 | "src", 9 | "bin", 10 | "index.js" 11 | ], 12 | "scripts": { 13 | "start": "node index.js", 14 | "test": "jest --passWithNoTests", 15 | "posttest": "npm run lint", 16 | "lint": "standard", 17 | "version": "chan release --allow-yanked ${npm_package_version} && git add .", 18 | "prepublishOnly": "npm test" 19 | }, 20 | "dependencies": { 21 | "ltgt": "^2.2.1", 22 | "streamx": "^2.10.1" 23 | }, 24 | "devDependencies": { 25 | "@geut/chan": "^2.0.0", 26 | "hyperbee": "^1.5.0", 27 | "hypercore": "^9.7.0", 28 | "jest": "^24.8.0", 29 | "random-access-memory": "^3.1.2", 30 | "standard": "^16.0.3" 31 | }, 32 | "jest": { 33 | "testMatch": [ 34 | "**/tests/**/*.test.js" 35 | ] 36 | }, 37 | "standard": { 38 | "env": [ 39 | "jest", 40 | "node", 41 | "browser" 42 | ] 43 | }, 44 | "repository": { 45 | "type": "git", 46 | "url": "git+https://github.com/geut/hyperbee-live-stream.git" 47 | }, 48 | "keywords": [ 49 | "hyperbee", 50 | "stream", 51 | "live" 52 | ], 53 | "author": { 54 | "name": "GEUT", 55 | "email": "contact@geutstudio.com" 56 | }, 57 | "license": "MIT", 58 | "bugs": { 59 | "url": "https://github.com/geut/hyperbee-live-stream/issues" 60 | }, 61 | "homepage": "https://github.com/geut/hyperbee-live-stream#readme", 62 | "publishConfig": { 63 | "access": "public" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @typedef {Object} Hyperbee 3 | */ 4 | 5 | const { Readable } = require('streamx') 6 | const ltgt = require('ltgt') 7 | 8 | const SEP = Buffer.alloc(1) 9 | const MAX = Buffer.from([255]) 10 | 11 | function encRange (e, opts) { 12 | if (!e) return opts 13 | if (opts.gt !== undefined) opts.gt = enc(e, opts.gt) 14 | if (opts.gte !== undefined) opts.gte = enc(e, opts.gte) 15 | if (opts.lt !== undefined) opts.lt = enc(e, opts.lt) 16 | if (opts.lte !== undefined) opts.lte = enc(e, opts.lte) 17 | if (opts.sub && !opts.gt && !opts.gte) opts.gt = enc(e, SEP) 18 | if (opts.sub && !opts.lt && !opts.lte) opts.lt = enc(e, MAX) 19 | return opts 20 | } 21 | 22 | function enc (e, v) { 23 | if (v === undefined || v === null) return null 24 | if (e !== null) return e.encode(v) 25 | if (typeof v === 'string') return Buffer.from(v) 26 | return v 27 | } 28 | 29 | class HyperbeeLiveStream extends Readable { 30 | /** 31 | * 32 | * @param {Hyperbee} db 33 | * @param {Object} [opts] 34 | * @param {boolean} [opts.old=true] Iterate over the old items before start to watching 35 | * @param {Buffer|String} [opts.gt] Only return keys > than this 36 | * @param {Buffer|String} [opts.gte] Only return keys >= than this 37 | * @param {Buffer|String} [opts.lt] Only return keys < than this 38 | * @param {Buffer|String} [opts.lte] Only return keys <= than this 39 | * @param {boolean} [opts.reverse=false] Set to true to get them in reverse order 40 | * @param {number} [opts.limit=-1] Set to the max number of entries you want 41 | * 42 | */ 43 | constructor (db, opts = {}) { 44 | super() 45 | 46 | const { old = true, ...hyperbeeStreamOptions } = opts 47 | this._db = db 48 | this._old = old 49 | this._opts = hyperbeeStreamOptions 50 | this._range = encRange(this._db.keyEncoding, { ...this._opts, sub: this._db._sub }) 51 | this._pushOldValue = this._pushOldValue.bind(this) 52 | this._pushNextValue = this._pushNextValue.bind(this) 53 | this._version = 0 54 | } 55 | 56 | /** 57 | * Returns the last matched version readed 58 | * @type {number} 59 | */ 60 | get version () { 61 | return this._version 62 | } 63 | 64 | _open (cb) { 65 | this._db.ready() 66 | .then(() => { 67 | if (this._old) this._oldIterator = this._db.createReadStream(this._opts)[Symbol.asyncIterator]() 68 | cb(null) 69 | }) 70 | .catch(err => cb(err)) 71 | } 72 | 73 | _read (cb) { 74 | if (this._oldIterator) { 75 | return this._oldIterator.next() 76 | .then(this._pushOldValue) 77 | .then(done => { 78 | if (done) { 79 | this._read(cb) 80 | } else { 81 | cb(null, null) 82 | } 83 | }) 84 | .catch(cb) 85 | } 86 | 87 | if (!this._nextIterator) { 88 | let startVersion = this._version 89 | if (this._version === 0) { 90 | startVersion = this._version = this._db.version 91 | } else if (this._old) { 92 | startVersion++ 93 | } 94 | 95 | this.emit('synced', this._version) 96 | this._nextIterator = this._db.createHistoryStream({ live: true, gte: startVersion })[Symbol.asyncIterator]() 97 | } 98 | 99 | return this._nextIterator.next() 100 | .then(this._pushNextValue) 101 | .then(readMore => { 102 | if (readMore) { 103 | this._read(cb) 104 | } else { 105 | cb(null, null) 106 | } 107 | }) 108 | .catch(cb) 109 | } 110 | 111 | _pushOldValue (data) { 112 | if (data.done) { 113 | this._oldIterator = null 114 | } else { 115 | if (data.value.seq > this._version) { 116 | this._version = data.value.seq 117 | } 118 | this.push(data.value) 119 | } 120 | 121 | return data.done 122 | } 123 | 124 | _pushNextValue (data) { 125 | if (data.done) { 126 | this.push(null) 127 | return false 128 | } 129 | 130 | if (ltgt.contains(this._range, (this._db.prefix ? this._db.prefix.toString() : '') + data.value.key)) { 131 | this._version = data.value.seq 132 | this.push(data.value) 133 | return false 134 | } 135 | 136 | return true 137 | } 138 | 139 | _predestroy () { 140 | const iterator = this._oldIterator || this._nextIterator 141 | if (iterator) { 142 | this._iteratorToDestroy = iterator.return() 143 | } 144 | } 145 | 146 | _destroy (cb) { 147 | if (this._iteratorToDestroy) { 148 | this._iteratorToDestroy.then(cb.bind(null, null)).catch(cb) 149 | return 150 | } 151 | 152 | cb(null) 153 | } 154 | } 155 | 156 | module.exports = { HyperbeeLiveStream } 157 | 158 | /** 159 | * Emitted when the stream is synced with the last version in the database 160 | * @event HyperbeeLiveStream#synced 161 | * @param {number} version 162 | */ 163 | -------------------------------------------------------------------------------- /tests/index.test.js: -------------------------------------------------------------------------------- 1 | const hypercore = require('hypercore') 2 | const Hyperbee = require('hyperbee') 3 | const ram = require('random-access-memory') 4 | 5 | const { HyperbeeLiveStream } = require('..') 6 | 7 | const createDB = (opts = {}) => new Hyperbee(hypercore(ram), opts) 8 | 9 | const getResult = async (db, opts, end = 500) => { 10 | const result = [] 11 | const stream = new HyperbeeLiveStream(db, opts) 12 | const timer = setTimeout(() => stream.destroy(), end) 13 | try { 14 | for await (const data of stream) { 15 | result.push(data) 16 | } 17 | } catch (err) { 18 | if (err.message.includes('destroyed')) return result 19 | throw err 20 | } 21 | clearTimeout(timer) 22 | return result 23 | } 24 | 25 | test('HyperbeeLiveStream opts = {}', async () => { 26 | const db = createDB({ keyEncoding: 'utf-8', valueEncoding: 'utf-8' }) 27 | 28 | const seed = [0, 1, 4, 3, 2].map(v => v.toString()) 29 | await Promise.all(seed.map(key => db.put(key))) 30 | 31 | let result = getResult(db, undefined) 32 | 33 | db.put('5') 34 | db.put('0') 35 | 36 | result = (await result).map(data => data.key) 37 | await expect(result).toEqual([...seed.sort(), '5', '0']) 38 | }) 39 | 40 | test('HyperbeeLiveStream opts = { gte: "b", lte: "c" }', async () => { 41 | const db = createDB({ keyEncoding: 'utf-8', valueEncoding: 'utf-8' }) 42 | 43 | const seed = ['a', 'c', 'd', 'f', 'b', 'e'] 44 | await Promise.all(seed.map(key => db.put(key))) 45 | 46 | let result = getResult(db, { gte: Buffer.from('b'), lte: Buffer.from('d') }) 47 | 48 | await Promise.all([ 49 | db.put('a'), 50 | db.put('b') 51 | ]) 52 | 53 | result = (await result).map(data => data.key) 54 | await expect(result).toEqual([...seed.sort().filter(w => w >= 'b' && w <= 'd'), 'b']) 55 | }) 56 | 57 | test('HyperbeeLiveStream opts = { old: false }', async () => { 58 | const db = createDB({ keyEncoding: 'utf-8', valueEncoding: 'utf-8' }) 59 | 60 | const seed = [0, 1, 2].map(v => v.toString()) 61 | await Promise.all(seed.map(key => db.put(key))) 62 | 63 | let result = getResult(db, { old: false }) 64 | 65 | await Promise.all([ 66 | db.put('3'), 67 | db.put('4') 68 | ]) 69 | 70 | result = (await result).map(data => data.key) 71 | await expect(result).toEqual(['3', '4']) 72 | }) 73 | --------------------------------------------------------------------------------