├── example ├── kv.js └── mem.js ├── index.js ├── license ├── package.json ├── readme.md ├── schema.proto └── test ├── duplex.js ├── pubsub.js └── string-key.js /example/kv.js: -------------------------------------------------------------------------------- 1 | var minimist = require('minimist') 2 | var argv = minimist(process.argv.slice(2), { 3 | alias: { d: 'datadir' } 4 | }) 5 | var pump = require('pump') 6 | var to = require('to2') 7 | var path = require('path') 8 | var { Readable } = require('readable-stream') 9 | 10 | var Protocol = require('hypercore-protocol') 11 | var swarm = require('discovery-swarm') 12 | 13 | var umkvl = require('unordered-materialized-kv-live') 14 | var db = require('level')(path.join(argv.datadir,'db')) 15 | var kv = umkvl(db) 16 | 17 | var raf = require('random-access-file') 18 | var Storage = require('multifeed-storage') 19 | var storage = new Storage(function (name) { 20 | return raf(path.join(argv.datadir,name)) 21 | }) 22 | var Replicate = require('multifeed-replicate') 23 | var Query = require('../') 24 | 25 | if (argv._[0] === 'put') { 26 | var doc = { 27 | key: argv._[1], 28 | value: argv._[2], 29 | links: [].concat(argv.link || []) 30 | } 31 | storage.getOrCreateLocal('feed', { valueEncoding: 'json' }, function (err, feed) { 32 | if (err) return console.error(err) 33 | feed.append(doc, function (err, seq) { 34 | if (err) console.error(err) 35 | var kdoc = { 36 | id: feed.key.toString('hex') + '@' + seq, 37 | key: doc.key, 38 | links: doc.links 39 | } 40 | kv.batch([kdoc], function (err) { 41 | if (err) console.error(err) 42 | }) 43 | }) 44 | }) 45 | } else if (argv._[0] === 'get') { 46 | kv.open(argv._.slice(1)) 47 | kv.on('value', function (key, ids) { 48 | ids.forEach(function (id) { 49 | var [key,seq] = id.split('@') 50 | storage.getOrCreateRemote(key, { valueEncoding: 'json' }, function (err, feed) { 51 | feed.get(Number(seq), { valueEncoding: 'json' }, function (err, doc) { 52 | console.log(`${id} ${doc.key} => ${doc.value}`) 53 | }) 54 | }) 55 | }) 56 | }) 57 | connect(function (r, q) { 58 | var s = q.query('get', Buffer.from(argv._[1])) 59 | s.pipe(to.obj(function (row, enc, next) { 60 | storage.getOrCreateRemote(row.key, function (err, feed) { 61 | if (err) return next(err) 62 | if (feed.has(row.seq)) return 63 | r.open(row.key, { live: true, sparse: true }) 64 | feed.update(row.seq+1, function () { 65 | feed.get(row.seq, { valueEncoding: 'json' }, function (err, doc) { 66 | if (err) return next(err) 67 | var kdoc = { 68 | id: row.key.toString('hex') + '@' + row.seq, 69 | key: doc.key, 70 | links: doc.links || [] 71 | } 72 | kv.batch([kdoc], next) 73 | }) 74 | }) 75 | }) 76 | })) 77 | }) 78 | } else if (argv._[0] === 'connect') { 79 | connect() 80 | } 81 | 82 | function connect (f) { 83 | var sw = swarm() 84 | sw.join(argv.swarm) 85 | sw.on('connection', function (stream, info) { 86 | var p = new Protocol(info.initiator, { 87 | download: false, 88 | live: true 89 | }) 90 | var q = new Query({ api: { get } }) 91 | function get (data) { 92 | kv.open(data) 93 | 94 | var r = new Readable({ 95 | objectMode: true, 96 | read: function () { 97 | kv.get(data, function (err, ids) { 98 | if (err) return r.emit('error', err) 99 | ids.forEach(function (id) { 100 | var [key,seq] = id.split('@') 101 | r.push({ 102 | key: Buffer.from(key,'hex'), 103 | seq: Number(seq) 104 | }) 105 | }) 106 | r.push(null) 107 | }) 108 | } 109 | }) 110 | return r 111 | } 112 | q.register(p, 'kv') 113 | var r = new Replicate(storage, p, { live: true, sparse: true }) 114 | pump(stream, p, stream) 115 | if (f) f(r, q) 116 | }) 117 | } 118 | -------------------------------------------------------------------------------- /example/mem.js: -------------------------------------------------------------------------------- 1 | var { Readable, Transform } = require('readable-stream') 2 | var Query = require('../') 3 | var ram = require('random-access-memory') 4 | 5 | var hypercore = require('hypercore') 6 | var feed0 = hypercore(ram) 7 | 8 | setInterval(function () { 9 | var n = Math.floor(Math.random()*100) 10 | feed0.append(String(n)) 11 | }, 50) 12 | 13 | feed0.ready(function () { 14 | var feed1 = hypercore(ram, feed0.key) 15 | var r0 = feed0.replicate(false, { download: false, live: true }) 16 | var r1 = feed1.replicate(true, { sparse: true, live: true }) 17 | r0.pipe(r1).pipe(r0) 18 | var q0 = new Query({ api: api(feed0) }) 19 | var q1 = new Query({ api: api(feed1) }) 20 | r0.registerExtension('query-example', q0.extension()) 21 | r1.registerExtension('query-example', q1.extension()) 22 | var s = q1.query('subscribe', JSON.stringify({ start: 50, end: 70 })) 23 | s.pipe(new Transform({ 24 | objectMode: true, 25 | transform: function (row, enc, next) { 26 | if (!row.key.equals(feed0.key)) return next() 27 | feed1.update(row.seq, function () { 28 | feed1.get(row.seq, function (err, buf) { 29 | if (err) return next(err) 30 | console.log('n=', Number(buf.toString())) 31 | next() 32 | }) 33 | }) 34 | } 35 | })) 36 | }) 37 | 38 | function api (feed) { 39 | var subs = [] 40 | feed.on('append', function () { 41 | var seq = feed.length 42 | feed.get(seq, function (err, buf) { 43 | var n = Number(buf.toString()) 44 | subs.forEach(({ start, end, stream }) => { 45 | if (n >= start && n <= end) { 46 | stream.push({ key: feed.key, seq }) 47 | } 48 | }) 49 | }) 50 | }) 51 | return { subscribe } 52 | function subscribe (args) { 53 | var { start, end } = JSON.parse(args.toString()) 54 | var stream = new Readable({ 55 | objectMode: true, 56 | read: function () {} 57 | }) 58 | subs.push({ start, end, stream }) 59 | return stream 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var { EventEmitter } = require('events') 2 | var { Duplex } = require('readable-stream') 3 | var onend = require('end-of-stream') 4 | var messages = require('./messages.js') 5 | var types = { 6 | Open: 0, 7 | Read: 1, 8 | Control: 2, 9 | QueryDef: 3, 10 | Response: 4, 11 | FeedDef: 5, 12 | Write: 6 13 | } 14 | var codes = messages.Control.ControlCode 15 | 16 | module.exports = Query 17 | 18 | function Query (opts) { 19 | if (!(this instanceof Query)) return new Query(opts) 20 | if (!opts) opts = {} 21 | this._api = opts.api || {} 22 | this._queryDefs = {} 23 | this._feedDefs = {} 24 | this._queries = {} 25 | this._readers = {} 26 | this._sentQueries = {} 27 | this._sentQueryId = 0 28 | this._sentQueryDefs = {} 29 | this._sentQueryDefId = 0 30 | this._sentFeedDefs = {} 31 | this._sentFeedDefId = 0 32 | } 33 | Query.prototype = Object.create(EventEmitter.prototype) 34 | 35 | Query.prototype.query = function (name, data) { 36 | var self = this 37 | if (!self._sentQueries.hasOwnProperty(name)) { 38 | var qid = self._sentQueryDefId++ 39 | self._sentQueryDefs[name] = qid 40 | self._send('QueryDef', { id: qid, name }) 41 | } 42 | var id = self._sentQueryId++ 43 | self._sentQueries[id] = new Duplex({ 44 | objectMode: true, 45 | read: function (n) { 46 | self._send('Read', { id, n }) 47 | }, 48 | write: function (data, enc, next) { 49 | self._send('Write', { id, data }) 50 | next() 51 | } 52 | }) 53 | self._send('Open', { 54 | id, 55 | query_id: self._sentQueryDefs[name], 56 | data 57 | }) 58 | return self._sentQueries[id] 59 | } 60 | 61 | Query.prototype._send = function (type, msg) { 62 | this._ext.send(Buffer.concat([ 63 | Buffer.from([types[type]]), 64 | messages[type].encode(msg) 65 | ])) 66 | } 67 | 68 | Query.prototype._handle = function (msg) { 69 | if (msg[0] === types.Open) { 70 | this._handleOpen(messages.Open.decode(msg, 1)) 71 | } else if (msg[0] === types.Read) { 72 | this._handleRead(messages.Read.decode(msg, 1)) 73 | } else if (msg[0] === types.Control) { 74 | this._handleControl(messages.Control.decode(msg, 1)) 75 | } else if (msg[0] === types.QueryDef) { 76 | var m = messages.QueryDef.decode(msg.slice(1)) 77 | this._queryDefs[m.id] = m.name 78 | } else if (msg[0] === types.Response) { 79 | var m = messages.Response.decode(msg, 1) 80 | var q = this._sentQueries[m.query_id] 81 | if (!q) return 82 | q.push({ 83 | key: this._feedDefs[m.result.id], 84 | seq: m.result.seq 85 | }) 86 | } else if (msg[0] === types.FeedDef) { 87 | var m = messages.FeedDef.decode(msg, 1) 88 | this._feedDefs[m.id] = m.key 89 | } else if (msg[0] === types.Write) { 90 | this._handleWrite(messages.Write.decode(msg, 1)) 91 | } 92 | } 93 | 94 | Query.prototype._handleOpen = function (m) { 95 | var self = this 96 | var name = self._queryDefs[m.query_id] 97 | if (!self._api.hasOwnProperty(name)) return 98 | if (typeof self._api[name] !== 'function') return 99 | var q = self._api[name](m.data) 100 | if (!q || typeof q.pipe !== 'function') return 101 | self._queries[m.id] = q 102 | self._readers[m.id] = reader(q) 103 | q.on('end', () => { 104 | self._send('Control', { 105 | id: m.query_id, 106 | code: codes.END 107 | }) 108 | }) 109 | onend(q, function () { 110 | delete self._queries[m.id] 111 | delete self._readers[m.id] 112 | }) 113 | } 114 | 115 | Query.prototype._handleRead = function (m) { 116 | var self = this 117 | if (!self._readers[m.id]) return 118 | self._readers[m.id](m.n, function (err, res) { 119 | if (typeof res.key === 'string' && /^[0-9A-Fa-f]+$/.test(res.key)) { 120 | var hkey = res.key 121 | var key = Buffer.from(hkey, 'hex') 122 | } else if (Buffer.isBuffer(res.key)) { 123 | var key = res.key 124 | var hkey = key.toString('hex') 125 | } else { 126 | return 127 | } 128 | if (!self._sentFeedDefs.hasOwnProperty(hkey)) { 129 | self._send('FeedDef', { 130 | key, 131 | id: self._sentFeedDefId 132 | }) 133 | self._sentFeedDefs[hkey] = self._sentFeedDefId++ 134 | } 135 | self._send('Response', { 136 | query_id: m.id, 137 | result: { 138 | id: self._sentFeedDefs[hkey], 139 | seq: res.seq 140 | } 141 | }) 142 | }) 143 | } 144 | 145 | Query.prototype._handleWrite = function (m) { 146 | var self = this 147 | var q = self._queries[m.id] 148 | if (!q) return 149 | if (typeof q.write === 'function') q.write(m.data) 150 | } 151 | 152 | Query.prototype._handleControl = function (m) { 153 | const queries = m.sentQuery ? this._sentQueries : this._queries 154 | if (m.code === codes.END) { 155 | var q = this._sentQueries[m.id] 156 | q.push(null) 157 | delete this._sentQueries[m.id] 158 | } else if (m.code === codes.CLOSE) { 159 | var q = this._queries[m.id] 160 | if (q && typeof q.close === 'function') q.close() 161 | if (m.sentQuery) q.push(null) 162 | delete this._queries[m.id] 163 | } else if (m.code === codes.DESTROY) { 164 | q = this._queries[m.id] 165 | if (q && typeof q.destroy === 'function') q.destroy() 166 | delete this._queries[m.id] 167 | } 168 | } 169 | 170 | Query.prototype.extension = function () { 171 | var self = this 172 | return function (ext) { 173 | self._ext = ext 174 | return { 175 | encoding: 'binary', 176 | onmessage: function (msg, peer) { 177 | self._handle(msg) 178 | }, 179 | onerror: function (err) { 180 | self.emit('error', err) 181 | } 182 | } 183 | } 184 | } 185 | 186 | function reader (stream) { 187 | var queue = [], ready = true 188 | stream.on('readable', onreadable) 189 | return function (n, cb) { 190 | queue.push([Math.max(n || 1, 1),cb]) 191 | if (ready) read() 192 | } 193 | function onreadable () { 194 | ready = true 195 | read() 196 | } 197 | function read () { 198 | while (ready) { 199 | if (queue.length === 0) return 200 | var q = queue[0] 201 | var res = stream.read(q[0]) 202 | if (res === null) break 203 | q[1](null, res) 204 | if (--q[0] === 0) queue.shift() 205 | } 206 | ready = false 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /license: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 James Halliday 2 | 3 | Redistribution and use in source and binary forms, with or without modification, 4 | are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list 7 | of conditions and the following disclaimer. 8 | 9 | Redistributions in binary form must reproduce the above copyright notice, this 10 | list of conditions and the following disclaimer in the documentation and/or 11 | other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hypercore-query-extension", 3 | "version": "1.3.0", 4 | "description": "query peers about hypercore feed data", 5 | "keywords": [ 6 | "hypercore", 7 | "sparse", 8 | "query" 9 | ], 10 | "dependencies": { 11 | "end-of-stream": "^1.4.4", 12 | "protocol-buffers-encodings": "^1.1.0", 13 | "readable-stream": "^3.4.0" 14 | }, 15 | "devDependencies": { 16 | "discovery-swarm": "^6.0.0", 17 | "hypercore": "^8.3.0", 18 | "hypercore-protocol": "^7.7.1", 19 | "level": "^6.0.0", 20 | "minimist": "^1.2.0", 21 | "multifeed-replicate": "^1.0.0", 22 | "protocol-buffers": "^4.1.2", 23 | "pump": "^3.0.0", 24 | "random-access-file": "^2.1.3", 25 | "random-access-memory": "^3.1.1", 26 | "tape": "^4.11.0", 27 | "to2": "^1.0.0", 28 | "unordered-materialized-kv-live": "^1.0.1" 29 | }, 30 | "scripts": { 31 | "protobuf": "protocol-buffers schema.proto -o messages.js", 32 | "test": "tape test/*.js" 33 | }, 34 | "license": "bsd" 35 | } 36 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # hypercore-query-extension 2 | 3 | ask peers over an extension for which sequences in which feeds are relevant to 4 | custom query logic 5 | 6 | This approach is very useful for sparse data from a potentially large number of 7 | feeds. Instead of downloading everything for each feed, you can ask peers which 8 | feeds and sequences are relevant to particular queries. For example, if you have 9 | a chat application, you might ask peers about the latest 20 messages in a 10 | channel. 11 | 12 | # example 13 | 14 | This example builds a feed and a clone of that feed in memory. The feed is 15 | populated with a number between 0 and 99, inclusive every 50 milliseconds. The 16 | clone feed is opened in sparse mode and the clone tells the main feed through 17 | the query extension that it is only interested in numbers between 50 and 70, 18 | inclusive. The clone then downloads only those sequences that were mentioned in 19 | the query results. 20 | 21 | ``` js 22 | var Query = require('hypercore-query-extension') 23 | var { Readable, Transform } = require('readable-stream') 24 | var ram = require('random-access-memory') 25 | 26 | var hypercore = require('hypercore') 27 | var feed0 = hypercore(ram) 28 | 29 | setInterval(function () { 30 | var n = Math.floor(Math.random()*100) 31 | feed0.append(String(n)) 32 | }, 50) 33 | 34 | feed0.ready(function () { 35 | var feed1 = hypercore(ram, feed0.key) 36 | var r0 = feed0.replicate(false, { download: false, live: true }) 37 | var r1 = feed1.replicate(true, { sparse: true, live: true }) 38 | r0.pipe(r1).pipe(r0) 39 | var q0 = new Query({ api: api(feed0) }) 40 | var q1 = new Query({ api: api(feed1) }) 41 | r0.registerExtension('query-example', q0.extension()) 42 | r1.registerExtension('query-example', q1.extension()) 43 | var s = q1.query('subscribe', JSON.stringify({ start: 50, end: 70 })) 44 | s.pipe(new Transform({ 45 | objectMode: true, 46 | transform: function (row, enc, next) { 47 | if (!row.key.equals(feed0.key)) return next() 48 | feed1.update(row.seq, function () { 49 | feed1.get(row.seq, function (err, buf) { 50 | if (err) return next(err) 51 | console.log('n=', Number(buf.toString())) 52 | next() 53 | }) 54 | }) 55 | } 56 | })) 57 | }) 58 | 59 | function api (feed) { 60 | var subs = [] 61 | feed.on('append', function () { 62 | var seq = feed.length 63 | feed.get(seq, function (err, buf) { 64 | var n = Number(buf.toString()) 65 | subs.forEach(({ start, end, stream }) => { 66 | if (n >= start && n <= end) { 67 | stream.push({ key: feed.key, seq }) 68 | } 69 | }) 70 | }) 71 | }) 72 | return { subscribe } 73 | function subscribe (args) { 74 | var { start, end } = JSON.parse(args.toString()) 75 | var stream = new Readable({ 76 | objectMode: true, 77 | read: function () {} 78 | }) 79 | subs.push({ start, end, stream }) 80 | return stream 81 | } 82 | } 83 | ``` 84 | 85 | # api 86 | 87 | ``` js 88 | var Query = require('hypercore-query-extension') 89 | ``` 90 | 91 | ## var q = Query(opts) 92 | 93 | Create a new `Query` instance `q` from: 94 | 95 | * `opts.api` - object mapping query names to implementation functions. 96 | 97 | Each api function receives a `Buffer` of optional argument payload and must 98 | return a readable or duplex `readableObjectMode` stream that pushes object with 99 | a feed `key` (as a `Buffer`) and a sequence number. 100 | 101 | If the stream is duplex, you'll be able to receive messages on the channel from 102 | the other side of the query. You can use this to adjust query parameters on the 103 | fly, for example if you have a query for a map and the user pans the map, 104 | changing the bounding box. The updated bounding box can be sent back up the 105 | stream without having to open a new query. 106 | 107 | ## q.extension() 108 | 109 | Return a function that can be passed to `feed.registerExtension()` or 110 | `proto.registerExtension()`. You'll almost always want to do: 111 | 112 | ``` js 113 | feed.registerExtension('your-extension-name', q.extension()) 114 | // or: 115 | proto.registerExtension('your-extension-name', q.extension()) 116 | ``` 117 | 118 | ## var stream = q.query(name, data) 119 | 120 | Return a duplex objectMode `stream` with results from calling the api endpoint 121 | `name` with an optional `data` payload (as a `Buffer`). 122 | 123 | Each `row` from the readable side of the stream contains: 124 | 125 | * `row.key` - feed key (`Buffer` or hex string) 126 | * `row.seq` - feed sequence number 127 | 128 | If the query supports duplex mode, you can write messages to the remote api by 129 | calling `stream.write()` with `Buffer` payloads. 130 | 131 | # license 132 | 133 | BSD 134 | -------------------------------------------------------------------------------- /schema.proto: -------------------------------------------------------------------------------- 1 | // type=0 2 | message Open { 3 | required uint32 id = 1; 4 | required uint32 query_id = 2; 5 | optional bytes data = 3; 6 | } 7 | 8 | // type=1 9 | message Read { 10 | required uint32 id = 1; 11 | required uint32 n = 2; 12 | } 13 | 14 | // type=2 15 | message Control { 16 | required uint32 id = 1; 17 | enum ControlCode { 18 | CLOSE = 0; 19 | DESTROY = 1; 20 | END = 2; 21 | } 22 | required ControlCode code = 2; 23 | } 24 | 25 | // type=3 26 | message QueryDef { 27 | required uint32 id = 1; 28 | required string name = 2; 29 | } 30 | 31 | // type=4 32 | message Response { 33 | required uint32 query_id = 1; 34 | required FeedSeq result = 2; 35 | } 36 | 37 | // type=5 38 | message FeedDef { 39 | required uint32 id = 1; 40 | required bytes key = 2; 41 | } 42 | 43 | message FeedSeq { 44 | required uint32 id = 1; 45 | required uint32 seq = 2; 46 | } 47 | 48 | // type=6 49 | message Write { 50 | required uint32 id = 1; 51 | required bytes data = 2; 52 | } 53 | -------------------------------------------------------------------------------- /test/duplex.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var { Duplex, Transform } = require('readable-stream') 3 | var Query = require('../') 4 | var ram = require('random-access-memory') 5 | var hypercore = require('hypercore') 6 | 7 | test('full duplex', function (t) { 8 | var feed0 = hypercore(ram) 9 | var expected = [] 10 | var iv = setInterval(function () { 11 | var n = Math.floor(Math.random()*100) 12 | feed0.append(String(n)) 13 | if (expected.length < 10) { 14 | if (n >= 50 && n <= 70) expected.push(n) 15 | } else { 16 | if (n >= 10 && n <= 40) expected.push(n) 17 | } 18 | if (expected.length === 20) clearInterval(iv) 19 | }, 5) 20 | var received = [] 21 | feed0.ready(function () { 22 | var feed1 = hypercore(ram, feed0.key) 23 | var r0 = feed0.replicate(false, { download: false, live: true }) 24 | var r1 = feed1.replicate(true, { sparse: true, live: true }) 25 | r0.pipe(r1).pipe(r0) 26 | var q0 = new Query({ api: api(feed0) }) 27 | var q1 = new Query({ api: api(feed1) }) 28 | r0.registerExtension('test', q0.extension()) 29 | r1.registerExtension('test', q1.extension()) 30 | var s = q1.query('subscribe', JSON.stringify({ start: 50, end: 70 })) 31 | s.on('error', function (err) { t.error(err) }) 32 | s.pipe(new Transform({ 33 | objectMode: true, 34 | transform: function (row, enc, next) { 35 | feed1.update(row.seq, function () { 36 | feed1.get(row.seq, function (err, buf) { 37 | if (err) return next(err) 38 | var n = Number(buf.toString()) 39 | received.push(n) 40 | if (received.length === 10) { 41 | s.write('10,40') 42 | } else if (received.length === 20) { 43 | check() 44 | } 45 | next() 46 | }) 47 | }) 48 | } 49 | })) 50 | }) 51 | function check () { 52 | t.deepEqual(expected, received) 53 | t.end() 54 | } 55 | function api (feed) { 56 | var subs = [] 57 | feed.on('append', function () { 58 | var seq = feed.length 59 | feed.get(seq, function (err, buf) { 60 | t.ifError(err) 61 | var n = Number(buf.toString()) 62 | subs.forEach(({ start, end, stream }) => { 63 | if (n >= start && n <= end) { 64 | stream.push({ key: feed.key, seq }) 65 | } 66 | }) 67 | }) 68 | }) 69 | return { subscribe } 70 | function subscribe (args) { 71 | var sub = JSON.parse(args.toString()) 72 | sub.stream = new Duplex({ 73 | objectMode: true, 74 | read: function () {}, 75 | write: function (buf, enc, next) { 76 | ;[sub.start,sub.end] = buf.toString().split(',').map(Number) 77 | next() 78 | } 79 | }) 80 | subs.push(sub) 81 | return sub.stream 82 | } 83 | } 84 | }) 85 | -------------------------------------------------------------------------------- /test/pubsub.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var { Readable, Transform } = require('readable-stream') 3 | var Query = require('../') 4 | var ram = require('random-access-memory') 5 | var hypercore = require('hypercore') 6 | 7 | test('pubsub', function (t) { 8 | var feed0 = hypercore(ram) 9 | var expected = [] 10 | var iv = setInterval(function () { 11 | var n = Math.floor(Math.random()*100) 12 | feed0.append(String(n)) 13 | if (n >= 50 && n <= 70) expected.push(n) 14 | if (expected.length === 10) clearInterval(iv) 15 | }, 5) 16 | var received = [] 17 | feed0.ready(function () { 18 | var feed1 = hypercore(ram, feed0.key) 19 | var r0 = feed0.replicate(false, { download: false, live: true }) 20 | var r1 = feed1.replicate(true, { sparse: true, live: true }) 21 | r0.pipe(r1).pipe(r0) 22 | var q0 = new Query({ api: api(feed0) }) 23 | var q1 = new Query({ api: api(feed1) }) 24 | r0.registerExtension('test', q0.extension()) 25 | r1.registerExtension('test', q1.extension()) 26 | var s = q1.query('subscribe', JSON.stringify({ start: 50, end: 70 })) 27 | s.on('error', function (err) { t.error(err) }) 28 | s.pipe(new Transform({ 29 | objectMode: true, 30 | transform: function (row, enc, next) { 31 | feed1.update(row.seq, function () { 32 | feed1.get(row.seq, function (err, buf) { 33 | if (err) return next(err) 34 | var n = Number(buf.toString()) 35 | received.push(n) 36 | if (received.length === 10) check() 37 | next() 38 | }) 39 | }) 40 | } 41 | })) 42 | }) 43 | function check () { 44 | t.deepEqual(expected, received) 45 | t.end() 46 | } 47 | function api (feed) { 48 | var subs = [] 49 | feed.on('append', function () { 50 | var seq = feed.length 51 | feed.get(seq, function (err, buf) { 52 | t.ifError(err) 53 | var n = Number(buf.toString()) 54 | subs.forEach(({ start, end, stream }) => { 55 | if (n >= start && n <= end) { 56 | stream.push({ key: feed.key, seq }) 57 | } 58 | }) 59 | }) 60 | }) 61 | return { subscribe } 62 | function subscribe (args) { 63 | var { start, end } = JSON.parse(args.toString()) 64 | var stream = new Readable({ 65 | objectMode: true, 66 | read: function () {} 67 | }) 68 | subs.push({ start, end, stream }) 69 | return stream 70 | } 71 | } 72 | }) 73 | -------------------------------------------------------------------------------- /test/string-key.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var { Readable, Transform } = require('readable-stream') 3 | var Query = require('../') 4 | var ram = require('random-access-memory') 5 | var hypercore = require('hypercore') 6 | 7 | test('string key', function (t) { 8 | var feed0 = hypercore(ram) 9 | var expected = [] 10 | var iv = setInterval(function () { 11 | var n = Math.floor(Math.random()*100) 12 | feed0.append(String(n)) 13 | if (n >= 50 && n <= 70) expected.push(n) 14 | if (expected.length === 10) clearInterval(iv) 15 | }, 5) 16 | var received = [] 17 | feed0.ready(function () { 18 | var feed1 = hypercore(ram, feed0.key) 19 | var r0 = feed0.replicate(false, { download: false, live: true }) 20 | var r1 = feed1.replicate(true, { sparse: true, live: true }) 21 | r0.pipe(r1).pipe(r0) 22 | var q0 = new Query({ api: api(feed0) }) 23 | var q1 = new Query({ api: api(feed1) }) 24 | r0.registerExtension('test', q0.extension()) 25 | r1.registerExtension('test', q1.extension()) 26 | var s = q1.query('subscribe', JSON.stringify({ start: 50, end: 70 })) 27 | s.on('error', function (err) { t.error(err) }) 28 | s.pipe(new Transform({ 29 | objectMode: true, 30 | transform: function (row, enc, next) { 31 | feed1.update(row.seq, function () { 32 | feed1.get(row.seq, function (err, buf) { 33 | if (err) return next(err) 34 | var n = Number(buf.toString()) 35 | received.push(n) 36 | if (received.length === 10) check() 37 | next() 38 | }) 39 | }) 40 | } 41 | })) 42 | }) 43 | function check () { 44 | t.deepEqual(expected, received) 45 | t.end() 46 | } 47 | function api (feed) { 48 | var subs = [] 49 | feed.on('append', function () { 50 | var seq = feed.length 51 | feed.get(seq, function (err, buf) { 52 | t.ifError(err) 53 | var n = Number(buf.toString()) 54 | subs.forEach(({ start, end, stream }) => { 55 | if (n >= start && n <= end) { 56 | stream.push({ key: feed.key.toString('hex'), seq }) 57 | } 58 | }) 59 | }) 60 | }) 61 | return { subscribe } 62 | function subscribe (args) { 63 | var { start, end } = JSON.parse(args.toString()) 64 | var stream = new Readable({ 65 | objectMode: true, 66 | read: function () {} 67 | }) 68 | subs.push({ start, end, stream }) 69 | return stream 70 | } 71 | } 72 | }) 73 | --------------------------------------------------------------------------------