├── .gitignore ├── .jshintrc ├── eventProxy.js ├── test ├── batcher.js └── test.js ├── index.js ├── package.json ├── writable.js ├── readme.md └── readable.js /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | coverage 3 | node_modules 4 | .DS_Store -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node" : true, 3 | "curly": true, 4 | "latedef": "nofunc", 5 | "eqeqeq": true, 6 | "immed": true, 7 | "newcap": true, 8 | "noarg": true, 9 | "sub": true, 10 | "undef": "nofunc", 11 | "strict": true, 12 | "white": true, 13 | "indent": 2, 14 | "trailing": true, 15 | "quotmark": "single" 16 | } -------------------------------------------------------------------------------- /eventProxy.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | module.exports = function (source, target, events) { 3 | source.on('newListener', function (event, listener) { 4 | if (~events.indexOf(event)) { 5 | target.on(event, listener); 6 | } 7 | }); 8 | source.on('removeListener', function (event, listener) { 9 | if (~events.indexOf(event)) { 10 | target.removeListener(event, listener); 11 | } 12 | }); 13 | }; 14 | -------------------------------------------------------------------------------- /test/batcher.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var inherits = require('inherits'); 3 | var Transform = require('readable-stream').Transform; 4 | 5 | module.exports = Batcher; 6 | inherits(Batcher, Transform); 7 | 8 | function Batcher(size) { 9 | if (!(this instanceof Batcher)) { 10 | return new Batcher(size); 11 | } 12 | Transform.call(this, { 13 | objectMode: true 14 | }); 15 | this.size = size; 16 | this.queue = []; 17 | } 18 | Batcher.prototype._transform = function (chunk, _, next) { 19 | this.queue.push(chunk); 20 | if (chunk.length >= this.size) { 21 | this.push(this.queue); 22 | this.queue = []; 23 | next(); 24 | } else { 25 | next(); 26 | } 27 | }; 28 | 29 | Batcher.prototype._flush = function (next) { 30 | if (this.queue.length) { 31 | this.push(this.queue); 32 | } 33 | next(); 34 | }; 35 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var Writable = require('./writable'); 3 | var Readable = require('./readable'); 4 | 5 | exports.createReadStream = function (opts) { 6 | opts = opts || {}; 7 | return new Readable(this, opts); 8 | }; 9 | exports.createWriteStream = function (opts) { 10 | opts = opts || {}; 11 | return new Writable(this, opts); 12 | }; 13 | var ep = require('./eventProxy'); 14 | var streamEvents = [ 15 | 'drain', 16 | 'pipe', 17 | 'unpipe', 18 | 'error' 19 | ]; 20 | exports.write = function (chunk, encoding, callback) { 21 | if (!this.__stream) { 22 | this.__stream = new Writable(this); 23 | ep(this, this.__stream, streamEvents); 24 | this.on('destroy', function () { 25 | this.__stream.end(); 26 | }); 27 | } 28 | return this.__stream.write(chunk, encoding, callback); 29 | }; 30 | exports.end = function () { 31 | this.emit('done'); 32 | }; 33 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pouch-stream", 3 | "version": "0.4.1", 4 | "description": "Experimental streaming version of PouchDB", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git@github.com:calvinmetcalf/PouchStream.git" 9 | }, 10 | "keywords": [ 11 | "pouch", 12 | "stream", 13 | "couch", 14 | "db" 15 | ], 16 | "author": "Calvin Metcalf", 17 | "license": "MIT", 18 | "bugs": { 19 | "url": "https://github.com/calvinmetcalf/PouchStream/issues" 20 | }, 21 | "homepage": "https://github.com/calvinmetcalf/PouchStream", 22 | "dependencies": { 23 | "inherits": "^2.0.1", 24 | "readable-stream": "^1.0.27-1" 25 | }, 26 | "devDependencies": { 27 | "pouchdb": "^3.3.1", 28 | "random-document-stream": "0.0.0", 29 | "tap-spec": "^2.2.2", 30 | "tape": "^2.12.3", 31 | "through2": "^0.4.1" 32 | }, 33 | "scripts": { 34 | "test": "node test/test.js | tspec" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /writable.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var inherits = require('inherits'); 3 | var Writable = require('readable-stream').Writable; 4 | 5 | module.exports = WriteStream; 6 | inherits(WriteStream, Writable); 7 | 8 | function WriteStream(db, opts) { 9 | if (!(this instanceof WriteStream)) { 10 | return new WriteStream(db); 11 | } 12 | Writable.call(this, { 13 | objectMode: true 14 | }); 15 | this.db = db; 16 | this.opts = opts || {}; 17 | } 18 | 19 | WriteStream.prototype._write = function (chunk, _, next) { 20 | if (Buffer.isBuffer(chunk)) { 21 | chunk = chunk.toString(); 22 | } 23 | if (typeof chunk === 'string') { 24 | try { 25 | chunk = JSON.parse(chunk); 26 | } catch (e) { 27 | return next(e); 28 | } 29 | } 30 | if (Array.isArray(chunk)) { 31 | this.db.bulkDocs({docs: chunk}, this.opts).then(function () { 32 | next(); 33 | }, next); 34 | } else if ('_id' in chunk) { 35 | this.db.put(chunk, this.opts).then(function () { 36 | next(); 37 | }, next); 38 | } else { 39 | this.db.post(chunk, this.opts).then(function () { 40 | next(); 41 | }, next); 42 | } 43 | }; 44 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Pouch Stream 2 | ==== 3 | 4 | Streaming plugin for PouchDB 5 | 6 | ```bash 7 | npm install pouch-stream 8 | ``` 9 | 10 | ```js 11 | PouchDB.plugin(require('pouch-stream')); 12 | ``` 13 | 14 | Writable 15 | --- 16 | 17 | note: the docs you give it can have _ids's or not and it will do post or put depending, you can also pass an array for bulk docs, it also takes an option object which will be passed verbatem to bulkDocs, put, or post. 18 | 19 | ```js 20 | var stream = db.createWriteStream(); 21 | stream.write({ 22 | foo: 'bar', 23 | _id: 'testDoc' 24 | }, function () { 25 | // chunk is flushed 26 | }); 27 | ``` 28 | 29 | but wait there is more, the database itself is a write streem though you can't close it, you can do 30 | 31 | 32 | ```js 33 | var random = require("random-document-stream"); 34 | random(100).pipe(db); 35 | ``` 36 | 37 | Readable 38 | --- 39 | 40 | ```js 41 | var db = new PouchDB('foo'); 42 | var stream = db.createReadStream(); 43 | stream.on('data', function (d) { 44 | // deal with data 45 | }); 46 | ``` 47 | 48 | you can also set `since` 49 | 50 | var stream = db.createReadStream({since:19}); 51 | stream.on('data', function (d) { 52 | // deal with data after seq 19 53 | }); 54 | -------------------------------------------------------------------------------- /readable.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var inherits = require('inherits'); 3 | var Readable = require('readable-stream').Readable; 4 | 5 | module.exports = ReadStream; 6 | inherits(ReadStream, Readable); 7 | 8 | function ReadStream(db, opts) { 9 | if (!(this instanceof ReadStream)) { 10 | return new ReadStream(db, opts); 11 | } 12 | Readable.call(this, { 13 | objectMode: true 14 | }); 15 | opts = opts || {}; 16 | var thisOpts = this.opts = { 17 | since: 0 18 | }; 19 | Object.keys(opts).forEach(function (key) { 20 | thisOpts[key] = opts[key]; 21 | }); 22 | this.opts.returnDocs = false; 23 | this.last = opts.since; 24 | this.db = db; 25 | this.changes = void 0; 26 | this.canceled = false; 27 | } 28 | ReadStream.prototype._read = function () { 29 | if (this.changes) { 30 | return; 31 | } 32 | var self = this; 33 | this.opts.since = this.last; 34 | this.changes = this.db.changes(this.opts).on('complete', function (resp) { 35 | self.cancel = function () {}; 36 | if (!resp.canceled) { 37 | self.push(null); 38 | } 39 | }); 40 | this.changes.on('change', function (change) { 41 | self.last = change.seq; 42 | var more = self.push(change); 43 | if (!more) { 44 | self.changes.cancel(); 45 | } 46 | }); 47 | }; 48 | ReadStream.prototype.cancel = function () { 49 | this.canceled = true; 50 | if (this.changes && typeof this.changes.cancel === 'function') { 51 | return this.changes.cancel(); 52 | } 53 | }; 54 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var PouchDB = require('pouchdb'); 4 | var random = require('random-document-stream'); 5 | var PouchStream = require('../'); 6 | PouchDB.plugin(PouchStream); 7 | var test = require('tape'); 8 | var batcher = require('./batcher'); 9 | var through = require('through2').obj; 10 | 11 | test('basic', function (t) { 12 | var db = new PouchDB('testDB1'); 13 | t.test('put stuff in', function (t) { 14 | random(100).pipe(db.createWriteStream()).on('finish', function () { 15 | db.allDocs().then(function (resp) { 16 | t.equals(resp.rows.length, 100, 'all put in'); 17 | t.end(); 18 | }); 19 | }); 20 | }); 21 | t.test('take stuff out', function (t) { 22 | var called = 0; 23 | db.createReadStream().pipe(through(function (d, _, next) { 24 | called++; 25 | next(); 26 | }, function (next) { 27 | t.equal(called, 100); 28 | t.end(); 29 | next(); 30 | })); 31 | }); 32 | t.test('delete db', function (t) { 33 | db.destroy(function () { 34 | t.end(); 35 | }); 36 | }); 37 | }); 38 | test('batched', function (t) { 39 | var db = new PouchDB('testDB2'); 40 | t.test('put stuff in', function (t) { 41 | random(100).pipe(batcher(14)).pipe(db.createWriteStream()).on('finish', function () { 42 | t.end(); 43 | }); 44 | }); 45 | t.test('take stuff out', function (t) { 46 | var called = 0; 47 | db.createReadStream().on('data', function () { 48 | called++; 49 | }).on('end', function () { 50 | t.equal(called, 100); 51 | t.end(); 52 | }); 53 | }); 54 | t.test('delete db', function (t) { 55 | db.destroy(function () { 56 | t.end(); 57 | }); 58 | }); 59 | }); 60 | test('basic 2', function (t) { 61 | var db = new PouchDB('testDB3'); 62 | t.test('put stuff in', function (t) { 63 | var changes = db.changes({ 64 | live: true 65 | }); 66 | var called = 100; 67 | changes.on('change', function () { 68 | if (--called) { 69 | return; 70 | } 71 | t.ok(true, 'called 100 times'); 72 | t.end(); 73 | }); 74 | random(100).pipe(db); 75 | }); 76 | t.test('take stuff out', function (t) { 77 | var called = 0; 78 | db.createReadStream().pipe(through(function (d, _, next) { 79 | called++; 80 | next(); 81 | }, function (next) { 82 | t.equal(called, 100); 83 | t.end(); 84 | next(); 85 | })); 86 | }); 87 | t.test('delete db', function (t) { 88 | db.destroy(function () { 89 | t.end(); 90 | }); 91 | }); 92 | }); 93 | test('basic with strings', function (t) { 94 | var db = new PouchDB('testDB4'); 95 | t.test('put stuff in', function (t) { 96 | var changes = db.changes({ 97 | live: true 98 | }); 99 | var called = 100; 100 | changes.on('change', function () { 101 | if (--called) { 102 | return; 103 | } 104 | t.ok(true, 'called 100 times'); 105 | t.end(); 106 | }); 107 | random(100).pipe(through(function (chunk, _, next){ 108 | this.push(JSON.stringify(chunk)); 109 | next(); 110 | })).pipe(db); 111 | }); 112 | t.test('take stuff out', function (t) { 113 | var called = 0; 114 | db.createReadStream().pipe(through(function (d, _, next) { 115 | called++; 116 | next(); 117 | }, function (next) { 118 | t.equal(called, 100); 119 | t.end(); 120 | next(); 121 | })); 122 | }); 123 | t.test('delete db', function (t) { 124 | db.destroy(function () { 125 | t.end(); 126 | }); 127 | }); 128 | }); 129 | --------------------------------------------------------------------------------