├── .gitignore ├── LICENSE ├── README.md ├── lib └── lazystream.js ├── package.json └── test ├── data.md ├── fs_test.js ├── helper.js ├── pipe_test.js ├── readable_test.js └── writable_test.js /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | npm-debug.log 3 | node_modules/ 4 | test/tmp/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 J. Pommerening, contributors. 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lazy Streams 2 | 3 | > *Create streams lazily when they are read from or written to.* 4 | > `lazystream: 1.0.1` 5 | 6 | ## Why? 7 | 8 | Sometimes you feel the itch to open *all the files* at once. You want to pass a bunch of streams around, so the consumer does not need to worry where the data comes from. 9 | From a software design point-of-view this sounds entirely reasonable. Then there is that neat little function `fs.createReadStream()` that opens a file and gives you a nice `fs.ReadStream` to pass around, so you use what the mighty creator deities of node bestowed upon you. 10 | 11 | > `Error: EMFILE, too many open files` 12 | > ─ *node* 13 | 14 | This package provides two classes based on the node's Streams3 API (courtesy of `readable-stream` to ensure a stable version). 15 | 16 | ## Class: lazystream.Readable 17 | 18 | A wrapper for readable streams. Extends [`stream.PassThrough`](http://nodejs.org/api/stream.html#stream_class_stream_passthrough). 19 | 20 | ### new lazystream.Readable(fn [, options]) 21 | 22 | * `fn` *{Function}* 23 | The function that the lazy stream will call to obtain the stream to actually read from. 24 | * `options` *{Object}* 25 | Options for the underlying `PassThrough` stream, accessible by `fn`. 26 | 27 | Creates a new readable stream. Once the stream is accessed (for example when you call its `read()` method, or attach a `data`-event listener) the `fn` function is called with the outer `lazystream.Readable` instance bound to `this`. 28 | 29 | If you pass an `options` object to the constuctor, you can access it in your `fn` function. 30 | 31 | ```javascript 32 | new lazystream.Readable(function (options) { 33 | return fs.createReadStream('/dev/urandom'); 34 | }); 35 | ``` 36 | 37 | ## Class: lazystream.Writable 38 | 39 | A wrapper for writable streams. Extends [`stream.PassThrough`](http://nodejs.org/api/stream.html#stream_class_stream_passthrough). 40 | 41 | ### new lazystream.Writable(fn [, options]) 42 | 43 | * `fn` *{Function}* 44 | The function that the lazy stream will call to obtain the stream to actually write to. 45 | * `options` *{Object}* 46 | Options for the underlying `PassThrough` stream, accessible by `fn`. 47 | 48 | Creates a new writable stream. Just like the one above but for writable streams. 49 | 50 | ```javascript 51 | new lazystream.Writable(function () { 52 | return fs.createWriteStream('/dev/null'); 53 | }); 54 | ``` 55 | 56 | ## Install 57 | 58 | ```console 59 | $ npm install lazystream --save 60 | lazystream@1.0.1 node_modules/lazystream 61 | └── readable-stream@2.0.5 62 | ``` 63 | 64 | ## Changelog 65 | 66 | ### v1.0.1 67 | 68 | - [#3](https://github.com/jpommerening/node-lazystream/issues/3): (finally) fixed a long-standing publishing error 69 | 70 | ### v1.0.0 71 | 72 | - [#2](https://github.com/jpommerening/node-lazystream/issues/2): [unconditionally](https://r.va.gg/2014/06/why-i-dont-use-nodes-core-stream-module.html) use `readable-stream` _2.x_. 73 | 74 | ### v0.2.0 75 | 76 | - [#1](https://github.com/jpommerening/node-lazystream/pull/1): error events are now propagated 77 | 78 | ### v0.1.0 79 | 80 | - _(this was the first release)_ 81 | 82 | ## Contributing 83 | 84 | Fork it, branch it, send me a pull request. We'll work out the rest together. 85 | 86 | ## Credits 87 | 88 | [Chris Talkington](https://github.com/ctalkington) and his [node-archiver](https://github.com/ctalkington/node-archiver) for providing a use-case. 89 | 90 | ## [License](LICENSE) 91 | 92 | Copyright (c) 2013 J. Pommerening, contributors. 93 | 94 | Permission is hereby granted, free of charge, to any person 95 | obtaining a copy of this software and associated documentation 96 | files (the "Software"), to deal in the Software without 97 | restriction, including without limitation the rights to use, 98 | copy, modify, merge, publish, distribute, sublicense, and/or sell 99 | copies of the Software, and to permit persons to whom the 100 | Software is furnished to do so, subject to the following 101 | conditions: 102 | 103 | The above copyright notice and this permission notice shall be 104 | included in all copies or substantial portions of the Software. 105 | 106 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 107 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 108 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 109 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 110 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 111 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 112 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 113 | OTHER DEALINGS IN THE SOFTWARE. 114 | 115 | -------------------------------------------------------------------------------- /lib/lazystream.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var PassThrough = require('readable-stream/passthrough'); 3 | 4 | module.exports = { 5 | Readable: Readable, 6 | Writable: Writable 7 | }; 8 | 9 | util.inherits(Readable, PassThrough); 10 | util.inherits(Writable, PassThrough); 11 | 12 | // Patch the given method of instance so that the callback 13 | // is executed once, before the actual method is called the 14 | // first time. 15 | function beforeFirstCall(instance, method, callback) { 16 | instance[method] = function() { 17 | delete instance[method]; 18 | callback.apply(this, arguments); 19 | return this[method].apply(this, arguments); 20 | }; 21 | } 22 | 23 | function Readable(fn, options) { 24 | if (!(this instanceof Readable)) 25 | return new Readable(fn, options); 26 | 27 | PassThrough.call(this, options); 28 | 29 | beforeFirstCall(this, '_read', function() { 30 | var source = fn.call(this, options); 31 | var emit = this.emit.bind(this, 'error'); 32 | source.on('error', emit); 33 | source.pipe(this); 34 | }); 35 | 36 | this.emit('readable'); 37 | } 38 | 39 | function Writable(fn, options) { 40 | if (!(this instanceof Writable)) 41 | return new Writable(fn, options); 42 | 43 | PassThrough.call(this, options); 44 | 45 | beforeFirstCall(this, '_write', function() { 46 | var destination = fn.call(this, options); 47 | var emit = this.emit.bind(this, 'error'); 48 | destination.on('error', emit); 49 | this.pipe(destination); 50 | }); 51 | 52 | this.emit('writable'); 53 | } 54 | 55 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lazystream", 3 | "version": "1.0.1", 4 | "description": "Open Node Streams on demand.", 5 | "homepage": "https://github.com/jpommerening/node-lazystream", 6 | "author": { 7 | "name": "Jonas Pommerening", 8 | "email": "jonas.pommerening@gmail.com", 9 | "url": "https://npmjs.org/~jpommerening" 10 | }, 11 | "contributors": [ 12 | "Mario Casciaro " 13 | ], 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/jpommerening/node-lazystream.git" 17 | }, 18 | "bugs": { 19 | "url": "https://github.com/jpommerening/node-lazystream/issues" 20 | }, 21 | "license": "MIT", 22 | "main": "lib/lazystream.js", 23 | "engines": { 24 | "node": ">= 0.6.3" 25 | }, 26 | "scripts": { 27 | "test": "nodeunit test/readable_test.js test/writable_test.js test/pipe_test.js test/fs_test.js" 28 | }, 29 | "files": [ 30 | "lib/lazystream.js", 31 | "test/*.js", 32 | "test/*.md" 33 | ], 34 | "dependencies": { 35 | "readable-stream": "^2.0.5" 36 | }, 37 | "devDependencies": { 38 | "nodeunit": "^0.9.1" 39 | }, 40 | "keywords": [ 41 | "emfile", 42 | "lazy", 43 | "streams", 44 | "stream" 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /test/data.md: -------------------------------------------------------------------------------- 1 | > Never mind, hey, this is really exciting, so much to find out about, so much to 2 | > look forward to, I'm quite dizzy with anticipation . . . Or is it the wind? 3 | > 4 | > There really is a lot of that now, isn't there? And wow! Hey! What's this thing 5 | > suddenly coming toward me very fast? Very, very fast. So big and flat and round, 6 | > it needs a big wide-sounding name like . . . ow . . . ound . . . round . . . 7 | > ground! That's it! That's a good name- ground! 8 | > 9 | > I wonder if it will be friends with me? 10 | > 11 | > Hello Ground! 12 | 13 | And the rest, after a sudden wet thud, was silence. 14 | -------------------------------------------------------------------------------- /test/fs_test.js: -------------------------------------------------------------------------------- 1 | 2 | var stream = require('../lib/lazystream'); 3 | var fs = require('fs'); 4 | var tmpDir = 'test/tmp/'; 5 | var readFile = 'test/data.md'; 6 | var writeFile = tmpDir + 'data.md'; 7 | 8 | exports.fs = { 9 | readwrite: function(test) { 10 | var readfd, writefd; 11 | 12 | var readable = new stream.Readable(function() { 13 | return fs.createReadStream(readFile) 14 | .on('open', function(fd) { 15 | readfd = fd; 16 | }) 17 | .on('close', function() { 18 | readfd = undefined; 19 | step(); 20 | }); 21 | }); 22 | 23 | var writable = new stream.Writable(function() { 24 | return fs.createWriteStream(writeFile) 25 | .on('open', function(fd) { 26 | writefd = fd; 27 | }) 28 | .on('close', function() { 29 | writefd = undefined; 30 | step(); 31 | }); 32 | }); 33 | 34 | test.expect(3); 35 | 36 | test.equal(readfd, undefined, 'Input file should not be opened until read'); 37 | test.equal(writefd, undefined, 'Output file should not be opened until write'); 38 | 39 | if (!fs.existsSync(tmpDir)) { 40 | fs.mkdirSync(tmpDir); 41 | } 42 | if (fs.existsSync(writeFile)) { 43 | fs.unlinkSync(writeFile); 44 | } 45 | 46 | readable.on('end', function() { step(); }); 47 | writable.on('end', function() { step(); }); 48 | 49 | var steps = 0; 50 | function step() { 51 | steps += 1; 52 | if (steps == 4) { 53 | var input = fs.readFileSync(readFile); 54 | var output = fs.readFileSync(writeFile); 55 | 56 | test.ok(input >= output && input <= output, 'Should be equal'); 57 | 58 | fs.unlinkSync(writeFile); 59 | fs.rmdirSync(tmpDir); 60 | 61 | test.done(); 62 | } 63 | }; 64 | 65 | readable.pipe(writable); 66 | } 67 | }; 68 | 69 | 70 | -------------------------------------------------------------------------------- /test/helper.js: -------------------------------------------------------------------------------- 1 | 2 | var _Readable = require('readable-stream/readable'); 3 | var _Writable = require('readable-stream/writable'); 4 | var util = require('util'); 5 | 6 | module.exports = { 7 | DummyReadable: DummyReadable, 8 | DummyWritable: DummyWritable 9 | }; 10 | 11 | function DummyReadable(strings) { 12 | _Readable.call(this); 13 | this.strings = strings; 14 | this.emit('readable'); 15 | } 16 | 17 | util.inherits(DummyReadable, _Readable); 18 | 19 | DummyReadable.prototype._read = function _read(n) { 20 | if (this.strings.length) { 21 | this.push(new Buffer(this.strings.shift())); 22 | } else { 23 | this.push(null); 24 | } 25 | }; 26 | 27 | function DummyWritable(strings) { 28 | _Writable.call(this); 29 | this.strings = strings; 30 | this.emit('writable'); 31 | } 32 | 33 | util.inherits(DummyWritable, _Writable); 34 | 35 | DummyWritable.prototype._write = function _write(chunk, encoding, callback) { 36 | this.strings.push(chunk.toString()); 37 | if (callback) callback(); 38 | }; 39 | 40 | -------------------------------------------------------------------------------- /test/pipe_test.js: -------------------------------------------------------------------------------- 1 | 2 | var stream = require('../lib/lazystream'); 3 | var helper = require('./helper'); 4 | 5 | exports.pipe = { 6 | readwrite: function(test) { 7 | var expected = [ 'line1\n', 'line2\n' ]; 8 | var actual = []; 9 | var readableInstantiated = false; 10 | var writableInstantiated = false; 11 | 12 | test.expect(3); 13 | 14 | var readable = new stream.Readable(function() { 15 | readableInstantiated = true; 16 | return new helper.DummyReadable([].concat(expected)); 17 | }); 18 | 19 | var writable = new stream.Writable(function() { 20 | writableInstantiated = true; 21 | return new helper.DummyWritable(actual); 22 | }); 23 | 24 | test.equal(readableInstantiated, false, 'DummyReadable should only be instantiated when it is needed'); 25 | test.equal(writableInstantiated, false, 'DummyWritable should only be instantiated when it is needed'); 26 | 27 | writable.on('end', function() { 28 | test.equal(actual.join(''), expected.join(''), 'Piping on demand streams should keep data intact'); 29 | test.done(); 30 | }); 31 | 32 | readable.pipe(writable); 33 | } 34 | }; 35 | 36 | 37 | -------------------------------------------------------------------------------- /test/readable_test.js: -------------------------------------------------------------------------------- 1 | 2 | var Readable = require('../lib/lazystream').Readable; 3 | var DummyReadable = require('./helper').DummyReadable; 4 | 5 | exports.readable = { 6 | dummy: function(test) { 7 | var expected = [ 'line1\n', 'line2\n' ]; 8 | var actual = []; 9 | 10 | test.expect(1); 11 | 12 | new DummyReadable([].concat(expected)) 13 | .on('data', function(chunk) { 14 | actual.push(chunk.toString()); 15 | }) 16 | .on('end', function() { 17 | test.equal(actual.join(''), expected.join(''), 'DummyReadable should produce the data it was created with'); 18 | test.done(); 19 | }); 20 | }, 21 | options: function(test) { 22 | test.expect(3); 23 | 24 | var readable = new Readable(function(options) { 25 | test.ok(this instanceof Readable, "Readable should bind itself to callback's this"); 26 | test.equal(options.encoding, "utf-8", "Readable should make options accessible to callback"); 27 | this.ok = true; 28 | return new DummyReadable(["test"]); 29 | }, {encoding: "utf-8"}); 30 | 31 | readable.read(4); 32 | 33 | test.ok(readable.ok); 34 | 35 | test.done(); 36 | }, 37 | streams2: function(test) { 38 | var expected = [ 'line1\n', 'line2\n' ]; 39 | var actual = []; 40 | var instantiated = false; 41 | 42 | test.expect(2); 43 | 44 | var readable = new Readable(function() { 45 | instantiated = true; 46 | return new DummyReadable([].concat(expected)); 47 | }); 48 | 49 | test.equal(instantiated, false, 'DummyReadable should only be instantiated when it is needed'); 50 | 51 | readable.on('readable', function() { 52 | var chunk; 53 | while ((chunk = readable.read())) { 54 | actual.push(chunk.toString()); 55 | } 56 | }); 57 | readable.on('end', function() { 58 | test.equal(actual.join(''), expected.join(''), 'Readable should not change the data of the underlying stream'); 59 | test.done(); 60 | }); 61 | 62 | readable.read(0); 63 | }, 64 | resume: function(test) { 65 | var expected = [ 'line1\n', 'line2\n' ]; 66 | var actual = []; 67 | var instantiated = false; 68 | 69 | test.expect(2); 70 | 71 | var readable = new Readable(function() { 72 | instantiated = true; 73 | return new DummyReadable([].concat(expected)); 74 | }); 75 | 76 | readable.pause(); 77 | 78 | readable.on('data', function(chunk) { 79 | actual.push(chunk.toString()); 80 | }); 81 | readable.on('end', function() { 82 | test.equal(actual.join(''), expected.join(''), 'Readable should not change the data of the underlying stream'); 83 | test.done(); 84 | }); 85 | 86 | test.equal(instantiated, false, 'DummyReadable should only be instantiated when it is needed'); 87 | 88 | readable.resume(); 89 | } 90 | }; 91 | -------------------------------------------------------------------------------- /test/writable_test.js: -------------------------------------------------------------------------------- 1 | 2 | var Writable = require('../lib/lazystream').Writable; 3 | var DummyWritable = require('./helper').DummyWritable; 4 | 5 | exports.writable = { 6 | options: function(test) { 7 | test.expect(3); 8 | 9 | var writable = new Writable(function(options) { 10 | test.ok(this instanceof Writable, "Writable should bind itself to callback's this"); 11 | test.equal(options.encoding, "utf-8", "Writable should make options accessible to callback"); 12 | this.ok = true; 13 | return new DummyWritable([]); 14 | }, {encoding: "utf-8"}); 15 | 16 | writable.write("test"); 17 | 18 | test.ok(writable.ok); 19 | 20 | test.done(); 21 | }, 22 | dummy: function(test) { 23 | var expected = [ 'line1\n', 'line2\n' ]; 24 | var actual = []; 25 | 26 | test.expect(0); 27 | 28 | var dummy = new DummyWritable(actual); 29 | 30 | expected.forEach(function(item) { 31 | dummy.write(new Buffer(item)); 32 | }); 33 | test.done(); 34 | }, 35 | streams2: function(test) { 36 | var expected = [ 'line1\n', 'line2\n' ]; 37 | var actual = []; 38 | var instantiated = false; 39 | 40 | test.expect(2); 41 | 42 | var writable = new Writable(function() { 43 | instantiated = true; 44 | return new DummyWritable(actual); 45 | }); 46 | 47 | test.equal(instantiated, false, 'DummyWritable should only be instantiated when it is needed'); 48 | 49 | writable.on('end', function() { 50 | test.equal(actual.join(''), expected.join(''), 'Writable should not change the data of the underlying stream'); 51 | test.done(); 52 | }); 53 | 54 | expected.forEach(function(item) { 55 | writable.write(new Buffer(item)); 56 | }); 57 | writable.end(); 58 | } 59 | }; 60 | --------------------------------------------------------------------------------