├── index.js ├── .travis.yml ├── .gitignore ├── test ├── test-empty.js ├── test-onestream.js ├── test-oldstream.js ├── test-pipe.js ├── test-multistream.js ├── test-objectMode.js └── test-separator.js ├── package.json ├── lib └── StreamStream.js └── README.md /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/StreamStream'); 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: node_js 3 | node_js: 4 | - '0.10' 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .*.sw[a-z] 2 | .*.un~ 3 | .tern-port 4 | coverage 5 | 6 | node_modules 7 | 8 | lib-cov 9 | *.seed 10 | *.log 11 | *.csv 12 | *.dat 13 | *.out 14 | *.pid 15 | *.gz 16 | 17 | pids 18 | logs 19 | results 20 | 21 | npm-debug.log 22 | -------------------------------------------------------------------------------- /test/test-empty.js: -------------------------------------------------------------------------------- 1 | var SS = require('../'); 2 | var sink = require('stream-sink'); 3 | 4 | 5 | exports.testEmpty = function(test) { 6 | var ss = SS(); 7 | var finished = false; 8 | ss.pipe(sink()).on('data', function(data) { 9 | test.equal(data, '', "There should be no data"); 10 | finished = true; 11 | clearTimeout(to); 12 | test.done(); 13 | }); 14 | 15 | var to = setTimeout(function() { 16 | if(!finished) { 17 | test.fail('No end detected'); 18 | test.done(); 19 | } 20 | }, 10); 21 | 22 | ss.end(); 23 | } 24 | -------------------------------------------------------------------------------- /test/test-onestream.js: -------------------------------------------------------------------------------- 1 | var stream = require('stream'); 2 | var SS = require('../'); 3 | var sink = require('stream-sink'); 4 | 5 | exports.testOneStream = function(test) { 6 | var data = new stream.PassThrough(); 7 | var ss = SS() 8 | var done = false; 9 | ss.pipe(sink()).on('data', function(data) { 10 | done = true; 11 | clearTimeout(to); 12 | test.equal('hello', data, "Data in sink should be identical"); 13 | test.done(); 14 | }); 15 | 16 | ss.end(data); 17 | data.end('hello'); 18 | 19 | var to = setTimeout(function(){ 20 | if(!done) { 21 | test.fail('no end detected'); 22 | test.done(); 23 | } 24 | }, 20) 25 | } 26 | -------------------------------------------------------------------------------- /test/test-oldstream.js: -------------------------------------------------------------------------------- 1 | var stream = require('stream'); 2 | var SS = require('../'); 3 | var sink = require('stream-sink'); 4 | var EventEmitter = require('events').EventEmitter; 5 | 6 | exports.testOldStream = function(test) { 7 | var data = new EventEmitter(); 8 | var ss = SS() 9 | var done = false; 10 | ss.pipe(sink()).on('data', function(data) { 11 | done = true; 12 | clearTimeout(to); 13 | test.equal('hello', data, "Data in sink should be identical"); 14 | test.done(); 15 | }); 16 | 17 | ss.end(data); 18 | 19 | data.emit('data', 'hello'); 20 | data.emit('end'); 21 | 22 | var to = setTimeout(function(){ 23 | if(!done) { 24 | test.fail('no end detected'); 25 | test.done(); 26 | } 27 | }, 20) 28 | } 29 | 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stream-stream", 3 | "version": "1.2.6", 4 | "description": "A stream of streams in order to concatenates the contents of several streams", 5 | "main": "index.js", 6 | "engines": { 7 | "node": ">=0.10" 8 | }, 9 | "scripts": { 10 | "test": "istanbul cover nodeunit test/test-*.js && istanbul check --branches 90 --statements 95 --lines 100 --functions 100" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git://github.com/Floby/node-stream-stream.git" 15 | }, 16 | "keywords": [ 17 | "stream", 18 | "streams", 19 | "concatenates", 20 | "concatenation", 21 | "utility" 22 | ], 23 | "devDependencies": { 24 | "nodeunit": "0.7.x", 25 | "stream-sink": "~1.1", 26 | "istanbul": "~0.1.42" 27 | }, 28 | "author": "Florent Jaby ", 29 | "license": "MIT", 30 | "readmeFilename": "README.md", 31 | "gitHead": "e3142a490c3ec5943c20a2a55c8e4cb705ce30ef" 32 | } 33 | -------------------------------------------------------------------------------- /test/test-pipe.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var stream = require('stream'); 3 | var SS = require('../'); 4 | var sink = require('stream-sink'); 5 | var domain = require('domain'); 6 | 7 | Function.prototype.withDomain = function(withStack) { 8 | var fn = this; 9 | return function(test) { 10 | var d = domain.create(); 11 | d.on('error', function(e) { 12 | test.fail('test failed with ' + e.message); 13 | if(withStack) { 14 | console.error(e.stack) 15 | } 16 | test.done(); 17 | }); 18 | d.run(fn.bind(this, test)); 19 | } 20 | } 21 | 22 | 23 | function EmitStream (array) { 24 | stream.Readable.call(this, {objectMode:true}); 25 | this._source = array.slice(); 26 | } 27 | util.inherits(EmitStream, stream.Readable); 28 | EmitStream.prototype._read = function(size) { 29 | var chunk = this._source.shift(); 30 | if(!chunk) return this.push(null); 31 | 32 | var s = new stream.PassThrough(); 33 | process.nextTick(function() { 34 | s.end(chunk); 35 | }); 36 | this.push(s) 37 | } 38 | 39 | exports.testPipingStreams = function(test) { 40 | var emitter = new EmitStream(['hello', ' world', '!']); 41 | var ss = SS(); 42 | var s = sink(); 43 | var done = false; 44 | var to = setTimeout(function() { 45 | if(!done) { 46 | test.fail('No end detected') 47 | test.done(); 48 | } 49 | }, 500); 50 | 51 | emitter.pipe(ss).pipe(s).on('data', function(data) { 52 | test.equal(data, "hello world!", "Data in sink should be identical"); 53 | done = true; 54 | test.done(); 55 | clearTimeout(to); 56 | }); 57 | 58 | }.withDomain(true) 59 | -------------------------------------------------------------------------------- /test/test-multistream.js: -------------------------------------------------------------------------------- 1 | var stream = require('stream'); 2 | var SS = require('../'); 3 | var sink = require('stream-sink'); 4 | var domain = require('domain'); 5 | 6 | Function.prototype.withDomain = function(withStack) { 7 | var fn = this; 8 | return function(test) { 9 | var d = domain.create(); 10 | d.on('error', function(e) { 11 | test.fail('test failed with ' + e.message); 12 | if(withStack) { 13 | console.error(e.stack) 14 | } 15 | test.done(); 16 | }); 17 | d.run(fn.bind(this, test)); 18 | } 19 | } 20 | 21 | 22 | exports.testMultiSync = function(test) { 23 | var a = new stream.PassThrough(); 24 | var b = new stream.PassThrough(); 25 | var c = new stream.PassThrough(); 26 | var ss = SS() 27 | var done = false; 28 | ss.pipe(sink()).on('data', function(data) { 29 | done = true; 30 | clearTimeout(to); 31 | test.equal('hello world!', data, "Data in sink should be identical"); 32 | test.done(); 33 | }); 34 | 35 | process.nextTick(function () { 36 | ss.write(a); 37 | a.end('hello'); 38 | b.end(' world'); 39 | ss.write(b); 40 | ss.end(c); 41 | c.end('!'); 42 | }) 43 | 44 | var to = setTimeout(function(){ 45 | if(!done) { 46 | test.fail('no end detected'); 47 | test.done(); 48 | } 49 | }, 20) 50 | }.withDomain(true) 51 | 52 | 53 | exports.testMultiAsync = function(test) { 54 | var a = new stream.PassThrough(); 55 | var b = new stream.PassThrough(); 56 | var c = new stream.PassThrough(); 57 | var ss = SS() 58 | var done = false; 59 | ss.pipe(sink()).on('data', function(data) { 60 | done = true; 61 | test.equal('hello world!', data, "Data in sink should be identical"); 62 | clearTimeout(to); 63 | test.done(); 64 | }); 65 | 66 | ss.write(a); 67 | ss.write(b); 68 | 69 | setTimeout(function() { 70 | a.write('hello'); 71 | }, 10) 72 | setTimeout(function() { 73 | b.write(' world'); 74 | }, 20); 75 | setTimeout(function() { 76 | a.end(); 77 | }, 30); 78 | setTimeout(function() { 79 | b.end(); 80 | }, 40) 81 | setTimeout(function() { 82 | ss.end(c); 83 | }, 50) 84 | setTimeout(function() { 85 | c.end('!'); 86 | }) 87 | 88 | var to = setTimeout(function(){ 89 | if(!done) { 90 | test.fail('no end detected'); 91 | test.done(); 92 | } 93 | }, 1000) 94 | } 95 | 96 | -------------------------------------------------------------------------------- /lib/StreamStream.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var stream = require('stream'); 3 | var Readable = stream.Readable; 4 | 5 | function StreamStream (options) { 6 | // use without new 7 | if(!(this instanceof StreamStream)) return new StreamStream(options); 8 | 9 | // super 10 | stream.Transform.call(this, options); 11 | options = options || {}; 12 | 13 | var separator = options.separator; 14 | switch(typeof separator) { 15 | // if the separator is a constant value 16 | // we make it a function to be treated in the 17 | // next case 18 | case 'string': 19 | var val = separator; 20 | separator = function(cb) { 21 | process.nextTick(function() { 22 | cb(val); 23 | }) 24 | } 25 | // make this._separator() always return 26 | // a stream filled with the results of the callback 27 | // if the reset is a stream, pipe it as separator 28 | case 'function': 29 | var fn = separator; 30 | separator = function() { 31 | var ps = new stream.PassThrough(options); 32 | ps._isSeparator = true; 33 | fn(function(res) { 34 | if(res.readable) 35 | res.pipe(ps); 36 | else 37 | ps.end(res); 38 | }); 39 | return ps; 40 | } 41 | break; 42 | default: 43 | separator = null; 44 | break; 45 | } 46 | this._separator = separator; 47 | 48 | this._readableState.objectMode = options.objectMode; 49 | this._writableState.objectMode = true; 50 | } 51 | util.inherits(StreamStream, stream.Transform); 52 | 53 | StreamStream.prototype._transform = function _transform(stream, encoding, done) { 54 | if(this._lastStream && this._separator && this._needSeparator) { 55 | var withSep = new StreamStream(this._readableState.objectMode); 56 | withSep.write(this._separator()); 57 | withSep.end(stream); 58 | stream = withSep; 59 | } 60 | 61 | if(!(stream instanceof Readable)) { 62 | var readable = new Readable({objectMode: true}); 63 | stream = readable.wrap(stream); 64 | } 65 | 66 | var self = this; 67 | function onData (data) { 68 | // TODO pause stream on overrun 69 | self.push(data); 70 | } 71 | stream.on('data', onData); 72 | stream.on('end', function () { 73 | stream.removeListener('data', onData); 74 | done(); 75 | }); 76 | this._needSeparator = true; 77 | this._lastStream = stream; 78 | }; 79 | 80 | module.exports = StreamStream; 81 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/Floby/node-stream-stream.png?branch=master)](https://travis-ci.org/Floby/node-stream-stream) 2 | 3 | node-stream-stream 4 | ================== 5 | 6 | > A stream of streams in order to concatenate the contents of several streams 7 | 8 | Installation 9 | ------------ 10 | 11 | npm install --save stream-stream 12 | 13 | Usage 14 | ----- 15 | 16 | A StreamStream is a special kind of transform stream to which you write readable streams. 17 | The contents of the readable streams will be concatenated in the order you wrote them. 18 | 19 | ```javascript 20 | var ss = require('stream-stream'); 21 | var fs = require('fs'); 22 | 23 | var files = ['a.txt', 'b.txt', 'c.txt']; 24 | var stream = ss(); 25 | 26 | files.forEach(function(f) { 27 | stream.write(fs.createReadStream(f)); 28 | }); 29 | stream.end(); 30 | 31 | stream.pipe(process.stdout); 32 | ``` 33 | 34 | You can also add a separator between the contents of each stream by specifying a `separator` 35 | field in the options. 36 | 37 | ```javascript 38 | var mystream = new ss({ 39 | separator: '\n', 40 | 41 | // if separator is a function, it will get called 42 | // everytime the stream needs to insert a separator 43 | separator: function(cb) { 44 | cb('\n'); 45 | }, 46 | separator: function(cb) { 47 | cb(someReadableStream) 48 | } 49 | }); 50 | ``` 51 | 52 | It is also possible to `pipe()` to this stream from a readable stream in objectMode. 53 | 54 | 55 | Test 56 | ---- 57 | 58 | You can run the tests with `npm test`. You will need [nodeunit](https://github.com/caolan/nodeunit) 59 | 60 | Contributing 61 | ------------ 62 | 63 | Anyone is welcome to submit issues and pull requests 64 | 65 | thanks to [vanthome](https://github.com/vanthome) 66 | 67 | 68 | License 69 | ------- 70 | 71 | [MIT](http://opensource.org/licenses/MIT) 72 | 73 | Copyright (c) 2012 Florent Jaby 74 | 75 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 76 | 77 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 78 | 79 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 80 | -------------------------------------------------------------------------------- /test/test-objectMode.js: -------------------------------------------------------------------------------- 1 | var stream = require('stream'); 2 | var SS = require('../'); 3 | var sink = require('stream-sink'); 4 | 5 | exports.testObjectMode = function(test) { 6 | var options = {objectMode: true}; 7 | var a = new stream.PassThrough(options); 8 | var b = new stream.PassThrough(options); 9 | var c = new stream.PassThrough(options); 10 | var ss = SS(options) 11 | var done = false; 12 | ss.pipe(sink(options)).on('data', function(data) { 13 | done = true; 14 | clearTimeout(to); 15 | test.doesNotThrow(function() { 16 | test.ok(Array.isArray(data), "Data should be an array"); 17 | test.equal(data.length, 3, "Data should have same length"); 18 | test.equal(data.join(''), 'abc', 'Data should be abc') 19 | }) 20 | test.done(); 21 | }); 22 | 23 | ss.write(a); 24 | a.end('a'); 25 | b.end('b'); 26 | ss.write(b); 27 | ss.end(c); 28 | c.end('c'); 29 | 30 | var to = setTimeout(function(){ 31 | if(!done) { 32 | test.fail('no end detected'); 33 | test.done(); 34 | } 35 | }, 20) 36 | } 37 | 38 | exports.testobjectModeAsync = function(test) { 39 | var options = {objectMode: true}; 40 | var a = new stream.PassThrough(options); 41 | var b = new stream.PassThrough(options); 42 | var c = new stream.PassThrough(options); 43 | var ss = SS(options) 44 | var done = false; 45 | ss.pipe(sink(options)).on('data', function(data) { 46 | done = true; 47 | clearTimeout(to); 48 | test.doesNotThrow(function() { 49 | test.ok(Array.isArray(data), "Data should be an array"); 50 | test.equal(data.length, 3, "Data should have same length"); 51 | test.equal(data.join(''), 'abc', 'Data should be abc') 52 | }) 53 | test.done(); 54 | }); 55 | 56 | ss.write(a); 57 | ss.write(b); 58 | 59 | setTimeout(function() { 60 | a.write('a'); 61 | }, 10) 62 | setTimeout(function() { 63 | b.write('b'); 64 | }, 20); 65 | setTimeout(function() { 66 | a.end(); 67 | }, 30); 68 | setTimeout(function() { 69 | b.end(); 70 | }, 40) 71 | setTimeout(function() { 72 | ss.end(c); 73 | }, 50) 74 | setTimeout(function() { 75 | c.end('c'); 76 | }) 77 | 78 | var to = setTimeout(function(){ 79 | if(!done) { 80 | test.fail('no end detected'); 81 | test.done(); 82 | } 83 | }, 1000) 84 | } 85 | 86 | exports.testObjectModeOverrun = function (test) { 87 | var options = {objectMode: true}; 88 | var a = new stream.PassThrough(options); 89 | var b = new stream.PassThrough(options); 90 | var c = new stream.PassThrough(options); 91 | var ss = SS(options) 92 | var done = false; 93 | ss.pipe(sink(options)).on('data', function(data) { 94 | done = true; 95 | clearTimeout(to); 96 | test.doesNotThrow(function() { 97 | test.ok(Array.isArray(data), "Data should be an array"); 98 | test.equal(data.length, 6, "Data should have same length"); 99 | test.equal(data.join(''), 'aabbcc', 'Data should be aabbcc') 100 | }) 101 | test.done(); 102 | }); 103 | 104 | a.write('a'); 105 | ss.write(a); 106 | a.end('a'); 107 | b.write('b'); 108 | b.end('b'); 109 | ss.write(b); 110 | ss.end(c); 111 | c.write('c'); 112 | c.end('c'); 113 | 114 | var to = setTimeout(function(){ 115 | if(!done) { 116 | test.fail('no end detected'); 117 | test.done(); 118 | } 119 | }, 20) 120 | } 121 | -------------------------------------------------------------------------------- /test/test-separator.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var stream = require('stream'); 3 | var SS = require('../'); 4 | var sink = require('stream-sink'); 5 | 6 | function EmitStream (array) { 7 | stream.Readable.call(this, {objectMode:true}); 8 | this._source = array.slice(); 9 | } 10 | util.inherits(EmitStream, stream.Readable); 11 | EmitStream.prototype._read = function(size) { 12 | var chunk = this._source.shift(); 13 | if(!chunk) return this.push(null); 14 | 15 | var s = new stream.PassThrough(); 16 | s.name = chunk; 17 | process.nextTick(function() { 18 | s.end(chunk); 19 | }); 20 | this.push(s); 21 | } 22 | 23 | exports.testStreamSeparator = function(test) { 24 | var emitter = new EmitStream(['one', 'two', 'three']); 25 | var ss = SS({separator: ':'}); 26 | var s = sink(); 27 | var done = false; 28 | var to = setTimeout(function() { 29 | if(!done) { 30 | test.fail('No end detected') 31 | test.done(); 32 | } 33 | }, 500); 34 | 35 | emitter.pipe(ss).pipe(s).on('data', function(data) { 36 | test.equal(data, "one:two:three", "Data in sink should be identical"); 37 | done = true; 38 | test.done(); 39 | clearTimeout(to); 40 | }); 41 | 42 | } 43 | 44 | exports.testStreamSeparatorWhenWritingAsync = function(test) { 45 | var emitter = new EmitStream(['one', 'two', 'three']); 46 | var ss = SS({separator: ':'}); 47 | var s = sink(); 48 | var done = false; 49 | var to = setTimeout(function() { 50 | if(!done) { 51 | test.fail('No end detected') 52 | test.done(); 53 | } 54 | }, 500); 55 | 56 | emitter.pipe(ss, {end: false}).pipe(s).on('data', function(data) { 57 | test.equal(data, "one:two:three:four", "Data in sink should be identical"); 58 | done = true; 59 | test.done(); 60 | clearTimeout(to); 61 | }); 62 | emitter.on('end', function() { 63 | var async = stream.PassThrough(); 64 | setTimeout(function() { 65 | ss.end(async); 66 | process.nextTick(function() { 67 | async.end('four'); 68 | }); 69 | }, 100); 70 | }); 71 | } 72 | 73 | exports.testSeparatorIsFunction = function(test) { 74 | var emitter = new EmitStream(['one', 'two', 'three']); 75 | var ss = SS({separator: function(cb) { 76 | process.nextTick(function() { 77 | cb(':'); 78 | }) 79 | }}); 80 | var s = sink(); 81 | var done = false; 82 | var to = setTimeout(function() { 83 | if(!done) { 84 | test.fail('No end detected') 85 | test.done(); 86 | } 87 | }, 500); 88 | 89 | emitter.pipe(ss).pipe(s).on('data', function(data) { 90 | test.equal(data, "one:two:three", "Data in sink should be identical"); 91 | done = true; 92 | test.done(); 93 | clearTimeout(to); 94 | }); 95 | } 96 | 97 | exports.testSeparatorReturnsStream = function(test) { 98 | var emitter = new EmitStream(['one', 'two', 'three']); 99 | var ss = SS({separator: function(cb) { 100 | var mystream = new stream.PassThrough(); 101 | process.nextTick(function() { 102 | cb(mystream); 103 | }); 104 | process.nextTick(function() { 105 | mystream.end(':') 106 | }) 107 | }}); 108 | var s = sink(); 109 | var done = false; 110 | var to = setTimeout(function() { 111 | if(!done) { 112 | test.fail('No end detected') 113 | test.done(); 114 | } 115 | }, 500); 116 | 117 | emitter.pipe(ss).pipe(s).on('data', function(data) { 118 | test.equal(data, "one:two:three", "Data in sink should be identical"); 119 | done = true; 120 | test.done(); 121 | clearTimeout(to); 122 | }); 123 | } 124 | 125 | --------------------------------------------------------------------------------