├── .eslintrc ├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── index.js ├── package.json └── test ├── advanceOnClose.test.js ├── file1.txt ├── file2.txt └── test.js /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": [ 3 | "mocha" 4 | ], 5 | "env": { 6 | "browser": true, 7 | "node": true, 8 | "es6": true 9 | }, 10 | "extends": [ 11 | "eslint:recommended", 12 | "plugin:mocha/recommended" 13 | ], 14 | "parserOptions": { 15 | "ecmaVersion": 2017 16 | }, 17 | "rules": { 18 | "array-bracket-spacing": ["error"], 19 | "array-callback-return": ["error"], 20 | "arrow-parens": ["error"], 21 | "arrow-spacing": ["error", { "before": true, "after": true }], 22 | "block-spacing": ["error", "always"], 23 | "brace-style": ["error", "1tbs", { "allowSingleLine": true }], 24 | "comma-spacing": ["error", {"before": false, "after": true}], 25 | "comma-style": ["error"], 26 | "consistent-this": ["error", "self"], 27 | "dot-notation": ["error"], 28 | "eol-last": ["error"], 29 | "eqeqeq": ["error", "always"], 30 | "func-style": ["error", "expression", { "allowArrowFunctions": true }], 31 | "handle-callback-err": ["error"], 32 | "indent": ["error", 2, {"SwitchCase": 1, "VariableDeclarator": { "let": 2, "const": 2}}], 33 | "key-spacing": ["error", {"beforeColon": false, "afterColon": true}], 34 | "mocha/no-hooks-for-single-case": ["off"], 35 | "new-cap": ["error", {"capIsNew": false}], 36 | "newline-per-chained-call": ["off"], 37 | "no-console": ["error"], 38 | "no-iterator": ["error"], 39 | "no-mixed-requires": ["error"], 40 | "no-nested-ternary": ["off"], 41 | "no-path-concat": ["error"], 42 | "no-unneeded-ternary": ["error"], 43 | "no-unused-vars": ["error", {"vars": "all", "args": "after-used"}], 44 | "no-useless-constructor": ["off"], 45 | "no-var": ["error"], 46 | "object-curly-spacing": ["error", "always"], 47 | "prefer-arrow-callback": ["error"], 48 | "prefer-const": ["error"], 49 | "prefer-template": ["off"], 50 | "quote-props": ["error", "as-needed"], 51 | "semi": ["error", "always"], 52 | "space-before-blocks": ["error"], 53 | "space-in-parens": ["error"], 54 | "space-infix-ops": ["error"], 55 | "strict": ["error", "never"], 56 | "quotes": ["error", "single"] 57 | } 58 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | package-lock.json 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | .eslintrc 3 | .gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Sanders DeNardi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-stream-concat 2 | Simple and efficient node stream concatenation. 3 | 4 | `node-stream-concat` concatenates several streams into one single readable stream. The input streams can either be existing streams or can be determined on the fly by a user specified function. Because the library and tests use modern APIs, `node-stream-concat` supports Node LTS versions. Prior versions of the library (`< 1.0.0`) have been tested from Node versions v8.0.0 through v10.0.0, but should work with versions down to v0.12 (tests will fail < 8.0.0 because of .destroy()). 5 | 6 | npm install stream-concat 7 | 8 | # Usage 9 | 10 | ```js 11 | const StreamConcat = require('stream-concat'); 12 | const combinedStream = new StreamConcat(streams, [options]); 13 | ``` 14 | 15 | ## streams 16 | The simplest way to use StreamConcat is to supply an array of readable streams. 17 | 18 | ```js 19 | const fs = require('fs'); 20 | 21 | const stream1 = fs.createReadStream('file1.csv'); 22 | const stream2 = fs.createReadStream('file2.csv'); 23 | const stream3 = fs.createReadStream('file3.csv'); 24 | 25 | const output = fs.createWriteStream('combined.csv'); 26 | 27 | const combinedStream = new StreamConcat([stream1, stream2, stream3]); 28 | combinedStream.pipe(output); 29 | ``` 30 | 31 | However, when working with large amounts of data, this can lead to high memory usage and relatively poor performance (versus the original stream). This is because all streams' read queues are buffered and waiting to be read. 32 | 33 | A better way is to defer opening a new stream until the moment it's needed. You can do this by passing a function into the constructor that returns the next available stream, or `null` if there are no more streams. 34 | 35 | If we're reading from several large files, we can do the following. 36 | 37 | ```js 38 | const fs = require('fs'); 39 | 40 | const fileNames = ['file1.csv', 'file2.csv', 'file3.csv']; 41 | const fileIndex = 0; 42 | const nextStream = () => { 43 | if (fileIndex === fileNames.length) { 44 | return null; 45 | } 46 | return fs.createReadStream(fileNames[fileIndex++]); 47 | }; 48 | 49 | const combinedStream = new StreamConcat(nextStream); 50 | ``` 51 | 52 | Once StreamConcat is done with a stream it'll call `nextStream` and start using the returned stream (if not null). 53 | 54 | Additionally, the function you pass to the constructor can return a `Promise` that resolves to a `stream`. If the function fails, its error will be forwarded in an `error` event in the outer `StreamConcat` instance. 55 | 56 | ```js 57 | const fs = require('fs'); 58 | 59 | const fileNames = ['file1.csv', 'file2.csv', 'file3.csv']; 60 | const fileIndex = 0; 61 | const nextStreamAsync = () => { 62 | return new Promise((res) => { 63 | if (fileIndex === fileNames.length) { 64 | return null; 65 | } 66 | return fs.createReadStream(fileNames[fileIndex++]); 67 | }); 68 | }; 69 | 70 | const combinedStream = new StreamConcat(nextStreamAsync); 71 | ``` 72 | 73 | Errors emitted in the provided streams will also be forwarded to the outer `StreamConcat` instance: 74 | 75 | ```js 76 | const stream = require('stream'); 77 | const StreamConcat = require('stream-concat'); 78 | 79 | const fileIndex = 0; 80 | const nextStream = () => { 81 | if (fileIndex === 3) { 82 | return null; 83 | } 84 | return new stream.Readable({ 85 | read(){ throw new Error('Read failed'); } 86 | }).once('error', e=>console.log('Got inner error: ', e)); 87 | }; 88 | 89 | const combinedStream = new StreamConcat(nextStream); 90 | // will be called with the same "Read failed" error 91 | combinedStream.once('error', e=>console.log('Got outer error: ', e)); 92 | ``` 93 | 94 | ## options 95 | These are standard `Stream` [options](http://nodejs.org/api/stream.html#stream_new_stream_transform_options) passed to the underlying `Transform` stream. 96 | 97 | * `highWaterMark` Number The maximum number of bytes to store in the internal buffer before ceasing to read from the underlying resource. Default=16kb 98 | * `encoding` String If specified, then buffers will be decoded to strings using the specified encoding. Default=null 99 | * `objectMode` Boolean Whether this stream should behave as a stream of objects. Meaning that stream.read(n) returns a single value instead of a Buffer of size n. Default=false 100 | 101 | Additional options: 102 | * `advanceOnClose` Boolean Controls if the concatenation should move onto the next stream when the underlying streams emit close event, useful when operating on `Transform` streams and calling destroy on them to skip the remaining data (supported on node >=8). Default=false 103 | 104 | ## StreamConcat.addStream(newStream) 105 | If you've created the StreamConcat object from an array of streams, you can use `addStream()` as long as the last stream hasn't finishing being read (StreamConcat hasn't emitted the `end` event). 106 | 107 | To add streams to a StreamConcat object created from a function, you should modify the underlying data that the function is accessing. 108 | 109 | # Tests 110 | 111 | npm run test 112 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { Transform } = require('stream'); 2 | 3 | class StreamConcat extends Transform { 4 | constructor(streams, options = { }) { 5 | super(options); 6 | this.streams = streams; 7 | this.options = options; 8 | this.canAddStream = true; 9 | this.currentStream = null; 10 | this.streamIndex = 0; 11 | 12 | this.nextStream(); 13 | } 14 | addStream(newStream) { 15 | if (!this.canAddStream) { 16 | return this.emit('error', new Error('Can\'t add stream.')); 17 | } 18 | this.streams.push(newStream); 19 | } 20 | async nextStream() { 21 | this.currentStream = null; 22 | if (this.streams.constructor === Array && this.streamIndex < this.streams.length) { 23 | this.currentStream = this.streams[this.streamIndex++]; 24 | } else if (typeof this.streams === 'function') { 25 | this.canAddStream = false; 26 | this.currentStream = this.streams(); 27 | } 28 | 29 | const pipeStream = async () => { 30 | if (this.currentStream === null) { 31 | this.canAddStream = false; 32 | this.end(); 33 | } else if (typeof this.currentStream.then === 'function') { 34 | try { 35 | this.currentStream = await this.currentStream; 36 | } catch(e) { 37 | return this.emit('error', e); 38 | } 39 | await pipeStream(); 40 | } else { 41 | this.currentStream.pipe(this, { end: false }); 42 | let streamClosed = false; 43 | const goNext = async () => { 44 | if (streamClosed) { 45 | return; 46 | } 47 | streamClosed = true; 48 | await this.nextStream(); 49 | }; 50 | 51 | this.currentStream.once('error', (e) => this.emit('error', e)); 52 | this.currentStream.on('end', goNext); 53 | if (this.options.advanceOnClose) { 54 | this.currentStream.on('close', goNext); 55 | } 56 | } 57 | }; 58 | await pipeStream(); 59 | } 60 | _transform(chunk, encoding, callback) { 61 | callback(null, chunk); 62 | } 63 | } 64 | 65 | module.exports = StreamConcat; 66 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stream-concat", 3 | "version": "2.0.0", 4 | "engines": { 5 | "node": ">=12" 6 | }, 7 | "description": "Simple and efficient node stream concatenation.", 8 | "main": "index.js", 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/sedenardi/node-stream-concat.git" 12 | }, 13 | "keywords": [ 14 | "node", 15 | "stream", 16 | "concat", 17 | "streams", 18 | "concatenation" 19 | ], 20 | "author": "Sanders DeNardi (http://www.sandersdenardi.com/)", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/sedenardi/node-stream-concat/issues" 24 | }, 25 | "homepage": "https://github.com/sedenardi/node-stream-concat", 26 | "devDependencies": { 27 | "eslint": "^7.29.0", 28 | "eslint-plugin-mocha": "^9.0.0", 29 | "mocha": "^9.0.1" 30 | }, 31 | "scripts": { 32 | "lint": "eslint ./**/*.js", 33 | "test": "mocha" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/advanceOnClose.test.js: -------------------------------------------------------------------------------- 1 | /* eslint prefer-arrow-callback: ["off"] */ 2 | 3 | const fs = require('fs'); 4 | const path = require('path'); 5 | const assert = require('assert'); 6 | const { Readable } = require('stream'); 7 | 8 | const file1Path = path.join(__dirname, 'file1.txt'); 9 | const file2Path = path.join(__dirname, 'file2.txt'); 10 | const outputPath = path.join(__dirname, 'output-advanceOnClose.txt'); 11 | 12 | const StreamConcat = require('../index'); 13 | 14 | class CustomStream extends Readable { 15 | constructor(options) { 16 | super(options); 17 | } 18 | _read() { 19 | if (this.destroyed) { 20 | return; 21 | } 22 | 23 | this.push(', while this will be skipped,'); 24 | this.push(null); 25 | } 26 | _destroy(err, callback) { 27 | if (err) { 28 | return callback(err); 29 | } 30 | // This need to happen at least in the next event loop, 31 | // since destroy is called before registering the close event handler 32 | setTimeout(() => { 33 | this.destroyed = true; 34 | this.emit('close', null); 35 | callback(); 36 | }); 37 | } 38 | } 39 | 40 | describe('Concatenation with close', function() { 41 | before(function(done) { 42 | const streams = [ 43 | fs.createReadStream(file1Path), 44 | new CustomStream(), 45 | fs.createReadStream(file2Path), 46 | ]; 47 | 48 | let index = 0; 49 | const combinedStream = new StreamConcat(() => { 50 | const stream = streams[index]; 51 | if (!stream) { 52 | return null; 53 | } 54 | if (index === 1) { 55 | stream.destroy(); 56 | } 57 | index++; 58 | return stream; 59 | }, { 60 | advanceOnClose: true 61 | }); 62 | 63 | const output = fs.createWriteStream(outputPath); 64 | output.on('finish', () => { done(); }); 65 | 66 | combinedStream.pipe(output); 67 | }); 68 | 69 | it('output should be combination of two files, skipping the custom stream', function() { 70 | const output = fs.readFileSync(outputPath); 71 | assert.strictEqual(output.toString(), 'The quick brown fox jumps over the lazy dog.'); 72 | }); 73 | 74 | after(function() { 75 | fs.unlinkSync(outputPath); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /test/file1.txt: -------------------------------------------------------------------------------- 1 | The quick brown fox -------------------------------------------------------------------------------- /test/file2.txt: -------------------------------------------------------------------------------- 1 | jumps over the lazy dog. -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | /* eslint prefer-arrow-callback: ["off"] */ 2 | 3 | const fs = require('fs'); 4 | const path = require('path'); 5 | const assert = require('assert'); 6 | const { Readable } = require('stream'); 7 | 8 | const file1Path = path.join(__dirname, 'file1.txt'); 9 | const file2Path = path.join(__dirname, 'file2.txt'); 10 | const outputPath = path.join(__dirname, 'output.txt'); 11 | const outputPathIssue6 = path.join(__dirname, 'issue-6.dat'); 12 | 13 | const StreamConcat = require('../index'); 14 | 15 | describe('Concatenation', function() { 16 | before(function(done) { 17 | const file1 = fs.createReadStream(file1Path); 18 | const file2 = fs.createReadStream(file2Path); 19 | const combinedStream = new StreamConcat([file1, file2]); 20 | 21 | const output = fs.createWriteStream(outputPath); 22 | output.on('finish', () => { done(); }); 23 | 24 | combinedStream.pipe(output); 25 | }); 26 | it('output should be combination of two files', function() { 27 | const output = fs.readFileSync(outputPath); 28 | assert.strictEqual(output.toString(), 'The quick brown fox jumps over the lazy dog.'); 29 | }); 30 | 31 | it('#6)', function(done) { 32 | const stream = require('stream'); 33 | const $ = function(buff) { 34 | return new stream.Readable({ 35 | read: function() { 36 | this.push(buff); 37 | buff = null; 38 | } 39 | }); 40 | }; 41 | 42 | const header = Buffer.alloc(5); 43 | const footer = Buffer.alloc(5); 44 | let total = header.length + footer.length; 45 | const all = [$(header)]; 46 | for (let i = 0; i < 5; i++) { 47 | const one = Buffer.alloc(30 * 1024); 48 | const two = Buffer.alloc(30 * 1024); 49 | total += one.length + two.length; 50 | all.push(new StreamConcat([$(one), $(two)])); 51 | } 52 | all.push($(footer)); 53 | const master = new StreamConcat(all); 54 | const file = outputPathIssue6; 55 | const output = fs.createWriteStream(file); 56 | master.pipe(output); 57 | output.on('finish', () => { 58 | assert.strictEqual(fs.readFileSync(file).length, total); 59 | done(); 60 | }); 61 | 62 | }); 63 | 64 | it('output should be streamed with async callback', function(done) { 65 | const streams = [Readable.from(['concatenated']), Readable.from([' ']), Readable.from(['results'])]; 66 | const combined_stream = new StreamConcat(() => { 67 | return new Promise((resolve) => { 68 | setTimeout(() => { resolve(streams.shift() || null); }, 10); 69 | }); 70 | }); 71 | 72 | const chunks = []; 73 | combined_stream.on('data', (chunk) => { 74 | chunks.push(chunk); 75 | }); 76 | combined_stream.on('end', () => { 77 | assert.strictEqual(Buffer.concat(chunks).toString(), 'concatenated results'); 78 | done(); 79 | }); 80 | }); 81 | 82 | it('should emit errors in async callback', function(done) { 83 | const streams = [Readable.from(['concatenated']), Readable.from([' ']), Readable.from(['results'])]; 84 | const error = new Error('Stream fetch failed'); 85 | const combined_stream = new StreamConcat(() => { 86 | return new Promise((resolve, reject) => { 87 | setTimeout(() => { 88 | if (streams.length === 1) 89 | return reject(error); 90 | resolve(streams.shift() || null); 91 | }, 10); 92 | }); 93 | }); 94 | 95 | const chunks = []; 96 | combined_stream.on('data', (chunk) => { 97 | chunks.push(chunk); 98 | }); 99 | combined_stream.on('error', (e) => { 100 | assert.strictEqual(Buffer.concat(chunks).toString(), 'concatenated '); 101 | assert.strictEqual(e, error); 102 | done(); 103 | }); 104 | }); 105 | 106 | it('should forward errors from inner streams', function(done) { 107 | const error = new Error('Stream read failed'); 108 | const streams = [ 109 | Readable.from(['concatenated']), 110 | Readable.from([' ']), 111 | new Readable({ 112 | read() { throw error; } 113 | }), 114 | Readable.from(['results']), 115 | ]; 116 | const combined_stream = new StreamConcat(streams); 117 | 118 | const chunks = []; 119 | combined_stream.on('data', (chunk) => { 120 | chunks.push(chunk); 121 | }); 122 | combined_stream.on('error', (e) => { 123 | assert.strictEqual(Buffer.concat(chunks).toString(), 'concatenated '); 124 | assert.strictEqual(e, error); 125 | done(); 126 | }); 127 | }); 128 | 129 | after(function() { 130 | fs.unlinkSync(outputPath); 131 | fs.unlinkSync(outputPathIssue6); 132 | }); 133 | }); 134 | --------------------------------------------------------------------------------