├── .gitignore ├── LICENSE ├── MemDuplex.js ├── README.md ├── index.js ├── package.json ├── readysetstream.png └── tests ├── 256KB.js └── unit └── readySetStream.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Lauren Spiegel 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of ReadySetStream nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /MemDuplex.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | 4 | const assert = require ('assert'); 5 | const Duplex = require('stream').Duplex; 6 | 7 | /** 8 | * Creates in memory writable and readable stream 9 | * @return {MemDuplex} a MemDuplex instance 10 | */ 11 | class MemDuplex extends Duplex { 12 | constructor(location) { 13 | // Piping to the memDuplex will pause if writableBuffer 14 | // exceeds this amount 15 | super({ highWaterMark: 8 * 1024 * 1024 }); // 8MB 16 | this.buffers = []; 17 | this.location = location; 18 | // Once writable stream is finished it will emit a finish 19 | // event 20 | this.once('finish', () => { 21 | this._read(); 22 | }); 23 | } 24 | _write(chunk, enc, cb) { 25 | assert(Buffer.isBuffer(chunk)); 26 | this.buffers.push(chunk); 27 | this._read(); 28 | cb(); 29 | } 30 | _read() { 31 | while (this.buffers.length > 0) { 32 | const pushed = this.push(this.buffers.shift(), 'binary'); 33 | if (!pushed) { 34 | break; 35 | } 36 | } 37 | // finished is property of writable stream 38 | // indicating writing is done. 39 | // Once the writing is done and we have pushed 40 | // out everything buffered, we're done 41 | if (this.buffers.length === 0 && this._writableState.finished) { 42 | this.push(null); // End of buffer 43 | } 44 | } 45 | } 46 | 47 | module.exports = MemDuplex; 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![logo](readysetstream.png) 2 | 3 | Node streaming with double buffering 4 | ------------ 5 | 6 | This lightweight module (no runtime dependencies!) speeds up the streaming of data retrieved from multiple sources. 7 | 8 | If you have an object that is broken up into multiple parts and you have to retrieve each part and send to a response object, this module will significantly increase the speed of the response while maintaining the order of the parts. 9 | 10 | For instance, if data was put via a multipart upload and you have to retrieve all of the parts to respond to a get request, this module allows you to easily buffer two parts in memory at a time while streaming the next required part to the response. 11 | 12 | Installation 13 | ------------ 14 | 15 | $ npm install --save ready-set-stream 16 | 17 | Usage 18 | --------------- 19 | 20 | First, import the `readySetStream` function into your program: 21 | 22 | ```javascript 23 | import { readySetStream } from 'ready-set-stream'; 24 | ``` 25 | 26 | Second, call the `readySetStream` function 27 | with the following arguments: 28 | 29 | (a) an array of locations (each location serves as the first argument to your data retrieval function), 30 | (b) a data retrieval function which takes a location, a logger and a callback as arguments, 31 | (c) the response object, and 32 | (d) a logger object (optional) 33 | (e) an error handling function taking an error object as only argument (optional, default is to call response.connection.destroy() on error) 34 | 35 | Example 36 | --------------- 37 | 38 | If you would like to stream a number of files in a certain order to a response object simply: 39 | 40 | First, define your locations array with the file paths: 41 | 42 | ```javascript 43 | const locations = ["read me first", "i'm second", "don't forget about me!"]; 44 | ``` 45 | 46 | Second, wrap the fs.createReadStream function in a function so that you can send it a location argument, a logger argument and a callback argument. 47 | 48 | ```javascript 49 | function dataRetrievalFunction(location, logger, callback) { 50 | const readStream = fs.createReadStream(location); 51 | return callback(null, readStream); 52 | } 53 | ``` 54 | 55 | Third, call `readySetStream` with your locations array, dataRetrievalFunction and response object as arguments. 56 | 57 | ```javascript 58 | readySetStream(locations, dataRetrievalFunction, response); 59 | ``` 60 | 61 | Tests 62 | ------------ 63 | 64 | $ npm install -g mocha 65 | $ npm test 66 | 67 | Thanks 68 | ------ 69 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const MemDuplex = require('./MemDuplex.js'); 4 | 5 | 6 | function _sendMemDuplexToResponse(memDuplexes, index, errorHandlerFn, 7 | response, logger){ 8 | if(memDuplexes[index] === undefined){ 9 | return response.end(); 10 | } 11 | const memDuplexOnCall = memDuplexes[index]; 12 | memDuplexOnCall.on('data', chunk => { 13 | response.write(chunk); 14 | }); 15 | memDuplexOnCall.on('error', err => { 16 | logger.error('error piping data from source'); 17 | errorHandlerFn(err); 18 | }); 19 | memDuplexOnCall.on('end', () => { 20 | return process.nextTick(_sendMemDuplexToResponse, 21 | memDuplexes, index + 1, errorHandlerFn, response, logger); 22 | }); 23 | } 24 | 25 | function _fillMemDuplex(memDuplexes, index, dataRetrievalFn, errorHandlerFn, 26 | response, logger){ 27 | return dataRetrievalFn(memDuplexes[index].location, logger, 28 | (err, readable) => { 29 | if(err){ 30 | logger.error('failed to get full object', { 31 | error: err, 32 | method: '_fillMemDuplex', 33 | }); 34 | return errorHandlerFn(err); 35 | } 36 | readable.pipe(memDuplexes[index]); 37 | if(memDuplexes[index + 2]){ 38 | readable.on('end', () => { 39 | return process.nextTick(_fillMemDuplex, memDuplexes, index + 2, 40 | dataRetrievalFn, errorHandlerFn, response, logger); 41 | }); 42 | } 43 | readable.on('error', err => { 44 | logger.error('error piping data from readable to memDuplex'); 45 | return errorHandlerFn(err); 46 | }); 47 | }); 48 | } 49 | 50 | exports.readySetStream = function readySetStream(locations, dataRetrievalFn, 51 | response, logger, errorHandlerFn) { 52 | if (!logger) { 53 | logger = console; 54 | } 55 | if (locations.length === 0) { 56 | return response.end(); 57 | } 58 | if (errorHandlerFn === undefined) { 59 | errorHandlerFn = err => { response.connection.destroy(); } 60 | } 61 | const memDuplexes = locations.map((location) => { 62 | return new MemDuplex(location); 63 | }); 64 | 65 | _sendMemDuplexToResponse(memDuplexes, 0, errorHandlerFn, 66 | response, logger); 67 | _fillMemDuplex(memDuplexes, 0, dataRetrievalFn, errorHandlerFn, 68 | response, logger); 69 | if (memDuplexes.length > 1){ 70 | _fillMemDuplex(memDuplexes, 1, dataRetrievalFn, errorHandlerFn, 71 | response, logger); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ready-set-stream", 3 | "version": "1.1.0", 4 | "description": "Node streaming with double buffering", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha --timeout 5500 tests/unit" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/LaurenSpiegel/ReadySetStream.git" 12 | }, 13 | "keywords": [ 14 | "stream", 15 | "double", 16 | "buffer", 17 | "duplex" 18 | ], 19 | "author": "Lauren Spiegel", 20 | "license": "BSD-3-Clause", 21 | "bugs": { 22 | "url": "https://github.com/LaurenSpiegel/ReadySetStream/issues" 23 | }, 24 | "homepage": "https://github.com/LaurenSpiegel/ReadySetStream#readme", 25 | "devDependencies": { 26 | "async": "^2.0.0-rc.6", 27 | "mocha": "^2.5.3", 28 | "node-mocks-http": "^1.5.2" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /readysetstream.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LaurenSpiegel/ReadySetStream/aaee2840ec2e85b84428c564afda6ccb6d6e737c/readysetstream.png -------------------------------------------------------------------------------- /tests/unit/readySetStream.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const assert = require('assert'); 4 | const async = require('async'); 5 | const crypto = require('crypto'); 6 | const EventEmitter = require('events').EventEmitter; 7 | const fs = require('fs'); 8 | const httpMocks = require('node-mocks-http'); 9 | const proc = require('child_process'); 10 | const readySetStream = require('../../index').readySetStream; 11 | 12 | 13 | const b256KB = require("../256KB.js").b256KB; 14 | const b1MB = Buffer.concat([b256KB, b256KB, b256KB, b256KB]); 15 | const b10MB = Buffer.concat([b1MB, b1MB, b1MB, b1MB, b1MB, b1MB, b1MB, b1MB, b1MB, b1MB]); 16 | const file10MBData = b10MB; 17 | const file20MBData = new Buffer(20971520).fill('**wildcard**'); 18 | const file30MBData = Buffer.concat([b10MB, b10MB, b10MB]); 19 | const file40MBData = new Buffer(41943040) 20 | .fill('Random numbers should not be generated with a method chosen at random.'); 21 | 22 | const files = { 23 | file10MB: file10MBData, 24 | file20MB: file20MBData, 25 | file30MB: file30MBData, 26 | file40MB: file40MBData, 27 | }; 28 | 29 | function createFiles(fileNamesSizes, callback) { 30 | return async.forEachOf(fileNamesSizes, 31 | function createFile(data, name, next) { 32 | return fs.writeFile(name, data, next); 33 | }, 34 | err => { 35 | assert.ifError(err); 36 | return callback(); 37 | }); 38 | } 39 | 40 | function deleteFiles(files, callback) { 41 | proc.spawn('rm', files).on('exit', code => { 42 | assert.strictEqual(code, 0); 43 | return callback(); 44 | }); 45 | } 46 | 47 | function dataRetrieval(key, logger, callback) { 48 | const readStream = fs.createReadStream(key); 49 | return callback(null, readStream); 50 | } 51 | 52 | function testBody(filesTested, callback) { 53 | let expectedHash = crypto.createHash('md5'); 54 | filesTested.forEach(fileName => { 55 | expectedHash.update(`${files[fileName]}`); 56 | }); 57 | expectedHash = expectedHash.digest('hex'); 58 | const response = httpMocks.createResponse({ 59 | eventEmitter: EventEmitter, 60 | }); 61 | response.on('end', () => { 62 | const data = response._getData(); 63 | const finalHash = crypto.createHash('md5') 64 | .update(data).digest('hex'); 65 | assert.strictEqual(finalHash, expectedHash); 66 | return callback(); 67 | }); 68 | readySetStream(filesTested, dataRetrieval, response); 69 | } 70 | 71 | describe('readySetStream', () => { 72 | before(done => { 73 | createFiles(files, done); 74 | }); 75 | 76 | after(done => { 77 | deleteFiles(Object.keys(files), done); 78 | }); 79 | 80 | 81 | it('should get data that is stored in one location', done => { 82 | return testBody(['file10MB'], done); 83 | }); 84 | 85 | 86 | it('should get data of two parts where each part is of equal ' + 87 | 'size', done => { 88 | return testBody(['file10MB', 'file10MB'], done); 89 | }); 90 | 91 | 92 | it('should get data of two parts where the first part is bigger ' + 93 | 'than the second part', done => { 94 | return testBody(['file20MB', 'file10MB'], done); 95 | }); 96 | 97 | 98 | it('should get data of two parts where the first part is much bigger ' + 99 | 'than the second part', done => { 100 | return testBody(['file40MB', 'file10MB'], done); 101 | }); 102 | 103 | 104 | it('should get data of two parts where the first part is smaller ' + 105 | 'than the second part', done => { 106 | return testBody(['file10MB', 'file20MB'], done); 107 | }); 108 | 109 | 110 | it('should get data of two parts where the first part is ' + 111 | 'much smaller than the second part', done => { 112 | return testBody(['file10MB', 'file40MB'], done); 113 | }); 114 | 115 | 116 | it('should get data of three parts where the size of each part is ' + 117 | 'bigger than the previous', done => { 118 | return testBody(['file10MB', 'file20MB', 'file30MB'], done); 119 | }); 120 | 121 | 122 | it('should get data of three parts where the size of each part is ' + 123 | 'smaller than the previous', done => { 124 | return testBody(['file40MB', 'file30MB', 'file20MB'], done); 125 | }); 126 | 127 | 128 | it('should get data of four parts where the size of each part is ' + 129 | 'bigger than the previous', done => { 130 | return testBody(['file10MB', 'file20MB', 'file30MB', 'file40MB'], done); 131 | }); 132 | 133 | 134 | it('should get data of four parts where the size of each part is ' + 135 | 'smaller than the previous', done => { 136 | return testBody(['file40MB', 'file30MB', 'file20MB', 'file10MB'], done); 137 | }); 138 | 139 | 140 | it('should get data of four parts with mixed part sizes', done => { 141 | return testBody(['file20MB', 'file10MB', 'file30MB', 'file40MB'], done); 142 | }); 143 | 144 | 145 | it('should get data of five parts with mixed part sizes', done => { 146 | return testBody(['file20MB', 'file30MB', 'file40MB', 'file10MB', 147 | 'file30MB'], done); 148 | }); 149 | }); 150 | 151 | describe('ready set stream errors', () => { 152 | it('should destroy connection if error retrieving data', done => { 153 | // No file created to retrieve so will error. 154 | const response = httpMocks.createResponse({ 155 | eventEmitter: EventEmitter, 156 | }); 157 | response.connection = { 158 | destroy: () => { 159 | done(); 160 | }, 161 | }; 162 | response.on('end', () => { 163 | return done(new Error('end reached instead of destroying connection')); 164 | }); 165 | readySetStream(['file10MB'], dataRetrieval, response); 166 | }) 167 | }) 168 | --------------------------------------------------------------------------------