├── .gitignore ├── .travis.yml ├── test ├── array.js ├── server │ └── ls.js ├── nothing.js ├── infer.js ├── objects.js ├── buffer.js ├── typedarray.js └── string.js ├── collaborators.md ├── LICENSE ├── package.json ├── readme.md └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '6.0' 4 | -------------------------------------------------------------------------------- /test/array.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | 4 | test('array stream', function (t) { 5 | t.plan(1) 6 | var arrays = concat({ encoding: 'array' }, function(out) { 7 | t.deepEqual(out, [1,2,3,4,5,6]) 8 | }) 9 | arrays.write([1,2,3]) 10 | arrays.write([4,5,6]) 11 | arrays.end() 12 | }) 13 | -------------------------------------------------------------------------------- /collaborators.md: -------------------------------------------------------------------------------- 1 | ## Collaborators 2 | 3 | concat-stream is only possible due to the excellent work of the following collaborators: 4 | 5 | 6 | 7 |
maxogdenGitHub/maxogden
mafintoshGitHub/mafintosh
8 | -------------------------------------------------------------------------------- /test/server/ls.js: -------------------------------------------------------------------------------- 1 | var concat = require('../../') 2 | var spawn = require('child_process').spawn 3 | var exec = require('child_process').exec 4 | var test = require('tape') 5 | 6 | test('ls command', function (t) { 7 | t.plan(1) 8 | var cmd = spawn('ls', [ __dirname ]) 9 | cmd.stdout.pipe( 10 | concat(function(out) { 11 | exec('ls ' + __dirname, function (err, body) { 12 | t.equal(out.toString('utf8'), body.toString('utf8')) 13 | }) 14 | }) 15 | ) 16 | }) 17 | -------------------------------------------------------------------------------- /test/nothing.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | 4 | test('no callback stream', function (t) { 5 | var stream = concat() 6 | stream.write('space') 7 | stream.end(' cats') 8 | t.end() 9 | }) 10 | 11 | test('no encoding set, no data', function (t) { 12 | var stream = concat(function(data) { 13 | t.deepEqual(data, []) 14 | t.end() 15 | }) 16 | stream.end() 17 | }) 18 | 19 | test('encoding set to string, no data', function (t) { 20 | var stream = concat({ encoding: 'string' }, function(data) { 21 | t.deepEqual(data, '') 22 | t.end() 23 | }) 24 | stream.end() 25 | }) 26 | -------------------------------------------------------------------------------- /test/infer.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | var bufferFrom = require('buffer-from') 4 | 5 | test('type inference works as expected', function(t) { 6 | var stream = concat() 7 | t.equal(stream.inferEncoding(['hello']), 'array', 'array') 8 | t.equal(stream.inferEncoding(bufferFrom('hello')), 'buffer', 'buffer') 9 | t.equal(stream.inferEncoding(undefined), 'buffer', 'buffer') 10 | t.equal(stream.inferEncoding(new Uint8Array(1)), 'uint8array', 'uint8array') 11 | t.equal(stream.inferEncoding('hello'), 'string', 'string') 12 | t.equal(stream.inferEncoding(''), 'string', 'string') 13 | t.equal(stream.inferEncoding({hello: "world"}), 'object', 'object') 14 | t.equal(stream.inferEncoding(1), 'buffer', 'buffer') 15 | t.end() 16 | }) 17 | -------------------------------------------------------------------------------- /test/objects.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | 4 | test('writing objects', function (t) { 5 | var stream = concat({encoding: "objects"}, concatted) 6 | function concatted(objs) { 7 | t.equal(objs.length, 2) 8 | t.deepEqual(objs[0], {"foo": "bar"}) 9 | t.deepEqual(objs[1], {"baz": "taco"}) 10 | } 11 | stream.write({"foo": "bar"}) 12 | stream.write({"baz": "taco"}) 13 | stream.end() 14 | t.end() 15 | }) 16 | 17 | 18 | test('switch to objects encoding if no encoding specified and objects are written', function (t) { 19 | var stream = concat(concatted) 20 | function concatted(objs) { 21 | t.equal(objs.length, 2) 22 | t.deepEqual(objs[0], {"foo": "bar"}) 23 | t.deepEqual(objs[1], {"baz": "taco"}) 24 | } 25 | stream.write({"foo": "bar"}) 26 | stream.write({"baz": "taco"}) 27 | stream.end() 28 | t.end() 29 | }) 30 | -------------------------------------------------------------------------------- /test/buffer.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | var TA = require('typedarray') 4 | var U8 = typeof Uint8Array !== 'undefined' ? Uint8Array : TA.Uint8Array 5 | var bufferFrom = require('buffer-from') 6 | 7 | test('buffer stream', function (t) { 8 | t.plan(2) 9 | var buffers = concat(function(out) { 10 | t.ok(Buffer.isBuffer(out)) 11 | t.equal(out.toString('utf8'), 'pizza Array is not a stringy cat') 12 | }) 13 | buffers.write(bufferFrom('pizza Array is not a ', 'utf8')) 14 | buffers.write(bufferFrom('stringy cat')) 15 | buffers.end() 16 | }) 17 | 18 | test('buffer mixed writes', function (t) { 19 | t.plan(2) 20 | var buffers = concat(function(out) { 21 | t.ok(Buffer.isBuffer(out)) 22 | t.equal(out.toString('utf8'), 'pizza Array is not a stringy cat555') 23 | }) 24 | buffers.write(bufferFrom('pizza')) 25 | buffers.write(' Array is not a ') 26 | buffers.write([ 115, 116, 114, 105, 110, 103, 121 ]) 27 | var u8 = new U8(4) 28 | u8[0] = 32; u8[1] = 99; u8[2] = 97; u8[3] = 116 29 | buffers.write(u8) 30 | buffers.write(555) 31 | buffers.end() 32 | }) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2013 Max Ogden 4 | 5 | Permission is hereby granted, free of charge, 6 | to any person obtaining a copy of this software and 7 | associated documentation files (the "Software"), to 8 | deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, 10 | merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom 12 | the Software is furnished to do so, 13 | subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice 16 | shall be included in all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 19 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 21 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 22 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 23 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 24 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /test/typedarray.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | var TA = require('typedarray') 4 | var U8 = typeof Uint8Array !== 'undefined' ? Uint8Array : TA.Uint8Array 5 | var bufferFrom = require('buffer-from') 6 | 7 | test('typed array stream', function (t) { 8 | t.plan(2) 9 | var a = new U8(5) 10 | a[0] = 97; a[1] = 98; a[2] = 99; a[3] = 100; a[4] = 101; 11 | var b = new U8(3) 12 | b[0] = 32; b[1] = 102; b[2] = 103; 13 | var c = new U8(4) 14 | c[0] = 32; c[1] = 120; c[2] = 121; c[3] = 122; 15 | 16 | var arrays = concat({ encoding: 'Uint8Array' }, function(out) { 17 | t.equal(typeof out.subarray, 'function') 18 | t.deepEqual(bufferFrom(out).toString('utf8'), 'abcde fg xyz') 19 | }) 20 | arrays.write(a) 21 | arrays.write(b) 22 | arrays.end(c) 23 | }) 24 | 25 | test('typed array from strings, buffers, and arrays', function (t) { 26 | t.plan(2) 27 | var arrays = concat({ encoding: 'Uint8Array' }, function(out) { 28 | t.equal(typeof out.subarray, 'function') 29 | t.deepEqual(bufferFrom(out).toString('utf8'), 'abcde fg xyz') 30 | }) 31 | arrays.write('abcde') 32 | arrays.write(bufferFrom(' fg ')) 33 | arrays.end([ 120, 121, 122 ]) 34 | }) 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "concat-stream", 3 | "version": "2.0.0", 4 | "description": "writable stream that concatenates strings or binary data and calls a callback with the result", 5 | "tags": [ 6 | "stream", 7 | "simple", 8 | "util", 9 | "utility" 10 | ], 11 | "author": "Max Ogden ", 12 | "repository": { 13 | "type": "git", 14 | "url": "http://github.com/maxogden/concat-stream.git" 15 | }, 16 | "bugs": { 17 | "url": "http://github.com/maxogden/concat-stream/issues" 18 | }, 19 | "engines": [ 20 | "node >= 6.0" 21 | ], 22 | "main": "index.js", 23 | "files": [ 24 | "index.js" 25 | ], 26 | "scripts": { 27 | "test": "tape test/*.js test/server/*.js" 28 | }, 29 | "license": "MIT", 30 | "dependencies": { 31 | "buffer-from": "^1.0.0", 32 | "inherits": "^2.0.3", 33 | "readable-stream": "^3.0.2", 34 | "typedarray": "^0.0.6" 35 | }, 36 | "devDependencies": { 37 | "tape": "^4.6.3" 38 | }, 39 | "testling": { 40 | "files": "test/*.js", 41 | "browsers": [ 42 | "ie/8..latest", 43 | "firefox/17..latest", 44 | "firefox/nightly", 45 | "chrome/22..latest", 46 | "chrome/canary", 47 | "opera/12..latest", 48 | "opera/next", 49 | "safari/5.1..latest", 50 | "ipad/6.0..latest", 51 | "iphone/6.0..latest", 52 | "android-browser/4.2..latest" 53 | ] 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /test/string.js: -------------------------------------------------------------------------------- 1 | var concat = require('../') 2 | var test = require('tape') 3 | var TA = require('typedarray') 4 | var U8 = typeof Uint8Array !== 'undefined' ? Uint8Array : TA.Uint8Array 5 | var bufferFrom = require('buffer-from') 6 | 7 | test('string -> buffer stream', function (t) { 8 | t.plan(2) 9 | var strings = concat({ encoding: 'buffer'}, function(out) { 10 | t.ok(Buffer.isBuffer(out)) 11 | t.equal(out.toString('utf8'), 'nacho dogs') 12 | }) 13 | strings.write("nacho ") 14 | strings.write("dogs") 15 | strings.end() 16 | }) 17 | 18 | test('string stream', function (t) { 19 | t.plan(2) 20 | var strings = concat({ encoding: 'string' }, function(out) { 21 | t.equal(typeof out, 'string') 22 | t.equal(out, 'nacho dogs') 23 | }) 24 | strings.write("nacho ") 25 | strings.write("dogs") 26 | strings.end() 27 | }) 28 | 29 | test('end chunk', function (t) { 30 | t.plan(1) 31 | var endchunk = concat({ encoding: 'string' }, function(out) { 32 | t.equal(out, 'this is the end') 33 | }) 34 | endchunk.write("this ") 35 | endchunk.write("is the ") 36 | endchunk.end("end") 37 | }) 38 | 39 | test('string from mixed write encodings', function (t) { 40 | t.plan(2) 41 | var strings = concat({ encoding: 'string' }, function(out) { 42 | t.equal(typeof out, 'string') 43 | t.equal(out, 'nacho dogs') 44 | }) 45 | strings.write('na') 46 | strings.write(bufferFrom('cho')) 47 | strings.write([ 32, 100 ]) 48 | var u8 = new U8(3) 49 | u8[0] = 111; u8[1] = 103; u8[2] = 115; 50 | strings.end(u8) 51 | }) 52 | 53 | test('string from buffers with multibyte characters', function (t) { 54 | t.plan(2) 55 | var strings = concat({ encoding: 'string' }, function(out) { 56 | t.equal(typeof out, 'string') 57 | t.equal(out, '☃☃☃☃☃☃☃☃') 58 | }) 59 | var snowman = bufferFrom('☃') 60 | for (var i = 0; i < 8; i++) { 61 | strings.write(snowman.slice(0, 1)) 62 | strings.write(snowman.slice(1)) 63 | } 64 | strings.end() 65 | }) 66 | 67 | test('string infer encoding with empty string chunk', function (t) { 68 | t.plan(2) 69 | var strings = concat(function(out) { 70 | t.equal(typeof out, 'string') 71 | t.equal(out, 'nacho dogs') 72 | }) 73 | strings.write("") 74 | strings.write("nacho ") 75 | strings.write("dogs") 76 | strings.end() 77 | }) 78 | 79 | test('to string numbers', function (t) { 80 | var write = concat(function (str) { 81 | t.equal(str, 'a1000') 82 | t.end() 83 | }) 84 | 85 | write.write('a') 86 | write.write(1000) 87 | write.end() 88 | }) 89 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # concat-stream 2 | 3 | Writable stream that concatenates all the data from a stream and calls a callback with the result. Use this when you want to collect all the data from a stream into a single buffer. 4 | 5 | [![Build Status](https://travis-ci.org/maxogden/concat-stream.svg?branch=master)](https://travis-ci.org/maxogden/concat-stream) 6 | 7 | [![NPM](https://nodei.co/npm/concat-stream.png)](https://nodei.co/npm/concat-stream/) 8 | 9 | ### description 10 | 11 | Streams emit many buffers. If you want to collect all of the buffers, and when the stream ends concatenate all of the buffers together and receive a single buffer then this is the module for you. 12 | 13 | Only use this if you know you can fit all of the output of your stream into a single Buffer (e.g. in RAM). 14 | 15 | There are also `objectMode` streams that emit things other than Buffers, and you can concatenate these too. See below for details. 16 | 17 | ## Related 18 | 19 | `concat-stream` is part of the [mississippi stream utility collection](https://github.com/maxogden/mississippi) which includes more useful stream modules similar to this one. 20 | 21 | ### examples 22 | 23 | #### Buffers 24 | 25 | ```js 26 | var fs = require('fs') 27 | var concat = require('concat-stream') 28 | 29 | var readStream = fs.createReadStream('cat.png') 30 | var concatStream = concat(gotPicture) 31 | 32 | readStream.on('error', handleError) 33 | readStream.pipe(concatStream) 34 | 35 | function gotPicture(imageBuffer) { 36 | // imageBuffer is all of `cat.png` as a node.js Buffer 37 | } 38 | 39 | function handleError(err) { 40 | // handle your error appropriately here, e.g.: 41 | console.error(err) // print the error to STDERR 42 | process.exit(1) // exit program with non-zero exit code 43 | } 44 | 45 | ``` 46 | 47 | #### Arrays 48 | 49 | ```js 50 | var write = concat(function(data) {}) 51 | write.write([1,2,3]) 52 | write.write([4,5,6]) 53 | write.end() 54 | // data will be [1,2,3,4,5,6] in the above callback 55 | ``` 56 | 57 | #### Uint8Arrays 58 | 59 | ```js 60 | var write = concat(function(data) {}) 61 | var a = new Uint8Array(3) 62 | a[0] = 97; a[1] = 98; a[2] = 99 63 | write.write(a) 64 | write.write('!') 65 | write.end(Buffer.from('!!1')) 66 | ``` 67 | 68 | See `test/` for more examples 69 | 70 | # methods 71 | 72 | ```js 73 | var concat = require('concat-stream') 74 | ``` 75 | 76 | ## var writable = concat(opts={}, cb) 77 | 78 | Return a `writable` stream that will fire `cb(data)` with all of the data that 79 | was written to the stream. Data can be written to `writable` as strings, 80 | Buffers, arrays of byte integers, and Uint8Arrays. 81 | 82 | By default `concat-stream` will give you back the same data type as the type of the first buffer written to the stream. Use `opts.encoding` to set what format `data` should be returned as, e.g. if you if you don't want to rely on the built-in type checking or for some other reason. 83 | 84 | * `string` - get a string 85 | * `buffer` - get back a Buffer 86 | * `array` - get an array of byte integers 87 | * `uint8array`, `u8`, `uint8` - get back a Uint8Array 88 | * `object`, get back an array of Objects 89 | 90 | If you don't specify an encoding, and the types can't be inferred (e.g. you write things that aren't in the list above), it will try to convert concat them into a `Buffer`. 91 | 92 | If nothing is written to `writable` then `data` will be an empty array `[]`. 93 | 94 | # error handling 95 | 96 | `concat-stream` does not handle errors for you, so you must handle errors on whatever streams you pipe into `concat-stream`. This is a general rule when programming with node.js streams: always handle errors on each and every stream. Since `concat-stream` is not itself a stream it does not emit errors. 97 | 98 | We recommend using [`end-of-stream`](https://npmjs.org/end-of-stream) or [`pump`](https://npmjs.org/pump) for writing error tolerant stream code. 99 | 100 | # license 101 | 102 | MIT LICENSE 103 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Writable = require('readable-stream').Writable 2 | var inherits = require('inherits') 3 | var bufferFrom = require('buffer-from') 4 | 5 | if (typeof Uint8Array === 'undefined') { 6 | var U8 = require('typedarray').Uint8Array 7 | } else { 8 | var U8 = Uint8Array 9 | } 10 | 11 | function ConcatStream(opts, cb) { 12 | if (!(this instanceof ConcatStream)) return new ConcatStream(opts, cb) 13 | 14 | if (typeof opts === 'function') { 15 | cb = opts 16 | opts = {} 17 | } 18 | if (!opts) opts = {} 19 | 20 | var encoding = opts.encoding 21 | var shouldInferEncoding = false 22 | 23 | if (!encoding) { 24 | shouldInferEncoding = true 25 | } else { 26 | encoding = String(encoding).toLowerCase() 27 | if (encoding === 'u8' || encoding === 'uint8') { 28 | encoding = 'uint8array' 29 | } 30 | } 31 | 32 | Writable.call(this, { objectMode: true }) 33 | 34 | this.encoding = encoding 35 | this.shouldInferEncoding = shouldInferEncoding 36 | 37 | if (cb) this.on('finish', function () { cb(this.getBody()) }) 38 | this.body = [] 39 | } 40 | 41 | module.exports = ConcatStream 42 | inherits(ConcatStream, Writable) 43 | 44 | ConcatStream.prototype._write = function(chunk, enc, next) { 45 | this.body.push(chunk) 46 | next() 47 | } 48 | 49 | ConcatStream.prototype.inferEncoding = function (buff) { 50 | var firstBuffer = buff === undefined ? this.body[0] : buff; 51 | if (Buffer.isBuffer(firstBuffer)) return 'buffer' 52 | if (typeof Uint8Array !== 'undefined' && firstBuffer instanceof Uint8Array) return 'uint8array' 53 | if (Array.isArray(firstBuffer)) return 'array' 54 | if (typeof firstBuffer === 'string') return 'string' 55 | if (Object.prototype.toString.call(firstBuffer) === "[object Object]") return 'object' 56 | return 'buffer' 57 | } 58 | 59 | ConcatStream.prototype.getBody = function () { 60 | if (!this.encoding && this.body.length === 0) return [] 61 | if (this.shouldInferEncoding) this.encoding = this.inferEncoding() 62 | if (this.encoding === 'array') return arrayConcat(this.body) 63 | if (this.encoding === 'string') return stringConcat(this.body) 64 | if (this.encoding === 'buffer') return bufferConcat(this.body) 65 | if (this.encoding === 'uint8array') return u8Concat(this.body) 66 | return this.body 67 | } 68 | 69 | var isArray = Array.isArray || function (arr) { 70 | return Object.prototype.toString.call(arr) == '[object Array]' 71 | } 72 | 73 | function isArrayish (arr) { 74 | return /Array\]$/.test(Object.prototype.toString.call(arr)) 75 | } 76 | 77 | function isBufferish (p) { 78 | return typeof p === 'string' || isArrayish(p) || (p && typeof p.subarray === 'function') 79 | } 80 | 81 | function stringConcat (parts) { 82 | var strings = [] 83 | var needsToString = false 84 | for (var i = 0; i < parts.length; i++) { 85 | var p = parts[i] 86 | if (typeof p === 'string') { 87 | strings.push(p) 88 | } else if (Buffer.isBuffer(p)) { 89 | strings.push(p) 90 | } else if (isBufferish(p)) { 91 | strings.push(bufferFrom(p)) 92 | } else { 93 | strings.push(bufferFrom(String(p))) 94 | } 95 | } 96 | if (Buffer.isBuffer(parts[0])) { 97 | strings = Buffer.concat(strings) 98 | strings = strings.toString('utf8') 99 | } else { 100 | strings = strings.join('') 101 | } 102 | return strings 103 | } 104 | 105 | function bufferConcat (parts) { 106 | var bufs = [] 107 | for (var i = 0; i < parts.length; i++) { 108 | var p = parts[i] 109 | if (Buffer.isBuffer(p)) { 110 | bufs.push(p) 111 | } else if (isBufferish(p)) { 112 | bufs.push(bufferFrom(p)) 113 | } else { 114 | bufs.push(bufferFrom(String(p))) 115 | } 116 | } 117 | return Buffer.concat(bufs) 118 | } 119 | 120 | function arrayConcat (parts) { 121 | var res = [] 122 | for (var i = 0; i < parts.length; i++) { 123 | res.push.apply(res, parts[i]) 124 | } 125 | return res 126 | } 127 | 128 | function u8Concat (parts) { 129 | var len = 0 130 | for (var i = 0; i < parts.length; i++) { 131 | if (typeof parts[i] === 'string') { 132 | parts[i] = bufferFrom(parts[i]) 133 | } 134 | len += parts[i].length 135 | } 136 | var u8 = new U8(len) 137 | for (var i = 0, offset = 0; i < parts.length; i++) { 138 | var part = parts[i] 139 | for (var j = 0; j < part.length; j++) { 140 | u8[offset++] = part[j] 141 | } 142 | } 143 | return u8 144 | } 145 | --------------------------------------------------------------------------------