├── .gitignore
├── .travis.yml
├── test
├── array.js
├── server
│ └── ls.js
├── nothing.js
├── infer.js
├── objects.js
├── buffer.js
├── typedarray.js
└── string.js
├── collaborators.md
├── LICENSE
├── package.json
├── readme.md
└── index.js
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | node_js:
3 | - '6.0'
4 |
--------------------------------------------------------------------------------
/test/array.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 |
4 | test('array stream', function (t) {
5 | t.plan(1)
6 | var arrays = concat({ encoding: 'array' }, function(out) {
7 | t.deepEqual(out, [1,2,3,4,5,6])
8 | })
9 | arrays.write([1,2,3])
10 | arrays.write([4,5,6])
11 | arrays.end()
12 | })
13 |
--------------------------------------------------------------------------------
/collaborators.md:
--------------------------------------------------------------------------------
1 | ## Collaborators
2 |
3 | concat-stream is only possible due to the excellent work of the following collaborators:
4 |
5 |
8 |
--------------------------------------------------------------------------------
/test/server/ls.js:
--------------------------------------------------------------------------------
1 | var concat = require('../../')
2 | var spawn = require('child_process').spawn
3 | var exec = require('child_process').exec
4 | var test = require('tape')
5 |
6 | test('ls command', function (t) {
7 | t.plan(1)
8 | var cmd = spawn('ls', [ __dirname ])
9 | cmd.stdout.pipe(
10 | concat(function(out) {
11 | exec('ls ' + __dirname, function (err, body) {
12 | t.equal(out.toString('utf8'), body.toString('utf8'))
13 | })
14 | })
15 | )
16 | })
17 |
--------------------------------------------------------------------------------
/test/nothing.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 |
4 | test('no callback stream', function (t) {
5 | var stream = concat()
6 | stream.write('space')
7 | stream.end(' cats')
8 | t.end()
9 | })
10 |
11 | test('no encoding set, no data', function (t) {
12 | var stream = concat(function(data) {
13 | t.deepEqual(data, [])
14 | t.end()
15 | })
16 | stream.end()
17 | })
18 |
19 | test('encoding set to string, no data', function (t) {
20 | var stream = concat({ encoding: 'string' }, function(data) {
21 | t.deepEqual(data, '')
22 | t.end()
23 | })
24 | stream.end()
25 | })
26 |
--------------------------------------------------------------------------------
/test/infer.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 | var bufferFrom = require('buffer-from')
4 |
5 | test('type inference works as expected', function(t) {
6 | var stream = concat()
7 | t.equal(stream.inferEncoding(['hello']), 'array', 'array')
8 | t.equal(stream.inferEncoding(bufferFrom('hello')), 'buffer', 'buffer')
9 | t.equal(stream.inferEncoding(undefined), 'buffer', 'buffer')
10 | t.equal(stream.inferEncoding(new Uint8Array(1)), 'uint8array', 'uint8array')
11 | t.equal(stream.inferEncoding('hello'), 'string', 'string')
12 | t.equal(stream.inferEncoding(''), 'string', 'string')
13 | t.equal(stream.inferEncoding({hello: "world"}), 'object', 'object')
14 | t.equal(stream.inferEncoding(1), 'buffer', 'buffer')
15 | t.end()
16 | })
17 |
--------------------------------------------------------------------------------
/test/objects.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 |
4 | test('writing objects', function (t) {
5 | var stream = concat({encoding: "objects"}, concatted)
6 | function concatted(objs) {
7 | t.equal(objs.length, 2)
8 | t.deepEqual(objs[0], {"foo": "bar"})
9 | t.deepEqual(objs[1], {"baz": "taco"})
10 | }
11 | stream.write({"foo": "bar"})
12 | stream.write({"baz": "taco"})
13 | stream.end()
14 | t.end()
15 | })
16 |
17 |
18 | test('switch to objects encoding if no encoding specified and objects are written', function (t) {
19 | var stream = concat(concatted)
20 | function concatted(objs) {
21 | t.equal(objs.length, 2)
22 | t.deepEqual(objs[0], {"foo": "bar"})
23 | t.deepEqual(objs[1], {"baz": "taco"})
24 | }
25 | stream.write({"foo": "bar"})
26 | stream.write({"baz": "taco"})
27 | stream.end()
28 | t.end()
29 | })
30 |
--------------------------------------------------------------------------------
/test/buffer.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 | var TA = require('typedarray')
4 | var U8 = typeof Uint8Array !== 'undefined' ? Uint8Array : TA.Uint8Array
5 | var bufferFrom = require('buffer-from')
6 |
7 | test('buffer stream', function (t) {
8 | t.plan(2)
9 | var buffers = concat(function(out) {
10 | t.ok(Buffer.isBuffer(out))
11 | t.equal(out.toString('utf8'), 'pizza Array is not a stringy cat')
12 | })
13 | buffers.write(bufferFrom('pizza Array is not a ', 'utf8'))
14 | buffers.write(bufferFrom('stringy cat'))
15 | buffers.end()
16 | })
17 |
18 | test('buffer mixed writes', function (t) {
19 | t.plan(2)
20 | var buffers = concat(function(out) {
21 | t.ok(Buffer.isBuffer(out))
22 | t.equal(out.toString('utf8'), 'pizza Array is not a stringy cat555')
23 | })
24 | buffers.write(bufferFrom('pizza'))
25 | buffers.write(' Array is not a ')
26 | buffers.write([ 115, 116, 114, 105, 110, 103, 121 ])
27 | var u8 = new U8(4)
28 | u8[0] = 32; u8[1] = 99; u8[2] = 97; u8[3] = 116
29 | buffers.write(u8)
30 | buffers.write(555)
31 | buffers.end()
32 | })
33 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License
2 |
3 | Copyright (c) 2013 Max Ogden
4 |
5 | Permission is hereby granted, free of charge,
6 | to any person obtaining a copy of this software and
7 | associated documentation files (the "Software"), to
8 | deal in the Software without restriction, including
9 | without limitation the rights to use, copy, modify,
10 | merge, publish, distribute, sublicense, and/or sell
11 | copies of the Software, and to permit persons to whom
12 | the Software is furnished to do so,
13 | subject to the following conditions:
14 |
15 | The above copyright notice and this permission notice
16 | shall be included in all copies or substantial portions of the Software.
17 |
18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
20 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
22 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
/test/typedarray.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 | var TA = require('typedarray')
4 | var U8 = typeof Uint8Array !== 'undefined' ? Uint8Array : TA.Uint8Array
5 | var bufferFrom = require('buffer-from')
6 |
7 | test('typed array stream', function (t) {
8 | t.plan(2)
9 | var a = new U8(5)
10 | a[0] = 97; a[1] = 98; a[2] = 99; a[3] = 100; a[4] = 101;
11 | var b = new U8(3)
12 | b[0] = 32; b[1] = 102; b[2] = 103;
13 | var c = new U8(4)
14 | c[0] = 32; c[1] = 120; c[2] = 121; c[3] = 122;
15 |
16 | var arrays = concat({ encoding: 'Uint8Array' }, function(out) {
17 | t.equal(typeof out.subarray, 'function')
18 | t.deepEqual(bufferFrom(out).toString('utf8'), 'abcde fg xyz')
19 | })
20 | arrays.write(a)
21 | arrays.write(b)
22 | arrays.end(c)
23 | })
24 |
25 | test('typed array from strings, buffers, and arrays', function (t) {
26 | t.plan(2)
27 | var arrays = concat({ encoding: 'Uint8Array' }, function(out) {
28 | t.equal(typeof out.subarray, 'function')
29 | t.deepEqual(bufferFrom(out).toString('utf8'), 'abcde fg xyz')
30 | })
31 | arrays.write('abcde')
32 | arrays.write(bufferFrom(' fg '))
33 | arrays.end([ 120, 121, 122 ])
34 | })
35 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "concat-stream",
3 | "version": "2.0.0",
4 | "description": "writable stream that concatenates strings or binary data and calls a callback with the result",
5 | "tags": [
6 | "stream",
7 | "simple",
8 | "util",
9 | "utility"
10 | ],
11 | "author": "Max Ogden ",
12 | "repository": {
13 | "type": "git",
14 | "url": "http://github.com/maxogden/concat-stream.git"
15 | },
16 | "bugs": {
17 | "url": "http://github.com/maxogden/concat-stream/issues"
18 | },
19 | "engines": [
20 | "node >= 6.0"
21 | ],
22 | "main": "index.js",
23 | "files": [
24 | "index.js"
25 | ],
26 | "scripts": {
27 | "test": "tape test/*.js test/server/*.js"
28 | },
29 | "license": "MIT",
30 | "dependencies": {
31 | "buffer-from": "^1.0.0",
32 | "inherits": "^2.0.3",
33 | "readable-stream": "^3.0.2",
34 | "typedarray": "^0.0.6"
35 | },
36 | "devDependencies": {
37 | "tape": "^4.6.3"
38 | },
39 | "testling": {
40 | "files": "test/*.js",
41 | "browsers": [
42 | "ie/8..latest",
43 | "firefox/17..latest",
44 | "firefox/nightly",
45 | "chrome/22..latest",
46 | "chrome/canary",
47 | "opera/12..latest",
48 | "opera/next",
49 | "safari/5.1..latest",
50 | "ipad/6.0..latest",
51 | "iphone/6.0..latest",
52 | "android-browser/4.2..latest"
53 | ]
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/test/string.js:
--------------------------------------------------------------------------------
1 | var concat = require('../')
2 | var test = require('tape')
3 | var TA = require('typedarray')
4 | var U8 = typeof Uint8Array !== 'undefined' ? Uint8Array : TA.Uint8Array
5 | var bufferFrom = require('buffer-from')
6 |
7 | test('string -> buffer stream', function (t) {
8 | t.plan(2)
9 | var strings = concat({ encoding: 'buffer'}, function(out) {
10 | t.ok(Buffer.isBuffer(out))
11 | t.equal(out.toString('utf8'), 'nacho dogs')
12 | })
13 | strings.write("nacho ")
14 | strings.write("dogs")
15 | strings.end()
16 | })
17 |
18 | test('string stream', function (t) {
19 | t.plan(2)
20 | var strings = concat({ encoding: 'string' }, function(out) {
21 | t.equal(typeof out, 'string')
22 | t.equal(out, 'nacho dogs')
23 | })
24 | strings.write("nacho ")
25 | strings.write("dogs")
26 | strings.end()
27 | })
28 |
29 | test('end chunk', function (t) {
30 | t.plan(1)
31 | var endchunk = concat({ encoding: 'string' }, function(out) {
32 | t.equal(out, 'this is the end')
33 | })
34 | endchunk.write("this ")
35 | endchunk.write("is the ")
36 | endchunk.end("end")
37 | })
38 |
39 | test('string from mixed write encodings', function (t) {
40 | t.plan(2)
41 | var strings = concat({ encoding: 'string' }, function(out) {
42 | t.equal(typeof out, 'string')
43 | t.equal(out, 'nacho dogs')
44 | })
45 | strings.write('na')
46 | strings.write(bufferFrom('cho'))
47 | strings.write([ 32, 100 ])
48 | var u8 = new U8(3)
49 | u8[0] = 111; u8[1] = 103; u8[2] = 115;
50 | strings.end(u8)
51 | })
52 |
53 | test('string from buffers with multibyte characters', function (t) {
54 | t.plan(2)
55 | var strings = concat({ encoding: 'string' }, function(out) {
56 | t.equal(typeof out, 'string')
57 | t.equal(out, '☃☃☃☃☃☃☃☃')
58 | })
59 | var snowman = bufferFrom('☃')
60 | for (var i = 0; i < 8; i++) {
61 | strings.write(snowman.slice(0, 1))
62 | strings.write(snowman.slice(1))
63 | }
64 | strings.end()
65 | })
66 |
67 | test('string infer encoding with empty string chunk', function (t) {
68 | t.plan(2)
69 | var strings = concat(function(out) {
70 | t.equal(typeof out, 'string')
71 | t.equal(out, 'nacho dogs')
72 | })
73 | strings.write("")
74 | strings.write("nacho ")
75 | strings.write("dogs")
76 | strings.end()
77 | })
78 |
79 | test('to string numbers', function (t) {
80 | var write = concat(function (str) {
81 | t.equal(str, 'a1000')
82 | t.end()
83 | })
84 |
85 | write.write('a')
86 | write.write(1000)
87 | write.end()
88 | })
89 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # concat-stream
2 |
3 | Writable stream that concatenates all the data from a stream and calls a callback with the result. Use this when you want to collect all the data from a stream into a single buffer.
4 |
5 | [](https://travis-ci.org/maxogden/concat-stream)
6 |
7 | [](https://nodei.co/npm/concat-stream/)
8 |
9 | ### description
10 |
11 | Streams emit many buffers. If you want to collect all of the buffers, and when the stream ends concatenate all of the buffers together and receive a single buffer then this is the module for you.
12 |
13 | Only use this if you know you can fit all of the output of your stream into a single Buffer (e.g. in RAM).
14 |
15 | There are also `objectMode` streams that emit things other than Buffers, and you can concatenate these too. See below for details.
16 |
17 | ## Related
18 |
19 | `concat-stream` is part of the [mississippi stream utility collection](https://github.com/maxogden/mississippi) which includes more useful stream modules similar to this one.
20 |
21 | ### examples
22 |
23 | #### Buffers
24 |
25 | ```js
26 | var fs = require('fs')
27 | var concat = require('concat-stream')
28 |
29 | var readStream = fs.createReadStream('cat.png')
30 | var concatStream = concat(gotPicture)
31 |
32 | readStream.on('error', handleError)
33 | readStream.pipe(concatStream)
34 |
35 | function gotPicture(imageBuffer) {
36 | // imageBuffer is all of `cat.png` as a node.js Buffer
37 | }
38 |
39 | function handleError(err) {
40 | // handle your error appropriately here, e.g.:
41 | console.error(err) // print the error to STDERR
42 | process.exit(1) // exit program with non-zero exit code
43 | }
44 |
45 | ```
46 |
47 | #### Arrays
48 |
49 | ```js
50 | var write = concat(function(data) {})
51 | write.write([1,2,3])
52 | write.write([4,5,6])
53 | write.end()
54 | // data will be [1,2,3,4,5,6] in the above callback
55 | ```
56 |
57 | #### Uint8Arrays
58 |
59 | ```js
60 | var write = concat(function(data) {})
61 | var a = new Uint8Array(3)
62 | a[0] = 97; a[1] = 98; a[2] = 99
63 | write.write(a)
64 | write.write('!')
65 | write.end(Buffer.from('!!1'))
66 | ```
67 |
68 | See `test/` for more examples
69 |
70 | # methods
71 |
72 | ```js
73 | var concat = require('concat-stream')
74 | ```
75 |
76 | ## var writable = concat(opts={}, cb)
77 |
78 | Return a `writable` stream that will fire `cb(data)` with all of the data that
79 | was written to the stream. Data can be written to `writable` as strings,
80 | Buffers, arrays of byte integers, and Uint8Arrays.
81 |
82 | By default `concat-stream` will give you back the same data type as the type of the first buffer written to the stream. Use `opts.encoding` to set what format `data` should be returned as, e.g. if you if you don't want to rely on the built-in type checking or for some other reason.
83 |
84 | * `string` - get a string
85 | * `buffer` - get back a Buffer
86 | * `array` - get an array of byte integers
87 | * `uint8array`, `u8`, `uint8` - get back a Uint8Array
88 | * `object`, get back an array of Objects
89 |
90 | If you don't specify an encoding, and the types can't be inferred (e.g. you write things that aren't in the list above), it will try to convert concat them into a `Buffer`.
91 |
92 | If nothing is written to `writable` then `data` will be an empty array `[]`.
93 |
94 | # error handling
95 |
96 | `concat-stream` does not handle errors for you, so you must handle errors on whatever streams you pipe into `concat-stream`. This is a general rule when programming with node.js streams: always handle errors on each and every stream. Since `concat-stream` is not itself a stream it does not emit errors.
97 |
98 | We recommend using [`end-of-stream`](https://npmjs.org/end-of-stream) or [`pump`](https://npmjs.org/pump) for writing error tolerant stream code.
99 |
100 | # license
101 |
102 | MIT LICENSE
103 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | var Writable = require('readable-stream').Writable
2 | var inherits = require('inherits')
3 | var bufferFrom = require('buffer-from')
4 |
5 | if (typeof Uint8Array === 'undefined') {
6 | var U8 = require('typedarray').Uint8Array
7 | } else {
8 | var U8 = Uint8Array
9 | }
10 |
11 | function ConcatStream(opts, cb) {
12 | if (!(this instanceof ConcatStream)) return new ConcatStream(opts, cb)
13 |
14 | if (typeof opts === 'function') {
15 | cb = opts
16 | opts = {}
17 | }
18 | if (!opts) opts = {}
19 |
20 | var encoding = opts.encoding
21 | var shouldInferEncoding = false
22 |
23 | if (!encoding) {
24 | shouldInferEncoding = true
25 | } else {
26 | encoding = String(encoding).toLowerCase()
27 | if (encoding === 'u8' || encoding === 'uint8') {
28 | encoding = 'uint8array'
29 | }
30 | }
31 |
32 | Writable.call(this, { objectMode: true })
33 |
34 | this.encoding = encoding
35 | this.shouldInferEncoding = shouldInferEncoding
36 |
37 | if (cb) this.on('finish', function () { cb(this.getBody()) })
38 | this.body = []
39 | }
40 |
41 | module.exports = ConcatStream
42 | inherits(ConcatStream, Writable)
43 |
44 | ConcatStream.prototype._write = function(chunk, enc, next) {
45 | this.body.push(chunk)
46 | next()
47 | }
48 |
49 | ConcatStream.prototype.inferEncoding = function (buff) {
50 | var firstBuffer = buff === undefined ? this.body[0] : buff;
51 | if (Buffer.isBuffer(firstBuffer)) return 'buffer'
52 | if (typeof Uint8Array !== 'undefined' && firstBuffer instanceof Uint8Array) return 'uint8array'
53 | if (Array.isArray(firstBuffer)) return 'array'
54 | if (typeof firstBuffer === 'string') return 'string'
55 | if (Object.prototype.toString.call(firstBuffer) === "[object Object]") return 'object'
56 | return 'buffer'
57 | }
58 |
59 | ConcatStream.prototype.getBody = function () {
60 | if (!this.encoding && this.body.length === 0) return []
61 | if (this.shouldInferEncoding) this.encoding = this.inferEncoding()
62 | if (this.encoding === 'array') return arrayConcat(this.body)
63 | if (this.encoding === 'string') return stringConcat(this.body)
64 | if (this.encoding === 'buffer') return bufferConcat(this.body)
65 | if (this.encoding === 'uint8array') return u8Concat(this.body)
66 | return this.body
67 | }
68 |
69 | var isArray = Array.isArray || function (arr) {
70 | return Object.prototype.toString.call(arr) == '[object Array]'
71 | }
72 |
73 | function isArrayish (arr) {
74 | return /Array\]$/.test(Object.prototype.toString.call(arr))
75 | }
76 |
77 | function isBufferish (p) {
78 | return typeof p === 'string' || isArrayish(p) || (p && typeof p.subarray === 'function')
79 | }
80 |
81 | function stringConcat (parts) {
82 | var strings = []
83 | var needsToString = false
84 | for (var i = 0; i < parts.length; i++) {
85 | var p = parts[i]
86 | if (typeof p === 'string') {
87 | strings.push(p)
88 | } else if (Buffer.isBuffer(p)) {
89 | strings.push(p)
90 | } else if (isBufferish(p)) {
91 | strings.push(bufferFrom(p))
92 | } else {
93 | strings.push(bufferFrom(String(p)))
94 | }
95 | }
96 | if (Buffer.isBuffer(parts[0])) {
97 | strings = Buffer.concat(strings)
98 | strings = strings.toString('utf8')
99 | } else {
100 | strings = strings.join('')
101 | }
102 | return strings
103 | }
104 |
105 | function bufferConcat (parts) {
106 | var bufs = []
107 | for (var i = 0; i < parts.length; i++) {
108 | var p = parts[i]
109 | if (Buffer.isBuffer(p)) {
110 | bufs.push(p)
111 | } else if (isBufferish(p)) {
112 | bufs.push(bufferFrom(p))
113 | } else {
114 | bufs.push(bufferFrom(String(p)))
115 | }
116 | }
117 | return Buffer.concat(bufs)
118 | }
119 |
120 | function arrayConcat (parts) {
121 | var res = []
122 | for (var i = 0; i < parts.length; i++) {
123 | res.push.apply(res, parts[i])
124 | }
125 | return res
126 | }
127 |
128 | function u8Concat (parts) {
129 | var len = 0
130 | for (var i = 0; i < parts.length; i++) {
131 | if (typeof parts[i] === 'string') {
132 | parts[i] = bufferFrom(parts[i])
133 | }
134 | len += parts[i].length
135 | }
136 | var u8 = new U8(len)
137 | for (var i = 0, offset = 0; i < parts.length; i++) {
138 | var part = parts[i]
139 | for (var j = 0; j < part.length; j++) {
140 | u8[offset++] = part[j]
141 | }
142 | }
143 | return u8
144 | }
145 |
--------------------------------------------------------------------------------