├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── example.js ├── package.json ├── stringstream.js └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | node_modules 15 | npm-debug.log -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 4 4 | - 6 5 | - 8 6 | - 10 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Michael Hart 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Decode streams into strings without setEncoding 2 | 3 | ```js 4 | const fs = require('fs') 5 | const zlib = require('zlib') 6 | const strs = require('stringstream') 7 | 8 | const utf8Stream = fs.createReadStream('massiveLogFile.gz') 9 | .pipe(zlib.createGunzip()) 10 | .pipe(strs('utf8')) 11 | 12 | utf8Stream.on('data', str => console.log(`This will be a string: ${str}`)) 13 | ``` 14 | 15 | ## API 16 | 17 | - `strs(to, [options])` – creates a transform stream that converts the input into strings in `to` encoding (eg, `utf8`, `hex`, `base64`) 18 | - `strs(from, to, [options])` – creates a transform stream converts the input from strings in `from` encoding to strings in `to` encoding 19 | 20 | `options` can be anything compatible with the standard Node.js [`new stream.Transform([options])` constructor](https://nodejs.org/api/stream.html#stream_new_stream_transform_options) 21 | 22 | ## NB: This library was originally written before Node.js [correctly encoded base64 strings from streams](https://github.com/nodejs/node/commit/061f2075cf81017cdb40de80533ba18746743c94) 23 | 24 | Back in the day, calling `.setEncoding('base64')` on a readable stream didn't 25 | align correctly, which was one of the main reasons I wrote this library – 26 | however this hasn't been the case for a long time, so this library is 27 | now really only useful in scenarios where you don't want to call 28 | `.setEncoding()` for whatever reason. 29 | 30 | It also handles input and output text encodings: 31 | 32 | ```js 33 | // Stream from utf8 to hex to base64... Why not, ay. 34 | const hex64Stream = fs.createReadStream('myFile.txt') 35 | .pipe(strs('utf8', 'hex')) 36 | .pipe(strs('hex', 'base64')) 37 | ``` 38 | 39 | Also deals with `base64` output correctly by aligning each emitted data 40 | chunk so that there are no dangling `=` characters: 41 | 42 | ```js 43 | const stream = fs.createReadStream('myFile.jpg').pipe(strs('base64')) 44 | 45 | let base64Str = '' 46 | 47 | stream.on('data', data => base64Str += data) 48 | stream.on('end', () => { 49 | console.log('My base64 encoded file is: ' + base64Str) 50 | console.log('Original file is: ' + Buffer.from(base64Str, 'base64')) 51 | }) 52 | ``` 53 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const zlib = require('zlib') 3 | const strs = require('stringstream') 4 | 5 | const utf8Stream = fs.createReadStream('massiveLogFile.gz') 6 | .pipe(zlib.createGunzip()) 7 | .pipe(strs('utf8')) 8 | 9 | utf8Stream.pipe(process.stdout) 10 | 11 | // Stream from utf8 to hex to base64... Why not, ay. 12 | const hex64Stream = fs.createReadStream('myFile.txt') 13 | .pipe(strs('utf8', 'hex')) 14 | .pipe(strs('hex', 'base64')) 15 | 16 | hex64Stream.pipe(process.stdout) 17 | 18 | // Deals with base64 correctly by aligning chunks 19 | const stream = fs.createReadStream('myFile.jpg').pipe(strs('base64')) 20 | 21 | let base64Str = '' 22 | 23 | stream.on('data', data => base64Str += data) 24 | stream.on('end', () => { 25 | console.log('My base64 encoded file is: ' + base64Str) 26 | console.log('Original file size: ' + Buffer.from(base64Str, 'base64').length) 27 | }) 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stringstream", 3 | "version": "1.0.0", 4 | "description": "Encode and decode streams into string streams", 5 | "repository": "mhart/StringStream", 6 | "author": "Michael Hart ", 7 | "license": "MIT", 8 | "main": "stringstream.js", 9 | "engines": { 10 | "node": ">=4.0.0" 11 | }, 12 | "scripts": { 13 | "test": "node test.js" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /stringstream.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const util = require('util') 4 | const Transform = require('stream').Transform 5 | 6 | module.exports = StringStream 7 | 8 | const CHAR_ALIGN = { 9 | 'hex': 2, 10 | 'base64': 4, 11 | } 12 | 13 | function StringStream(from, to, options) { 14 | if (!(this instanceof StringStream)) return new StringStream(from, to, options) 15 | 16 | Transform.call(this, options) 17 | 18 | if (typeof to !== 'string') { 19 | options = to 20 | to = from || 'utf8' 21 | from = null 22 | } 23 | this.setEncoding(to) 24 | this.fromEncoding = from 25 | this.fromBuffer = '' 26 | this.fromAlign = CHAR_ALIGN[this.fromEncoding] 27 | } 28 | util.inherits(StringStream, Transform) 29 | 30 | StringStream.prototype._transform = function(chunk, encoding, cb) { 31 | if (!this.fromEncoding) { 32 | return cb(null, chunk) 33 | } 34 | let str = '' + chunk 35 | if (!this.fromAlign) { 36 | return cb(null, Buffer.from(str, this.fromEncoding)) 37 | } 38 | this.fromBuffer += str 39 | if (this.fromBuffer.length < this.fromAlign) { 40 | return cb() 41 | } 42 | const len = this.fromBuffer.length - (this.fromBuffer.length % this.fromAlign) 43 | str = this.fromBuffer.slice(0, len) 44 | this.fromBuffer = this.fromBuffer.slice(len) 45 | cb(null, Buffer.from(str, this.fromEncoding)) 46 | } 47 | 48 | StringStream.prototype._flush = function(cb) { 49 | if (this.fromBuffer) { 50 | const str = Buffer.from(this.fromBuffer, this.fromEncoding) 51 | str && this.push(str) 52 | } 53 | cb() // Can only supply data to callback from Node.js v7.0 onwards 54 | } 55 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | /* eslint-disable no-return-assign */ 4 | const assert = require('assert') 5 | const Readable = require('stream').Readable 6 | const strs = require('.') 7 | 8 | const HIGH = new Readable().readableHighWaterMark || 16384 9 | 10 | function assertStream(vals, streams, expected) { 11 | const readable = [new Readable({read() { vals.concat(null).forEach(c => this.push(c)) }})] 12 | let actual = '' 13 | readable.concat(streams).reduce((readable, stream) => readable.pipe(stream)).on('data', str => actual += str).on('end', () => { 14 | assert.strictEqual(actual, expected, `Failed for ${vals}: '${actual}' === '${expected}'`) 15 | console.log('ok') 16 | }) 17 | } 18 | 19 | assertStream(['a'], strs(), 'a') 20 | assertStream(['a'], strs('utf8'), 'a') 21 | 22 | assertStream(['a'], strs('hex'), '61') 23 | 24 | assertStream(['a'], strs('base64'), 'YQ==') 25 | 26 | assertStream(['aa'], strs('utf8'), 'aa') 27 | 28 | assertStream(['aa'], strs('hex'), '6161') 29 | 30 | assertStream(['aa'], strs('base64'), 'YWE=') 31 | assertStream(['aaa'], strs('base64'), 'YWFh') 32 | assertStream(['aaaa'], strs('base64'), 'YWFhYQ==') 33 | assertStream(['aaaaa'], strs('base64'), 'YWFhYWE=') 34 | assertStream(['aaaaaa'], strs('base64'), 'YWFhYWFh') 35 | assertStream(['abcb'], strs('base64', 'base64'), 'abcb') 36 | assertStream(['abc', 'b'], strs('base64', 'base64'), 'abcb') 37 | 38 | assertStream(['ab'], strs('hex', 'hex'), 'ab') 39 | assertStream(['a', 'b'], strs('hex', 'hex'), 'ab') 40 | assertStream(['a', 'b', 'ab'], strs('hex', 'hex'), 'abab') 41 | assertStream(['61'], strs('hex', 'utf8'), 'a') 42 | assertStream(['6', '1'], strs('hex', 'utf8'), 'a') 43 | assertStream(['6', '1', '6', '1'], strs('hex', 'utf8'), 'aa') 44 | 45 | assertStream(['abcbb', 'bbb'], strs('base64', 'base64'), 'abcbbbbb') 46 | 47 | assertStream(['YQ'], strs('base64', 'utf8'), 'a') 48 | assertStream(['YQ='], strs('base64', 'utf8'), 'a') 49 | assertStream(['YQ=='], strs('base64', 'utf8'), 'a') 50 | assertStream(['YWE'], strs('base64', 'utf8'), 'aa') 51 | assertStream(['YWE='], strs('base64', 'utf8'), 'aa') 52 | assertStream(['YWFh'], strs('base64', 'utf8'), 'aaa') 53 | assertStream(['YWFhYQ'], strs('base64', 'utf8'), 'aaaa') 54 | assertStream(['YWFhYQ='], strs('base64', 'utf8'), 'aaaa') 55 | assertStream(['YWFhYQ=='], strs('base64', 'utf8'), 'aaaa') 56 | assertStream(['YWFhYWE'], strs('base64', 'utf8'), 'aaaaa') 57 | assertStream(['YWFhYWE='], strs('base64', 'utf8'), 'aaaaa') 58 | assertStream(['YWFhYWFh'], strs('base64', 'utf8'), 'aaaaaa') 59 | 60 | assertStream([new Array(HIGH).join('a')], strs('utf8'), new Array(HIGH).join('a')) 61 | assertStream([new Array(HIGH + 1).join('a')], strs('utf8'), new Array(HIGH + 1).join('a')) 62 | assertStream([new Array(HIGH + 2).join('a')], strs('utf8'), new Array(HIGH + 2).join('a')) 63 | assertStream([new Array(HIGH).join('a'), 'a'], strs('utf8'), new Array(HIGH + 1).join('a')) 64 | assertStream([new Array(HIGH + 1).join('a'), 'a'], strs('utf8'), new Array(HIGH + 2).join('a')) 65 | assertStream([new Array(HIGH + 2).join('a'), 'a'], strs('utf8'), new Array(HIGH + 3).join('a')) 66 | 67 | assertStream([new Array(HIGH + 1).join('a')], strs('hex', 'hex'), new Array(HIGH + 1).join('a')) 68 | assertStream([new Array(HIGH + 3).join('a')], strs('hex', 'hex'), new Array(HIGH + 3).join('a')) 69 | assertStream([new Array(HIGH).join('a'), 'a'], strs('hex', 'hex'), new Array(HIGH + 1).join('a')) 70 | assertStream([new Array(HIGH + 2).join('a'), 'a'], strs('hex', 'hex'), new Array(HIGH + 3).join('a')) 71 | 72 | assertStream([new Array(HIGH).join('a')], strs('base64', 'base64'), new Array(HIGH - 1).join('a') + 'Y=') 73 | assertStream([new Array(HIGH + 1).join('a')], strs('base64', 'base64'), new Array(HIGH + 1).join('a')) 74 | assertStream([new Array(HIGH + 2).join('a')], strs('base64', 'base64'), new Array(HIGH + 1).join('a')) 75 | assertStream([new Array(HIGH + 3).join('a')], strs('base64', 'base64'), new Array(HIGH + 2).join('a') + 'Q==') 76 | assertStream([new Array(HIGH + 4).join('a')], strs('base64', 'base64'), new Array(HIGH + 3).join('a') + 'Y=') 77 | assertStream([new Array(HIGH + 5).join('a')], strs('base64', 'base64'), new Array(HIGH + 5).join('a')) 78 | assertStream([new Array(HIGH).join('a'), 'a'], strs('base64', 'base64'), new Array(HIGH + 1).join('a')) 79 | assertStream([new Array(HIGH + 1).join('a'), 'a'], strs('base64', 'base64'), new Array(HIGH + 1).join('a')) 80 | assertStream([new Array(HIGH + 2).join('a'), 'a'], strs('base64', 'base64'), new Array(HIGH + 2).join('a') + 'Q==') 81 | assertStream([new Array(HIGH + 3).join('a'), 'a'], strs('base64', 'base64'), new Array(HIGH + 3).join('a') + 'Y=') 82 | assertStream([new Array(HIGH + 4).join('a'), 'a'], strs('base64', 'base64'), new Array(HIGH + 5).join('a')) 83 | 84 | assertStream(['a'], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], 'a') 85 | assertStream(['a', 'a'], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], 'aa') 86 | assertStream([new Array(HIGH).join('a')], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], new Array(HIGH).join('a')) 87 | assertStream([new Array(HIGH + 1).join('a')], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], new Array(HIGH + 1).join('a')) 88 | assertStream([new Array(HIGH + 2).join('a')], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], new Array(HIGH + 2).join('a')) 89 | assertStream([new Array(HIGH).join('a'), 'a'], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], new Array(HIGH + 1).join('a')) 90 | assertStream([new Array(HIGH + 1).join('a'), 'a'], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], new Array(HIGH + 2).join('a')) 91 | assertStream([new Array(HIGH + 2).join('a'), 'a'], [strs('utf8', 'hex'), strs('hex', 'base64'), strs('base64', 'utf8')], new Array(HIGH + 3).join('a')) 92 | --------------------------------------------------------------------------------