├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── example.js ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # peek-stream 2 | 3 | Transform stream that lets you peek the first line before deciding how to parse it 4 | 5 | ``` 6 | npm install peek-stream 7 | ``` 8 | 9 | [![build status](http://img.shields.io/travis/mafintosh/peek-stream.svg?style=flat)](http://travis-ci.org/mafintosh/peek-stream) 10 | ![dat](http://img.shields.io/badge/Development%20sponsored%20by-dat-green.svg?style=flat) 11 | 12 | ## Usage 13 | 14 | ``` js 15 | var peek = require('peek-stream') 16 | var ldjson = require('ldjson-stream') 17 | var csv = require('csv-parser') 18 | 19 | var isCSV = function(data) { 20 | return data.toString().indexOf(',') > -1 21 | } 22 | 23 | var isJSON = function(data) { 24 | try { 25 | JSON.parse(data) 26 | return true 27 | } catch (err) { 28 | return false 29 | } 30 | } 31 | 32 | // call parser to create a new parser 33 | var parser = function() { 34 | return peek(function(data, swap) { 35 | // maybe it is JSON? 36 | if (isJSON(data)) return swap(null, ldjson()) 37 | 38 | // maybe it is CSV? 39 | if (isCSV(data)) return swap(null, csv()) 40 | 41 | // we do not know - bail 42 | swap(new Error('No parser available')) 43 | }) 44 | } 45 | ``` 46 | 47 | The above parser will be able to parse both line delimited JSON and CSV 48 | 49 | ``` js 50 | var parse = parser() 51 | 52 | parse.write('{"hello":"world"}\n{"hello":"another"}\n') 53 | parse.on('data', function(data) { 54 | console.log(data) // prints {hello:'world'} and {hello:'another'} 55 | }) 56 | ``` 57 | 58 | Or 59 | 60 | ``` js 61 | var parse = parser() 62 | 63 | parse.write('test,header\nvalue-1,value-2\n') 64 | parse.on('data', function(data) { 65 | console.log(data) // prints {test:'value-1', header:'value-2'} 66 | }) 67 | ``` 68 | 69 | Per default `data` is the first line (or the first `65535` bytes if no newline is found). 70 | To change the max buffer pass an options map to the constructor 71 | 72 | ``` js 73 | var parse = peek({ 74 | maxBuffer: 10000 75 | }, function(data, swap) { 76 | ... 77 | }) 78 | ``` 79 | 80 | If you want to emit an error if no newline is found set `strict: true` as well. 81 | 82 | 83 | ## License 84 | 85 | MIT -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var peek = require('peek-stream') 2 | var ldjson = require('ldjson-stream') 3 | var csv = require('csv-parser') 4 | 5 | var isCSV = function(data) { 6 | return data.toString().indexOf(',') > -1 7 | } 8 | 9 | var isJSON = function(data) { 10 | try { 11 | JSON.parse(data) 12 | return true 13 | } catch (err) { 14 | return false 15 | } 16 | } 17 | 18 | var parser = function() { 19 | return peek(function(data, swap) { 20 | // maybe it is JSON? 21 | if (isJSON(data)) return swap(null, ldjson()) 22 | 23 | // maybe it is CSV? 24 | if (isCSV(data)) return swap(null, csv()) 25 | 26 | // we do not know - bail 27 | swap(new Error('No parser available')) 28 | }) 29 | } 30 | 31 | var parse = parser() 32 | 33 | parse.write('{"hello":"world"}\n{"hello":"another"}\n') 34 | parse.on('data', function(data) { 35 | console.log('from ldj:', data) 36 | }) 37 | 38 | var parse = parser() 39 | 40 | parse.write('test,header\nvalue-1,value-2\n') 41 | parse.on('data', function(data) { 42 | console.log('from csv:', data) 43 | }) 44 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var duplexify = require('duplexify') 2 | var through = require('through2') 3 | var bufferFrom = require('buffer-from') 4 | 5 | var noop = function() {} 6 | 7 | var isObject = function(data) { 8 | return !Buffer.isBuffer(data) && typeof data !== 'string' 9 | } 10 | 11 | var peek = function(opts, onpeek) { 12 | if (typeof opts === 'number') opts = {maxBuffer:opts} 13 | if (typeof opts === 'function') return peek(null, opts) 14 | if (!opts) opts = {} 15 | 16 | var maxBuffer = typeof opts.maxBuffer === 'number' ? opts.maxBuffer : 65535 17 | var strict = opts.strict 18 | var newline = opts.newline !== false 19 | 20 | var buffer = [] 21 | var bufferSize = 0 22 | var dup = duplexify.obj() 23 | 24 | var peeker = through.obj({highWaterMark:1}, function(data, enc, cb) { 25 | if (isObject(data)) return ready(data, null, cb) 26 | if (!Buffer.isBuffer(data)) data = bufferFrom(data) 27 | 28 | if (newline) { 29 | var nl = Array.prototype.indexOf.call(data, 10) 30 | if (nl > 0 && data[nl-1] === 13) nl-- 31 | 32 | if (nl > -1) { 33 | buffer.push(data.slice(0, nl)) 34 | return ready(Buffer.concat(buffer), data.slice(nl), cb) 35 | } 36 | } 37 | 38 | buffer.push(data) 39 | bufferSize += data.length 40 | 41 | if (bufferSize < maxBuffer) return cb() 42 | if (strict) return cb(new Error('No newline found')) 43 | ready(Buffer.concat(buffer), null, cb) 44 | }) 45 | 46 | var onpreend = function() { 47 | if (strict) return dup.destroy(new Error('No newline found')) 48 | dup.cork() 49 | ready(Buffer.concat(buffer), null, function(err) { 50 | if (err) return dup.destroy(err) 51 | dup.uncork() 52 | }) 53 | } 54 | 55 | var ready = function(data, overflow, cb) { 56 | dup.removeListener('preend', onpreend) 57 | onpeek(data, function(err, parser) { 58 | if (err) return cb(err) 59 | 60 | dup.setWritable(parser) 61 | dup.setReadable(parser) 62 | 63 | if (data) parser.write(data) 64 | if (overflow) parser.write(overflow) 65 | 66 | overflow = buffer = peeker = null // free the data 67 | cb() 68 | }) 69 | } 70 | 71 | dup.on('preend', onpreend) 72 | dup.setWritable(peeker) 73 | 74 | return dup 75 | } 76 | 77 | module.exports = peek 78 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "peek-stream", 3 | "version": "1.1.3", 4 | "description": "Transform stream that lets you peek the first line before deciding how to parse it", 5 | "main": "index.js", 6 | "dependencies": { 7 | "buffer-from": "^1.0.0", 8 | "duplexify": "^3.5.0", 9 | "through2": "^2.0.3" 10 | }, 11 | "devDependencies": { 12 | "concat-stream": "^1.6.0", 13 | "tape": "^4.6.3" 14 | }, 15 | "scripts": { 16 | "test": "tape test.js" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "git://github.com/mafintosh/peek-stream" 21 | }, 22 | "keywords": [ 23 | "peek", 24 | "stream", 25 | "parse", 26 | "swap" 27 | ], 28 | "author": "Mathias Buus", 29 | "license": "MIT", 30 | "bugs": { 31 | "url": "https://github.com/mafintosh/peek-stream/issues" 32 | }, 33 | "homepage": "https://github.com/mafintosh/peek-stream" 34 | } 35 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var peek = require('./') 2 | var tape = require('tape') 3 | var concat = require('concat-stream') 4 | var through = require('through2') 5 | 6 | var uppercase = function(data, enc, cb) { 7 | cb(null, data.toString().toUpperCase()) 8 | } 9 | 10 | tape('swap to uppercase', function(t) { 11 | var p = peek(function(data, swap) { 12 | swap(null, through(uppercase)) 13 | }) 14 | 15 | p.pipe(concat(function(data) { 16 | t.same(data.toString(), 'HELLO\nWORLD\n') 17 | t.end() 18 | })) 19 | 20 | p.write('hello\n') 21 | p.write('world\n') 22 | p.end() 23 | }) 24 | 25 | tape('swap to uppercase no newline', function(t) { 26 | var p = peek(function(data, swap) { 27 | swap(null, through(uppercase)) 28 | }) 29 | 30 | p.pipe(concat(function(data) { 31 | t.same(data.toString(), 'HELLOWORLD') 32 | t.end() 33 | })) 34 | 35 | p.write('hello') 36 | p.write('world') 37 | p.end() 38 | }) 39 | 40 | tape('swap to uppercase async', function(t) { 41 | var p = peek(function(data, swap) { 42 | setTimeout(function() { 43 | swap(null, through(uppercase)) 44 | }, 100) 45 | }) 46 | 47 | p.pipe(concat(function(data) { 48 | t.same(data.toString(), 'HELLO\nWORLD\n') 49 | t.end() 50 | })) 51 | 52 | p.write('hello\n') 53 | p.write('world\n') 54 | p.end() 55 | }) 56 | 57 | tape('swap to error', function(t) { 58 | var p = peek(function(data, swap) { 59 | swap(new Error('nogo')) 60 | }) 61 | 62 | p.on('error', function(err) { 63 | t.ok(err) 64 | t.same(err.message, 'nogo') 65 | t.end() 66 | }) 67 | 68 | p.write('hello\n') 69 | p.write('world\n') 70 | p.end() 71 | }) 72 | 73 | tape('swap to error async', function(t) { 74 | var p = peek(function(data, swap) { 75 | setTimeout(function() { 76 | swap(new Error('nogo')) 77 | }, 100) 78 | }) 79 | 80 | p.on('error', function(err) { 81 | t.ok(err) 82 | t.same(err.message, 'nogo') 83 | t.end() 84 | }) 85 | 86 | p.write('hello\n') 87 | p.write('world\n') 88 | p.end() 89 | }) --------------------------------------------------------------------------------