├── .gitignore ├── LICENSE ├── bench.js ├── cli.js ├── collaborators.md ├── index.js ├── package.json ├── readme.md └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Max Ogden 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 9 | -------------------------------------------------------------------------------- /bench.js: -------------------------------------------------------------------------------- 1 | var csv = require('./') 2 | 3 | var c = csv() 4 | var obj = { 5 | hello: 'world', 6 | hej: 'med dig', 7 | lol: 'lulz', 8 | nu: 'yolo' 9 | } 10 | 11 | var i = 0 12 | 13 | c.on('data', function(data) { 14 | i++ 15 | }) 16 | 17 | c.on('end', function() { 18 | console.log(i+' rows, time: '+(Date.now() - now)) 19 | }) 20 | 21 | var now = Date.now() 22 | 23 | for (var i = 0; i < 2000000; i++) c.write(obj) 24 | c.end() 25 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var ArgumentParser = require('argparse').ArgumentParser 3 | var csv = require('./') 4 | var ndj = require('ndjson') 5 | var packageInfo = require('./package') 6 | 7 | argparser = new ArgumentParser({ 8 | addHelp: true, 9 | description: packageInfo.description + ' JSON is read from STDIN, formatted' + 10 | ' to CSV, and written to STDOUT.', 11 | version: packageInfo.version 12 | }) 13 | 14 | argparser.addArgument(['--separator'], { 15 | help: "The separator character to use. Defaults to ','.", 16 | defaultValue: ',' 17 | }) 18 | argparser.addArgument(['--newline'], { 19 | help: "The newline character to use. Defaults to $'\\n'.", 20 | defaultValue: '\n' 21 | }) 22 | argparser.addArgument(['--headers'], { 23 | nargs: '+', 24 | help: 'The list of headers to use. If omitted, the keys of the first row ' + 25 | 'written to STDIN will be used', 26 | }) 27 | argparser.addArgument(['--no-send-headers'], { 28 | action: 'storeFalse', 29 | help: "Don't print the header row.", 30 | defaultValue: true, 31 | dest: 'sendHeaders' 32 | }) 33 | 34 | args = argparser.parseArgs() 35 | 36 | process.stdin 37 | .pipe(ndj.parse()) 38 | .pipe(csv(args)) 39 | .pipe(process.stdout) 40 | -------------------------------------------------------------------------------- /collaborators.md: -------------------------------------------------------------------------------- 1 | ## Collaborators 2 | 3 | csv-write-stream is only possible due to the excellent work of the following collaborators: 4 | 5 | 6 | 7 | 8 | 9 |
maxogdenGitHub/maxogden
mafintoshGitHub/mafintosh
finnpGitHub/finnp
slang800GitHub/slang800
10 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var stream = require('stream') 2 | var util = require('util') 3 | var gen = require('generate-object-property') 4 | 5 | var CsvWriteStream = function(opts) { 6 | if (!opts) opts = {} 7 | stream.Transform.call(this, {objectMode:true, highWaterMark:16}) 8 | 9 | this.sendHeaders = opts.sendHeaders !== false 10 | this.headers = opts.headers || null 11 | this.separator = opts.separator || opts.seperator || ',' 12 | this.newline = opts.newline || '\n' 13 | 14 | this._objRow = null 15 | this._arrRow = null 16 | this._first = true 17 | this._destroyed = false 18 | } 19 | 20 | util.inherits(CsvWriteStream, stream.Transform) 21 | 22 | CsvWriteStream.prototype._compile = function(headers) { 23 | var newline = this.newline 24 | var sep = this.separator 25 | var str = 'function toRow(obj) {\n' 26 | 27 | if (!headers.length) str += '""' 28 | 29 | headers = headers.map(function(prop, i) { 30 | str += 'var a'+i+' = '+prop+' == null ? "" : '+prop+'\n' 31 | return 'a'+i 32 | }) 33 | 34 | for (var i = 0; i < headers.length; i += 500) { // do not overflowi the callstack on lots of cols 35 | var part = headers.length < 500 ? headers : headers.slice(i, i + 500) 36 | str += i ? 'result += "'+sep+'" + ' : 'var result = ' 37 | part.forEach(function(prop, j) { 38 | str += (j ? '+"'+sep+'"+' : '') + '(/['+sep+'\\r\\n"]/.test('+prop+') ? esc('+prop+'+"") : '+prop+')' 39 | }) 40 | str += '\n' 41 | } 42 | 43 | str += 'return result +'+JSON.stringify(newline)+'\n}' 44 | 45 | return new Function('esc', 'return '+str)(esc) 46 | } 47 | 48 | CsvWriteStream.prototype._transform = function(row, enc, cb) { 49 | var isArray = Array.isArray(row) 50 | 51 | if (!isArray && !this.headers) this.headers = Object.keys(row) 52 | 53 | if (this._first && this.headers) { 54 | this._first = false 55 | 56 | var objProps = [] 57 | var arrProps = [] 58 | var heads = [] 59 | 60 | for (var i = 0; i < this.headers.length; i++) { 61 | arrProps.push('obj['+i+']') 62 | objProps.push(gen('obj', this.headers[i])) 63 | } 64 | 65 | this._objRow = this._compile(objProps) 66 | this._arrRow = this._compile(arrProps) 67 | 68 | if (this.sendHeaders) this.push(this._arrRow(this.headers)) 69 | } 70 | 71 | if (isArray) { 72 | if (!this.headers) return cb(new Error('no headers specified')) 73 | this.push(this._arrRow(row)) 74 | } else { 75 | this.push(this._objRow(row)) 76 | } 77 | 78 | cb() 79 | } 80 | 81 | CsvWriteStream.prototype.destroy = function (err) { 82 | if (this._destroyed) return 83 | this._destroyed = true 84 | 85 | var self = this 86 | 87 | process.nextTick(function () { 88 | if (err) self.emit('error', err) 89 | self.emit('close') 90 | }) 91 | } 92 | 93 | module.exports = function(opts) { 94 | return new CsvWriteStream(opts) 95 | } 96 | 97 | function esc(cell) { 98 | return '"'+cell.replace(/"/g, '""')+'"' 99 | } 100 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "csv-write-stream", 3 | "description": "A CSV encoder stream that produces properly escaped CSVs.", 4 | "version": "2.0.0", 5 | "author": "max ogden", 6 | "bin": { 7 | "csv-write": "cli.js" 8 | }, 9 | "bugs": { 10 | "url": "https://github.com/maxogden/csv-write-stream/issues" 11 | }, 12 | "dependencies": { 13 | "argparse": "^1.0.7", 14 | "generate-object-property": "^1.0.0", 15 | "ndjson": "^1.3.0" 16 | }, 17 | "devDependencies": { 18 | "concat-stream": "~1.4.1", 19 | "tape": "~2.3.2" 20 | }, 21 | "homepage": "https://github.com/maxogden/csv-write-stream", 22 | "license": "BSD-2-Clause", 23 | "main": "index.js", 24 | "repository": { 25 | "type": "git", 26 | "url": "git@github.com:maxogden/csv-write-stream.git" 27 | }, 28 | "scripts": { 29 | "test": "node test.js" 30 | }, 31 | "testling": { 32 | "files": "test.js", 33 | "browsers": [ 34 | "ie/8..latest", 35 | "firefox/17..latest", 36 | "firefox/nightly", 37 | "chrome/22..latest", 38 | "chrome/canary", 39 | "opera/12..latest", 40 | "opera/next", 41 | "safari/5.1..latest", 42 | "ipad/6.0..latest", 43 | "iphone/6.0..latest", 44 | "android-browser/4.2..latest" 45 | ] 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # csv-write-stream 2 | 3 | A CSV encoder stream that produces properly escaped CSVs. 4 | 5 | [![NPM](https://nodei.co/npm/csv-write-stream.png)](https://nodei.co/npm/csv-write-stream/) 6 | 7 | [![browser support](http://ci.testling.com/maxogden/csv-write-stream.png)](http://ci.testling.com/maxogden/csv-write-stream) 8 | 9 | A through stream. Write arrays of strings (or JS objects) and you will receive a properly escaped CSV stream out the other end. 10 | 11 | ## usage 12 | 13 | ### var writer = csvWriter([options]) 14 | 15 | ```javascript 16 | var csvWriter = require('csv-write-stream') 17 | var writer = csvWriter() 18 | ``` 19 | 20 | `writer` is a duplex stream -- you can pipe data to it and it will emit a string for each line of the CSV 21 | 22 | ### default options 23 | 24 | ```javascript 25 | { 26 | separator: ',', 27 | newline: '\n', 28 | headers: undefined, 29 | sendHeaders: true 30 | } 31 | ``` 32 | 33 | `headers` can be an array of strings to use as the header row. if you don't specify a header row the keys of the first row written to the stream will be used as the header row IF the first row is an object (see the test suite for more details). if the `sendHeaders` option is set to false, the headers will be used for ordering the data but will never be written to the stream. 34 | 35 | example of auto headers: 36 | 37 | ```javascript 38 | var writer = csvWriter() 39 | writer.pipe(fs.createWriteStream('out.csv')) 40 | writer.write({hello: "world", foo: "bar", baz: "taco"}) 41 | writer.end() 42 | 43 | // produces: hello,foo,baz\nworld,bar,taco\n 44 | ``` 45 | 46 | example of specifying headers: 47 | 48 | ```javascript 49 | var writer = csvWriter({ headers: ["hello", "foo"]}) 50 | writer.pipe(fs.createWriteStream('out.csv')) 51 | writer.write(['world', 'bar']) 52 | writer.end() 53 | 54 | // produces: hello,foo\nworld,bar\n 55 | ``` 56 | 57 | example of not sending headers: 58 | 59 | ```javascript 60 | var writer = csvWriter({sendHeaders: false}) 61 | writer.pipe(fs.createWriteStream('out.csv')) 62 | writer.write({hello: "world", foo: "bar", baz: "taco"}) 63 | writer.end() 64 | 65 | // produces: world,bar,taco\n 66 | ``` 67 | 68 | see the test suite for more examples 69 | 70 | ## run the test suite 71 | 72 | ```bash 73 | $ npm install 74 | $ npm test 75 | ``` 76 | 77 | ## cli usage 78 | 79 | This module also includes a CLI, which you can pipe [ndjson](http://ndjson.org) to stdin and it will print csv on stdout. You can install it with `npm install -g csv-write-stream`. 80 | 81 | ```bash 82 | $ csv-write --help 83 | usage: csv-write [-h] [-v] [--separator SEPARATOR] [--newline NEWLINE] 84 | [--headers HEADERS [HEADERS ...]] [--no-send-headers] 85 | 86 | 87 | A CSV encoder stream that produces properly escaped CSVs. JSON is read from 88 | STDIN, formatted to CSV, and written to STDOUT. 89 | 90 | Optional arguments: 91 | -h, --help Show this help message and exit. 92 | -v, --version Show program's version number and exit. 93 | --separator SEPARATOR 94 | The separator character to use. Defaults to ','. 95 | --newline NEWLINE The newline character to use. Defaults to $'\n'. 96 | --headers HEADERS [HEADERS ...] 97 | The list of headers to use. If omitted, the keys of 98 | the first row written to STDIN will be used 99 | --no-send-headers Don't print the header row. 100 | ``` 101 | 102 | ```bash 103 | $ cat example.ndjson | csv-write > example.csv 104 | ``` 105 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var test = require('tape') 2 | var csv = require('./') 3 | var concat = require('concat-stream') 4 | 5 | test('encode from basic array', function(t) { 6 | var writer = csv({ headers: ["hello", "foo"]}) 7 | 8 | writer.pipe(concat(function(data) { 9 | t.equal('hello,foo\nworld,bar\n', data.toString()) 10 | t.end() 11 | })) 12 | 13 | writer.write(["world", "bar"]) 14 | writer.end() 15 | }) 16 | 17 | test('encode from array w/ escaped quotes in header row', function(t) { 18 | var writer = csv({ headers: ['why "hello" there', "foo"]}) 19 | 20 | writer.pipe(concat(function(data) { 21 | t.equal('"why ""hello"" there",foo\nworld,bar\n', data.toString()) 22 | t.end() 23 | })) 24 | 25 | writer.write(["world", "bar"]) 26 | writer.end() 27 | }) 28 | 29 | test('encode from array w/ escaped quotes in cells', function(t) { 30 | var writer = csv({ headers: ["hello", "foo"]}) 31 | 32 | writer.pipe(concat(function(data) { 33 | t.equal('hello,foo\nworld,"this is an ""escaped"" cell"\n', data.toString()) 34 | t.end() 35 | })) 36 | 37 | writer.write(["world", 'this is an "escaped" cell']) 38 | writer.end() 39 | }) 40 | 41 | test('encode from array w/ escaped newline in cells', function(t) { 42 | var writer = csv({ headers: ["hello", "foo"]}) 43 | 44 | writer.pipe(concat(function(data) { 45 | t.equal('hello,foo\nworld,"this is a\nmultiline cell"\n', data.toString()) 46 | t.end() 47 | })) 48 | 49 | writer.write(["world", 'this is a\nmultiline cell']) 50 | writer.end() 51 | }) 52 | 53 | test('encode from array w/ escaped comma in cells', function(t) { 54 | var writer = csv({ headers: ["hello", "foo"]}) 55 | 56 | writer.pipe(concat(function(data) { 57 | t.equal('hello,foo\nworld,"this is a cell with, commas, in it"\n', data.toString()) 58 | t.end() 59 | })) 60 | 61 | writer.write(["world", 'this is a cell with, commas, in it']) 62 | writer.end() 63 | }) 64 | 65 | test('encode from object w/ headers specified', function(t) { 66 | var writer = csv({ headers: ["hello", "foo"]}) 67 | 68 | writer.pipe(concat(function(data) { 69 | t.equal('hello,foo\nworld,bar\n', data.toString()) 70 | t.end() 71 | })) 72 | 73 | writer.write({hello: "world", foo: "bar", baz: "taco"}) 74 | writer.end() 75 | }) 76 | 77 | test('encode from object w/ auto headers', function(t) { 78 | var writer = csv() 79 | 80 | writer.pipe(concat(function(data) { 81 | t.equal('hello,foo,baz\nworld,bar,taco\n', data.toString()) 82 | t.end() 83 | })) 84 | 85 | writer.write({hello: "world", foo: "bar", baz: "taco"}) 86 | writer.end() 87 | }) 88 | 89 | test('no headers specified', function(t) { 90 | var writer = csv() 91 | 92 | writer.on('error', function(err) { 93 | t.equal(err.message, 'no headers specified') 94 | t.end() 95 | }) 96 | 97 | writer.write(['foo', 'bar']) 98 | writer.end() 99 | }) 100 | 101 | test('no headers displayed', function(t) { 102 | var writer = csv({sendHeaders: false}) 103 | 104 | writer.pipe(concat(function(data) { 105 | t.equal('world,bar,taco\n', data.toString()) 106 | t.end() 107 | })) 108 | 109 | writer.write({hello: "world", foo: "bar", baz: "taco"}) 110 | writer.end() 111 | }) 112 | 113 | test('serialize falsy values', function (t) { 114 | // see https://github.com/maxogden/csv-write-stream/issues/8#issuecomment-41873534 115 | var writer = csv({ 116 | headers: ['boolean','string','number','null','undefined'], 117 | sendHeaders: false 118 | }) 119 | writer.pipe(concat(function (data) { 120 | t.equal('false,false,0,,\n', data.toString()) 121 | t.end() 122 | })) 123 | 124 | writer.write([false,'false',0,null,undefined]) 125 | writer.end() 126 | }) 127 | 128 | test('handle objects and arrays', function (t) { 129 | var writer = csv({sendHeaders: false}) 130 | 131 | writer.pipe(concat(function (data) { 132 | t.equal(data, '1,"1,2,3",[object Object]\n') 133 | t.end() 134 | })) 135 | 136 | writer.write({a: 1, b: [1,2,3], c: {d: 1}}) 137 | writer.end() 138 | 139 | }) 140 | 141 | test('destroy with error', function (t) { 142 | var writer = csv({sendHeaders: false}) 143 | 144 | t.plan(2) 145 | 146 | writer.pipe(concat(function (data) { 147 | t.equal(data, '1,2\n', 'date received') 148 | })) 149 | 150 | writer.on('error', function (err) { 151 | writer.end() 152 | t.equal(err.message, 'error') 153 | }) 154 | 155 | writer.write({a: 1, b : 2}) 156 | writer.destroy(new Error('error')) 157 | 158 | }) 159 | 160 | test('lots of cols', function (t) { 161 | var writer = csv() 162 | var obj = {} 163 | 164 | writer.pipe(concat(function (data) { 165 | t.equal(data, Object.keys(obj).join(',') + '\n' + Object.keys(obj).join(',') + '\n') 166 | t.end() 167 | })) 168 | 169 | for (var i = 0; i < 5000; i++) obj[i] = '' + i 170 | writer.write(obj) 171 | writer.end() 172 | }) 173 | --------------------------------------------------------------------------------