├── .editorconfig ├── .eslintrc.json ├── .gitattributes ├── .github ├── CONTRIBUTING.md ├── ISSUE_TEMPLATE.md ├── ISSUE_TEMPLATE │ ├── BUG.md │ ├── DOCS.md │ ├── FEATURE.md │ └── MODIFICATION.md ├── PULL_REQUEST_TEMPLATE.md └── labels.json ├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── bin ├── bench └── csv-parser ├── commitlint.config.js ├── examples └── transform.js ├── index.d.ts ├── index.js ├── index.test-d.ts ├── package-lock.json ├── package.json └── test ├── byteOffset.test.js ├── escape.test.js ├── fixtures ├── backtick.csv ├── bad-data.csv ├── basic.csv ├── bench │ ├── latin.csv │ ├── mac-newlines.csv │ ├── utf16-big.csv │ ├── utf16.csv │ └── utf8.csv ├── comma-in-quote.csv ├── comment.csv ├── empty-columns.csv ├── escape-quotes.csv ├── geojson.csv ├── headers.csv ├── large-dataset.csv ├── newlines.csv ├── no-headers.csv ├── option-comment.csv ├── option-escape.csv ├── option-maxRowBytes.csv ├── option-newline.csv ├── option-quote-escape.csv ├── option-quote-many.csv ├── option-quote.csv ├── quotes+newlines.csv ├── strict+skipLines.csv ├── strict-false-less-columns.csv ├── strict-false-more-columns.csv └── strict.csv ├── headers.test.js ├── helpers └── helper.js ├── issues.test.js ├── mapHeaders.test.js ├── mapValues.test.js ├── maxRowBytes.test.js ├── newline.test.js ├── quote.test.js ├── skipComments.test.js ├── skipLines.test.js ├── snapshots ├── byteOffset.test.js.md ├── byteOffset.test.js.snap ├── escape.test.js.md ├── escape.test.js.snap ├── headers.test.js.md ├── headers.test.js.snap ├── issues.test.js.md ├── issues.test.js.snap ├── mapHeaders.test.js.md ├── mapHeaders.test.js.snap ├── mapValues.test.js.md ├── mapValues.test.js.snap ├── newline.test.js.md ├── newline.test.js.snap ├── quote.test.js.md ├── quote.test.js.snap ├── skipComments.test.js.md ├── skipComments.test.js.snap ├── strict.test.js.md ├── strict.test.js.snap ├── strictNo.test.js.md ├── strictNo.test.js.snap ├── test.js.md └── test.js.snap ├── strict.test.js ├── strictNo.test.js └── test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | end_of_line = lf 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | 12 | [*.md] 13 | insert_final_newline = true 14 | trim_trailing_whitespace = false -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "standard", 3 | "plugins": ["import"], 4 | "rules": { 5 | "no-var": "error", 6 | "object-shorthand": ["error", "always", { "ignoreConstructors": false, "avoidQuotes": true }], 7 | "prefer-arrow-callback": ["warn", { "allowNamedFunctions": false, "allowUnboundThis": true }], 8 | "prefer-const": ["error", { "destructuring": "any", "ignoreReadBeforeAssign": true }], 9 | "prefer-destructuring": ["warn", { "array": true, "object": true }, { "enforceForRenamedProperties": false }], 10 | "prefer-numeric-literals": "error", 11 | "prefer-rest-params": "error", 12 | "prefer-spread": "error", 13 | "prefer-template": "error" 14 | }, 15 | "settings": { 16 | "import/resolver": { 17 | "node": { 18 | "extensions": [".js", ".json"] 19 | } 20 | }, 21 | "import/extensions": [".js"], 22 | "import/core-modules": [], 23 | "import/ignore": ["node_modules", "\\.(coffee|scss|css|less|hbs|svg|json)$"] 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | package-lock.json -diff 2 | * text=auto 3 | bin/* eol=lf -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | We 💛 contributions! The rules for contributing to this org are few: 4 | 5 | 1. Search issues before opening a new one 6 | 1. Lint and run tests locally before submitting a PR 7 | 1. Adhere to the code style the project has chosen 8 | 1. Fill in the required Issue and Pull Request sections 9 | 10 | Unless your contribution _requires_ circumventing the rules below: 11 | 12 | 1. Don't commit `package-lock.json` or add that file to your PR. 13 | 1. Avoid committing Continuous Integration (CI) files, dotfiles, or rule files. 14 | 1. Avoid solutions which require experimental / harmony Node flags on Node versions supported by the project. 15 | 1. Avoid updating dependency versions. 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/BUG.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🐞 Bug Report 3 | about: Something went awry and you'd like to tell us about it. 4 | 5 | --- 6 | 7 | 12 | 13 | * Operating System: 14 | * Node Version: 15 | * NPM Version: 16 | * csv-parser Version: 17 | 18 | ### Expected Behavior 19 | 20 | 21 | ### Actual Behavior 22 | 23 | 24 | ### How Do We Reproduce? 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/DOCS.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 📚 Documentation 3 | about: Are the docs lacking or missing something? Do they need some new 🔥 hotness? Tell us here. 4 | 5 | --- 6 | 7 | 12 | 13 | Documentation Is: 14 | 15 | 16 | 17 | - [ ] Missing 18 | - [ ] Needed 19 | - [ ] Confusing 20 | - [ ] Not Sure? 21 | 22 | ### Please Explain in Detail... 23 | 24 | 25 | ### Your Proposal for Changes 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: ✨ Feature Request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | 12 | 13 | ### Feature Proposal 14 | 15 | 16 | ### Feature Use Case 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/MODIFICATION.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 🔧 Modification Request 3 | about: Would you like something work differently? Have an alternative approach? This is the template for you. 4 | 5 | --- 6 | 7 | 12 | 13 | * Operating System: 14 | * Node Version: 15 | * NPM Version: 16 | * csv-parser Version: 17 | 18 | ### Expected Behavior / Situation 19 | 20 | 21 | ### Actual Behavior / Situation 22 | 23 | 24 | ### Modification Proposal 25 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 8 | 9 | This PR contains: 10 | 11 | - [ ] bugfix 12 | - [ ] feature 13 | - [ ] refactor 14 | - [ ] tests 15 | - [ ] documentation 16 | - [ ] metadata 17 | 18 | ### Breaking Changes? 19 | 20 | - [ ] yes 21 | - [ ] no 22 | 23 | If yes, please describe the breakage. 24 | 25 | ### Please Describe Your Changes 26 | 27 | 32 | -------------------------------------------------------------------------------- /.github/labels.json: -------------------------------------------------------------------------------- 1 | [ 2 | { "name": "💩 template incomplete", "color": "#4E342E" }, 3 | { "name": "💩 template removed", "color": "#4E342E" }, 4 | 5 | { "name": "c¹ ⋅ discussion", "color": "#1976D2" }, 6 | { "name": "c² ⋅ feedback wanted", "color": "#F9A825" }, 7 | { "name": "c³ ⋅ PR welcome", "color": "#1B5E20" }, 8 | { "name": "c⁴ ⋅ need more info", "color": "#6A1B9A" }, 9 | { "name": "c⁵ ⋅ question", "color": "#C2185B" }, 10 | { "name": "c⁶ ⋅ request for comments", "color": "#BBDEFB" }, 11 | 12 | { "name": "p¹ ⋅ electron", "color": "#B2DFDB" }, 13 | { "name": "p² ⋅ linux", "color": "#B2DFDB" }, 14 | { "name": "p³ ⋅ mac", "color": "#B2DFDB" }, 15 | { "name": "p⁴ ⋅ windows", "color": "#B2DFDB" }, 16 | 17 | { "name": "pr¹ 🔧 chore", "color": "#D7CCC8" }, 18 | { "name": "pr² 🔧 docs", "color": "#D7CCC8" }, 19 | { "name": "pr³ 🔧 feature", "color": "#D7CCC8" }, 20 | { "name": "pr⁴ 🔧 fix", "color": "#D7CCC8" }, 21 | { "name": "pr⁵ 🔧 performance", "color": "#D7CCC8" }, 22 | { "name": "pr⁶ 🔧 refactor", "color": "#D7CCC8" }, 23 | { "name": "pr⁷ 🔧 style", "color": "#D7CCC8" }, 24 | { "name": "pr⁸ 🔧 test", "color": "#D7CCC8" }, 25 | 26 | { "name": "s¹ 🔥🔥🔥 critical", "color": "#E53935" }, 27 | { "name": "s² 🔥🔥 important", "color": "#FB8C00" }, 28 | { "name": "s³ 🔥 nice to have", "color": "#FDD835" }, 29 | { "name": "s⁴ 💧 low", "color": "#039BE5" }, 30 | { "name": "s⁵ 💧💧 inconvenient", "color": "#c0e0f7" }, 31 | 32 | { "name": "t¹ 🐞 bug", "color": "#F44336" }, 33 | { "name": "t² 📚 documentation", "color": "#FDD835" }, 34 | { "name": "t³ ✨ enhancement", "color": "#03a9f4" }, 35 | { "name": "t⁴ ✨ feature", "color": "#8bc34A" }, 36 | { "name": "t⁵ ⋅ regression", "color": "#0052cc" }, 37 | { "name": "t⁶ ⋅ todo", "color": "#311B92" }, 38 | { "name": "t⁷ ⋅ waiting on upstream", "color": "#0D47A1" }, 39 | 40 | { "name": "v¹ ⋅ alpha", "color": "#CDDC39" }, 41 | { "name": "v² ⋅ beta", "color": "#FFEB3B" }, 42 | { "name": "v³ ⋅ major", "color": "#FF9800" }, 43 | { "name": "v⁴ ⋅ minor", "color": "#FFC107" }, 44 | { "name": "v⁵ ⋅ next", "color": "#CDDC39" }, 45 | 46 | { "name": "x¹ ⋅ abandoned", "color": "#CFD8DC" }, 47 | { "name": "x² ⋅ duplicate", "color": "#CFD8DC" }, 48 | { "name": "x³ ⋅ hold", "color": "#CFD8DC" }, 49 | { "name": "x⁴ ⋅ in progress", "color": "#4CAF50" }, 50 | { "name": "x⁵ ⋅ invalid", "color": "#CFD8DC" }, 51 | { "name": "x⁶ ⋅ wontfix", "color": "#CFD8DC" } 52 | ] 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .nyc_output 3 | coverage.lcov 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | bin/bench 2 | examples/ 3 | test/ 4 | .travis.yml 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 14 4 | - 12 5 | - 10 6 | script: 7 | - 'npm run lint' 8 | - 'npm run test' 9 | after_success: 10 | - npm run coverage 11 | - bash <(curl -s https://codecov.io/bash) 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [tests]: http://img.shields.io/travis/mafintosh/csv-parser.svg 2 | [tests-url]: http://travis-ci.org/mafintosh/csv-parser 3 | 4 | [cover]: https://codecov.io/gh/mafintosh/csv-parser/branch/master/graph/badge.svg 5 | [cover-url]: https://codecov.io/gh/mafintosh/csv-parser 6 | 7 | [size]: https://packagephobia.now.sh/badge?p=csv-parser 8 | [size-url]: https://packagephobia.now.sh/result?p=csv-parser 9 | 10 | # csv-parser 11 | 12 | [![tests][tests]][tests-url] 13 | [![cover][cover]][cover-url] 14 | [![size][size]][size-url] 15 | 16 | Streaming CSV parser that aims for maximum speed as well as compatibility with 17 | the [csv-spectrum](https://npmjs.org/csv-spectrum) CSV acid test suite. 18 | 19 | `csv-parser` can convert CSV into JSON at at rate of around 90,000 rows per 20 | second. Performance varies with the data used; try `bin/bench.js ` 21 | to benchmark your data. 22 | 23 | `csv-parser` can be used in the browser with [browserify](http://browserify.org/). 24 | 25 | [neat-csv](https://github.com/sindresorhus/neat-csv) can be used if a `Promise` 26 | based interface to `csv-parser` is needed. 27 | 28 | _Note: This module requires Node v8.16.0 or higher._ 29 | 30 | ## Benchmarks 31 | 32 | ⚡️ `csv-parser` is greased-lightning fast 33 | 34 | ```console 35 | → npm run bench 36 | 37 | Filename Rows Parsed Duration 38 | backtick.csv 2 3.5ms 39 | bad-data.csv 3 0.55ms 40 | basic.csv 1 0.26ms 41 | comma-in-quote.csv 1 0.29ms 42 | comment.csv 2 0.40ms 43 | empty-columns.csv 1 0.40ms 44 | escape-quotes.csv 3 0.38ms 45 | geojson.csv 3 0.46ms 46 | large-dataset.csv 7268 73ms 47 | newlines.csv 3 0.35ms 48 | no-headers.csv 3 0.26ms 49 | option-comment.csv 2 0.24ms 50 | option-escape.csv 3 0.25ms 51 | option-maxRowBytes.csv 4577 39ms 52 | option-newline.csv 0 0.47ms 53 | option-quote-escape.csv 3 0.33ms 54 | option-quote-many.csv 3 0.38ms 55 | option-quote.csv 2 0.22ms 56 | quotes+newlines.csv 3 0.20ms 57 | strict.csv 3 0.22ms 58 | latin.csv 2 0.38ms 59 | mac-newlines.csv 2 0.28ms 60 | utf16-big.csv 2 0.33ms 61 | utf16.csv 2 0.26ms 62 | utf8.csv 2 0.24ms 63 | ``` 64 | 65 | ## Install 66 | 67 | Using npm: 68 | 69 | ```console 70 | $ npm install csv-parser 71 | ``` 72 | 73 | Using yarn: 74 | 75 | ```console 76 | $ yarn add csv-parser 77 | ``` 78 | 79 | ## Usage 80 | 81 | To use the module, create a readable stream to a desired CSV file, instantiate 82 | `csv`, and pipe the stream to `csv`. 83 | 84 | Suppose you have a CSV file `data.csv` which contains the data: 85 | 86 | ``` 87 | NAME,AGE 88 | Daffy Duck,24 89 | Bugs Bunny,22 90 | ``` 91 | 92 | It could then be parsed, and results shown like so: 93 | 94 | ``` js 95 | const csv = require('csv-parser') 96 | const fs = require('fs') 97 | const results = []; 98 | 99 | fs.createReadStream('data.csv') 100 | .pipe(csv()) 101 | .on('data', (data) => results.push(data)) 102 | .on('end', () => { 103 | console.log(results); 104 | // [ 105 | // { NAME: 'Daffy Duck', AGE: '24' }, 106 | // { NAME: 'Bugs Bunny', AGE: '22' } 107 | // ] 108 | }); 109 | ``` 110 | 111 | To specify options for `csv`, pass an object argument to the function. For 112 | example: 113 | 114 | ```js 115 | csv({ separator: '\t' }); 116 | ``` 117 | 118 | ## API 119 | 120 | ### csv([options | headers]) 121 | 122 | Returns: `Array[Object]` 123 | 124 | #### options 125 | 126 | Type: `Object` 127 | 128 | As an alternative to passing an `options` object, you may pass an `Array[String]` 129 | which specifies the headers to use. For example: 130 | 131 | ```js 132 | csv(['Name', 'Age']); 133 | ``` 134 | 135 | If you need to specify options _and_ headers, please use the the object notation 136 | with the `headers` property as shown below. 137 | 138 | #### escape 139 | 140 | Type: `String`
141 | Default: `"` 142 | 143 | A single-character string used to specify the character used to escape strings 144 | in a CSV row. 145 | 146 | #### headers 147 | 148 | Type: `Array[String] | Boolean` 149 | 150 | Specifies the headers to use. Headers define the property key for each value in 151 | a CSV row. If no `headers` option is provided, `csv-parser` will use the first 152 | line in a CSV file as the header specification. 153 | 154 | If `false`, specifies that the first row in a data file does _not_ contain 155 | headers, and instructs the parser to use the column index as the key for each column. 156 | Using `headers: false` with the same `data.csv` example from above would yield: 157 | 158 | ``` js 159 | [ 160 | { '0': 'Daffy Duck', '1': 24 }, 161 | { '0': 'Bugs Bunny', '1': 22 } 162 | ] 163 | ``` 164 | 165 | _Note: If using the `headers` for an operation on a file which contains headers on the first line, specify `skipLines: 1` to skip over the row, or the headers row will appear as normal row data. Alternatively, use the `mapHeaders` option to manipulate existing headers in that scenario._ 166 | 167 | #### mapHeaders 168 | 169 | Type: `Function` 170 | 171 | A function that can be used to modify the values of each header. Return a `String` to modify the header. Return `null` to remove the header, and it's column, from the results. 172 | 173 | ```js 174 | csv({ 175 | mapHeaders: ({ header, index }) => header.toLowerCase() 176 | }) 177 | ``` 178 | 179 | ##### Parameters 180 | 181 | **header** _String_ The current column header.
182 | **index** _Number_ The current column index. 183 | 184 | #### mapValues 185 | 186 | Type: `Function` 187 | 188 | A function that can be used to modify the content of each column. The return value will replace the current column content. 189 | 190 | ```js 191 | csv({ 192 | mapValues: ({ header, index, value }) => value.toLowerCase() 193 | }) 194 | ``` 195 | 196 | ##### Parameters 197 | 198 | **header** _String_ The current column header.
199 | **index** _Number_ The current column index.
200 | **value** _String_ The current column value (or content). 201 | 202 | ##### newline 203 | 204 | Type: `String`
205 | Default: `\n` 206 | 207 | Specifies a single-character string to denote the end of a line in a CSV file. 208 | 209 | #### quote 210 | 211 | Type: `String`
212 | Default: `"` 213 | 214 | Specifies a single-character string to denote a quoted string. 215 | 216 | #### raw 217 | 218 | Type: `Boolean`
219 | 220 | If `true`, instructs the parser not to decode UTF-8 strings. 221 | 222 | #### separator 223 | 224 | Type: `String`
225 | Default: `,` 226 | 227 | Specifies a single-character string to use as the column separator for each row. 228 | 229 | #### skipComments 230 | 231 | Type: `Boolean | String`
232 | Default: `false` 233 | 234 | Instructs the parser to ignore lines which represent comments in a CSV file. Since there is no specification that dictates what a CSV comment looks like, comments should be considered non-standard. The "most common" character used to signify a comment in a CSV file is `"#"`. If this option is set to `true`, lines which begin with `#` will be skipped. If a custom character is needed to denote a commented line, this option may be set to a string which represents the leading character(s) signifying a comment line. 235 | 236 | #### skipLines 237 | 238 | Type: `Number`
239 | Default: `0` 240 | 241 | Specifies the number of lines at the beginning of a data file that the parser should 242 | skip over, prior to parsing headers. 243 | 244 | #### maxRowBytes 245 | 246 | Type: `Number`
247 | Default: `Number.MAX_SAFE_INTEGER` 248 | 249 | Maximum number of bytes per row. An error is thrown if a line exeeds this value. The default value is on 8 peta byte. 250 | 251 | #### strict 252 | 253 | Type: `Boolean`
254 | Default: `false` 255 | 256 | If `true`, instructs the parser that the number of columns in each row must match 257 | the number of `headers` specified or throws an exception. 258 | if `false`: the headers are mapped to the column index 259 | less columns: any missing column in the middle will result in a wrong property mapping! 260 | more columns: the aditional columns will create a "_"+index properties - eg. "_10":"value" 261 | 262 | #### outputByteOffset 263 | 264 | Type: `Boolean`
265 | Default: `false` 266 | 267 | If `true`, instructs the parser to emit each row with a `byteOffset` property. 268 | The byteOffset represents the offset in bytes of the beginning of the parsed row in the original stream. 269 | Will change the output format of stream to be `{ byteOffset, row }`. 270 | 271 | ## Events 272 | 273 | The following events are emitted during parsing: 274 | 275 | ### `data` 276 | 277 | Emitted for each row of data parsed with the notable exception of the header 278 | row. Please see [Usage](#Usage) for an example. 279 | 280 | ### `headers` 281 | 282 | Emitted after the header row is parsed. The first parameter of the event 283 | callback is an `Array[String]` containing the header names. 284 | 285 | ```js 286 | fs.createReadStream('data.csv') 287 | .pipe(csv()) 288 | .on('headers', (headers) => { 289 | console.log(`First header: ${headers[0]}`) 290 | }) 291 | ``` 292 | 293 | ### Readable Stream Events 294 | 295 | Events available on Node built-in 296 | [Readable Streams](https://nodejs.org/api/stream.html#stream_class_stream_readable) 297 | are also emitted. The `end` event should be used to detect the end of parsing. 298 | 299 | ## CLI 300 | 301 | This module also provides a CLI which will convert CSV to 302 | [newline-delimited](http://ndjson.org/) JSON. The following CLI flags can be 303 | used to control how input is parsed: 304 | 305 | ``` 306 | Usage: csv-parser [filename?] [options] 307 | 308 | --escape,-e Set the escape character (defaults to quote value) 309 | --headers,-h Explicitly specify csv headers as a comma separated list 310 | --help Show this help 311 | --output,-o Set output file. Defaults to stdout 312 | --quote,-q Set the quote character ('"' by default) 313 | --remove Remove columns from output by header name 314 | --separator,-s Set the separator character ("," by default) 315 | --skipComments,-c Skip CSV comments that begin with '#'. Set a value to change the comment character. 316 | --skipLines,-l Set the number of lines to skip to before parsing headers 317 | --strict Require column length match headers length 318 | --version,-v Print out the installed version 319 | ``` 320 | 321 | For example; to parse a TSV file: 322 | 323 | ``` 324 | cat data.tsv | csv-parser -s $'\t' 325 | ``` 326 | 327 | ## Encoding 328 | 329 | Users may encounter issues with the encoding of a CSV file. Transcoding the 330 | source stream can be done neatly with a modules such as: 331 | - [`iconv-lite`](https://www.npmjs.com/package/iconv-lite) 332 | - [`iconv`](https://www.npmjs.com/package/iconv) 333 | 334 | Or native [`iconv`](http://man7.org/linux/man-pages/man1/iconv.1.html) if part 335 | of a pipeline. 336 | 337 | ## Byte Order Marks 338 | 339 | Some CSV files may be generated with, or contain a leading [Byte Order Mark](https://en.wikipedia.org/wiki/Byte_order_mark#UTF-8). This may cause issues parsing headers and/or data from your file. From Wikipedia: 340 | 341 | >The Unicode Standard permits the BOM in UTF-8, but does not require nor recommend its use. Byte order has no meaning in UTF-8. 342 | 343 | To use this module with a file containing a BOM, please use a module like [strip-bom-stream](https://github.com/sindresorhus/strip-bom-stream) in your pipeline: 344 | 345 | ```js 346 | const fs = require('fs'); 347 | 348 | const csv = require('csv-parser'); 349 | const stripBom = require('strip-bom-stream'); 350 | 351 | fs.createReadStream('data.csv') 352 | .pipe(stripBom()) 353 | .pipe(csv()) 354 | ... 355 | ``` 356 | 357 | When using the CLI, the BOM can be removed by first running: 358 | 359 | ```console 360 | $ sed $'s/\xEF\xBB\xBF//g' data.csv 361 | ``` 362 | 363 | ## Meta 364 | 365 | [CONTRIBUTING](./.github/CONTRIBUTING) 366 | 367 | [LICENSE (MIT)](./LICENSE) 368 | -------------------------------------------------------------------------------- /bin/bench: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | require('loud-rejection')() 3 | 4 | const { createReadStream } = require('fs') 5 | const { basename } = require('path') 6 | const globby = require('globby') 7 | const chalk = require('chalk') 8 | const table = require('text-table') 9 | const timeSpan = require('time-span') 10 | const strip = require('strip-ansi') 11 | 12 | const csv = require('../') 13 | 14 | const run = async () => { 15 | const paths = process.argv[2] || await globby(['test/fixtures/**/*.csv']) 16 | const rows = [] 17 | 18 | for (const path of [].concat(paths)) { 19 | await new Promise((resolve) => { 20 | let rowsParsed = 0 21 | const end = timeSpan() 22 | const data = [] 23 | createReadStream(path) 24 | .pipe(csv()) 25 | .on('data', (line) => { 26 | rowsParsed++ 27 | data.push(line) 28 | }) 29 | .on('finish', () => { 30 | const duration = end().toPrecision(2) 31 | const color = duration <= 10 ? 'green' : (duration > 100 ? 'red' : 'yellow') 32 | const fileName = chalk.blue(basename(path)) 33 | rows.push(['', fileName, rowsParsed, chalk[color](`${duration}ms`)]) 34 | resolve() 35 | }) 36 | }) 37 | } 38 | 39 | rows.unshift(['', 'Filename', 'Rows Parsed', 'Duration'].map(h => chalk.dim.underline(h))) 40 | 41 | const results = table(rows, { 42 | align: [ 'l', 'l', 'r', 'r' ], 43 | stringLength (str) { 44 | return strip(str).length 45 | } 46 | }) 47 | 48 | console.log(`\n${results}`) 49 | } 50 | 51 | run() 52 | -------------------------------------------------------------------------------- /bin/csv-parser: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { EOL } = require('os') 4 | const { Transform } = require('stream'); 5 | const fs = require('fs') 6 | const csv = require('../') 7 | const pkg = require('../package.json') 8 | 9 | function parseArgs () { 10 | let skipComments 11 | let escape = '"' 12 | let headers 13 | let output 14 | let quote = '"' 15 | let skipLines 16 | let separator = ',' 17 | let version 18 | let help 19 | const defaultArgs = [] 20 | 21 | const args = process.argv.slice(2) 22 | for (let i = 0; i < args.length; i++) { 23 | const arg = args[i] 24 | switch (arg) { 25 | case '--skipComments': 26 | case '-c': 27 | skipComments = args[++i] 28 | break 29 | case '--escape': 30 | case '-e': 31 | escape = args[++i] 32 | break 33 | case '--headers': 34 | case '-h': 35 | headers = args[++i] 36 | break 37 | case '--output': 38 | case '-o': 39 | output = args[++i] 40 | break 41 | case '--quote': 42 | case '-q': 43 | quote = args[++i] 44 | break 45 | case '--skipLines': 46 | case '-l': 47 | skipLines = args[++i] 48 | break 49 | case '--separator': 50 | case '-s': 51 | separator = args[++i] 52 | break 53 | case '--version': 54 | case '-v': 55 | version = true 56 | break 57 | case '--help': 58 | help = true 59 | break 60 | default: 61 | defaultArgs.push(arg) 62 | break 63 | } 64 | } 65 | return { 66 | skipComments, 67 | escape, 68 | headers, 69 | output, 70 | quote, 71 | skipLines, 72 | separator, 73 | version, 74 | help, 75 | defaultArgs 76 | } 77 | } 78 | 79 | const argv = parseArgs() 80 | const [filename] = argv.defaultArgs 81 | 82 | if (argv.version) { 83 | console.log(pkg.version) 84 | process.exit(0) 85 | } 86 | 87 | if (argv.help || (process.stdin.isTTY && !filename)) { 88 | console.error(`Usage: csv-parser [filename?] [options] 89 | --escape,-e Set the escape character (defaults to quote value) 90 | --headers,-h Explicitly specify csv headers as a comma separated list 91 | --help Show this help 92 | --output,-o Set output file. Defaults to stdout 93 | --quote,-q Set the quote character ('"' by default) 94 | --remove Remove headers from output 95 | --separator,-s Set the separator character ("," by default) 96 | --skipComments,-c Skip CSV comments that begin with '#'. Set a value to change the comment character. 97 | --skipLines,-l Set the number of lines to skip to before parsing headers 98 | --strict Require column length match headers length 99 | --version,-v Print out the installed version 100 | `) 101 | process.exit(1) 102 | } 103 | 104 | let input 105 | const output = (argv.output && argv.output !== '-') ? fs.createWriteStream(argv.output) : process.stdout 106 | const options = { 107 | separator: argv.separator, 108 | strict: argv.strict, 109 | skipComments: argv.skipComments, 110 | skipLines: argv.skipLines 111 | } 112 | 113 | if (argv.headers) { 114 | options.headers = argv.headers.toString().split(argv.separator) 115 | } 116 | 117 | if (argv.remove) { 118 | const removeHeaders = argv.remove.split(',') 119 | options.mapHeaders = (name, i) => { 120 | return removeHeaders.indexOf(name) === -1 ? name : null 121 | } 122 | } 123 | 124 | if (filename === '-' || !filename) { 125 | input = process.stdin 126 | } else if (fs.existsSync(filename)) { 127 | input = fs.createReadStream(filename) 128 | } else { 129 | console.error(`File: ${filename} does not exist`) 130 | process.exit(2) 131 | } 132 | 133 | const serialize = () => { 134 | return new Transform({ 135 | objectMode: true, 136 | transform(obj, enc, cb) { 137 | cb(null, JSON.stringify(obj) + EOL) 138 | } 139 | }); 140 | } 141 | 142 | input 143 | .pipe(csv(options)) 144 | .pipe(serialize()) 145 | .pipe(output) 146 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { extends: ['@commitlint/config-conventional'] } 2 | -------------------------------------------------------------------------------- /examples/transform.js: -------------------------------------------------------------------------------- 1 | const write = require('csv-write-stream') 2 | const { Transform } = require('stream') 3 | const parse = require('../') 4 | const path = require('path') 5 | const fs = require('fs') 6 | 7 | // Read a file, transform it and send it 8 | // to stdout. Optionally could also write 9 | // to a file instead of stdout with: 10 | // .pipe(fs.createWriteStream('./file')) 11 | fs.createReadStream(path.join(__dirname, '../test/data/dummy.csv')) 12 | .pipe(parse()) 13 | .pipe(new Transform({ objectMode: true, transform })) 14 | .pipe(write()) 15 | .pipe(process.stdout) 16 | 17 | // Prepend all chunks with `value: `. 18 | // @param {Object} chunk 19 | // @param {String} encoding 20 | // @param {Function} callback 21 | function transform (chunk, enc, cb) { 22 | Object.keys(chunk).forEach((k) => { 23 | chunk[k] = `value: ${chunk[k]}` 24 | }) 25 | this.push(chunk) 26 | cb() 27 | } 28 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { Transform } from 'stream'; 3 | 4 | declare namespace csvParser { 5 | type CsvParser = Transform; 6 | 7 | interface Options { 8 | /** 9 | * A single-character string used to specify the character used to escape strings in a CSV row. 10 | * 11 | * @default '"' 12 | */ 13 | readonly escape?: string; 14 | 15 | /** 16 | * Specifies the headers to use. Headers define the property key for each value in a CSV row. If no `headers` option is provided, `csv-parser` will use the first line in a CSV file as the header specification. 17 | * 18 | * If `false`, specifies that the first row in a data file does _not_ contain headers, and instructs the parser to use the row index as the key for each row. 19 | * 20 | * Suppose you have a CSV file `data.csv` which contains the data: 21 | * 22 | * ``` 23 | NAME,AGE 24 | Daffy Duck,24 25 | Bugs Bunny,22 26 | ``` 27 | * Using `headers: false` with the data from `data.csv` would yield: 28 | * ``` 29 | [ 30 | { '0': 'Daffy Duck', '1': 24 }, 31 | { '0': 'Bugs Bunny', '1': 22 } 32 | ] 33 | ``` 34 | */ 35 | readonly headers?: ReadonlyArray | boolean; 36 | 37 | /** 38 | * A function that can be used to modify the values of each header. Return `null` to remove the header, and it's column, from the results. 39 | * 40 | * @example 41 | * 42 | * csv({ 43 | * mapHeaders: ({ header, index }) => header.toLowerCase() 44 | * }); 45 | */ 46 | readonly mapHeaders?: (args: { header: string; index: number }) => string | null; 47 | 48 | /** 49 | * A function that can be used to modify the value of each column value. 50 | * 51 | * @example 52 | * 53 | * csv({ 54 | * mapValues: ({ header, index, value }) => value.toLowerCase() 55 | * }); 56 | */ 57 | readonly mapValues?: (args: { header: string; index: number; value: any }) => any; 58 | 59 | /** 60 | * Specifies a single-character string to denote the end of a line in a CSV file. 61 | * 62 | * @default '\n' 63 | */ 64 | readonly newline?: string; 65 | 66 | /** 67 | * Specifies a single-character string to denote a quoted string. 68 | * 69 | * @default '"' 70 | */ 71 | readonly quote?: string; 72 | 73 | /** 74 | * If `true`, instructs the parser not to decode UTF-8 strings. 75 | */ 76 | readonly raw?: boolean; 77 | 78 | /** 79 | * Specifies a single-character string to use as the column separator for each row. 80 | * 81 | * @default ',' 82 | */ 83 | readonly separator?: string; 84 | 85 | /** 86 | * Instructs the parser to ignore lines which represent comments in a CSV file. Since there is no specification that dictates what a CSV comment looks like, comments should be considered non-standard. The "most common" character used to signify a comment in a CSV file is `"#"`. If this option is set to `true`, lines which begin with `#` will be skipped. If a custom character is needed to denote a commented line, this option may be set to a string which represents the leading character(s) signifying a comment line. 87 | * 88 | * @default false 89 | */ 90 | readonly skipComments?: boolean | string; 91 | 92 | /** 93 | * Specifies the number of lines at the beginning of a data file that the parser should skip over, prior to parsing headers. 94 | * 95 | * @default 0 96 | */ 97 | readonly skipLines?: number; 98 | 99 | /** 100 | * Maximum number of bytes per row. An error is thrown if a line exeeds this value. The default value is on 8 peta byte. 101 | * 102 | * @default Number.MAX_SAFE_INTEGER 103 | */ 104 | readonly maxRowBytes?: number; 105 | 106 | /** 107 | * If `true`, instructs the parser that the number of columns in each row must match the number of `headers` specified. 108 | */ 109 | readonly strict?: boolean; 110 | 111 | /** 112 | * If `true`, instructs the parser to emit each row with a `byteOffset` property. 113 | * The byteOffset represents the offset in bytes of the beginning of the parsed row in the original stream. 114 | * Will change the output format of stream to be `{ byteOffset, row }`. 115 | */ 116 | readonly outputByteOffset?: boolean; 117 | } 118 | } 119 | 120 | /** 121 | * Streaming CSV parser that aims for maximum speed as well as compatibility with the [csv-spectrum](https://npmjs.org/csv-spectrum) CSV acid test suite. 122 | * 123 | * @param optionsOrHeaders - As an alternative to passing an `options` object, you may pass an `Array[String]` which specifies the headers to use. If you need to specify options _and_ headers, please use the the object notation with the `headers` property. 124 | * 125 | * @example 126 | * 127 | * // data.csv: 128 | * // 129 | * // NAME,AGE 130 | * // Daffy Duck,24 131 | * // Bugs Bunny,22 132 | * 133 | * import csv = require('csv-parser'); 134 | * import * as fs from 'fs'; 135 | * 136 | * const results = []; 137 | * 138 | * fs.createReadStream('data.csv') 139 | * .pipe(csv()) 140 | * .on('data', (data) => results.push(data)) 141 | * .on('end', () => { 142 | * console.log(results); 143 | * // [ 144 | * // { NAME: 'Daffy Duck', AGE: '24' }, 145 | * // { NAME: 'Bugs Bunny', AGE: '22' } 146 | * // ] 147 | * }); 148 | */ 149 | declare const csvParser: ( 150 | optionsOrHeaders?: csvParser.Options | ReadonlyArray 151 | ) => csvParser.CsvParser; 152 | 153 | export = csvParser; 154 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { Transform } = require('stream') 2 | 3 | const [cr] = Buffer.from('\r') 4 | const [nl] = Buffer.from('\n') 5 | const defaults = { 6 | escape: '"', 7 | headers: null, 8 | mapHeaders: ({ header }) => header, 9 | mapValues: ({ value }) => value, 10 | newline: '\n', 11 | quote: '"', 12 | raw: false, 13 | separator: ',', 14 | skipComments: false, 15 | skipLines: null, 16 | maxRowBytes: Number.MAX_SAFE_INTEGER, 17 | strict: false, 18 | outputByteOffset: false 19 | } 20 | 21 | class CsvParser extends Transform { 22 | constructor (opts = {}) { 23 | super({ objectMode: true, highWaterMark: 16 }) 24 | 25 | if (Array.isArray(opts)) opts = { headers: opts } 26 | 27 | const options = Object.assign({}, defaults, opts) 28 | 29 | options.customNewline = options.newline !== defaults.newline 30 | 31 | for (const key of ['newline', 'quote', 'separator']) { 32 | if (typeof options[key] !== 'undefined') { 33 | ([options[key]] = Buffer.from(options[key])) 34 | } 35 | } 36 | 37 | // if escape is not defined on the passed options, use the end value of quote 38 | options.escape = (opts || {}).escape ? Buffer.from(options.escape)[0] : options.quote 39 | 40 | this.state = { 41 | empty: options.raw ? Buffer.alloc(0) : '', 42 | escaped: false, 43 | first: true, 44 | lineNumber: 0, 45 | previousEnd: 0, 46 | rowLength: 0, 47 | quoted: false 48 | } 49 | 50 | this._prev = null 51 | 52 | if (options.headers === false) { 53 | // enforce, as the column length check will fail if headers:false 54 | options.strict = false 55 | } 56 | 57 | if (options.headers || options.headers === false) { 58 | this.state.first = false 59 | } 60 | 61 | this.options = options 62 | this.headers = options.headers 63 | this.bytesRead = 0 64 | } 65 | 66 | parseCell (buffer, start, end) { 67 | const { escape, quote } = this.options 68 | // remove quotes from quoted cells 69 | if (buffer[start] === quote && buffer[end - 1] === quote) { 70 | start++ 71 | end-- 72 | } 73 | 74 | let y = start 75 | 76 | for (let i = start; i < end; i++) { 77 | // check for escape characters and skip them 78 | if (buffer[i] === escape && i + 1 < end && buffer[i + 1] === quote) { 79 | i++ 80 | } 81 | 82 | if (y !== i) { 83 | buffer[y] = buffer[i] 84 | } 85 | y++ 86 | } 87 | 88 | return this.parseValue(buffer, start, y) 89 | } 90 | 91 | parseLine (buffer, start, end) { 92 | const { customNewline, escape, mapHeaders, mapValues, quote, separator, skipComments, skipLines } = this.options 93 | 94 | end-- // trim newline 95 | if (!customNewline && buffer.length && buffer[end - 1] === cr) { 96 | end-- 97 | } 98 | 99 | const comma = separator 100 | const cells = [] 101 | let isQuoted = false 102 | let offset = start 103 | 104 | if (skipComments) { 105 | const char = typeof skipComments === 'string' ? skipComments : '#' 106 | if (buffer[start] === Buffer.from(char)[0]) { 107 | return 108 | } 109 | } 110 | 111 | const mapValue = (value) => { 112 | if (this.state.first) { 113 | return value 114 | } 115 | 116 | const index = cells.length 117 | const header = this.headers[index] 118 | 119 | return mapValues({ header, index, value }) 120 | } 121 | 122 | for (let i = start; i < end; i++) { 123 | const isStartingQuote = !isQuoted && buffer[i] === quote 124 | const isEndingQuote = isQuoted && buffer[i] === quote && i + 1 <= end && buffer[i + 1] === comma 125 | const isEscape = isQuoted && buffer[i] === escape && i + 1 < end && buffer[i + 1] === quote 126 | 127 | if (isStartingQuote || isEndingQuote) { 128 | isQuoted = !isQuoted 129 | continue 130 | } else if (isEscape) { 131 | i++ 132 | continue 133 | } 134 | 135 | if (buffer[i] === comma && !isQuoted) { 136 | let value = this.parseCell(buffer, offset, i) 137 | value = mapValue(value) 138 | cells.push(value) 139 | offset = i + 1 140 | } 141 | } 142 | 143 | if (offset < end) { 144 | let value = this.parseCell(buffer, offset, end) 145 | value = mapValue(value) 146 | cells.push(value) 147 | } 148 | 149 | if (buffer[end - 1] === comma) { 150 | cells.push(mapValue(this.state.empty)) 151 | } 152 | 153 | const skip = skipLines && skipLines > this.state.lineNumber 154 | this.state.lineNumber++ 155 | 156 | if (this.state.first && !skip) { 157 | this.state.first = false 158 | this.headers = cells.map((header, index) => mapHeaders({ header, index })) 159 | 160 | this.emit('headers', this.headers) 161 | return 162 | } 163 | 164 | if (!skip && this.options.strict && cells.length !== this.headers.length) { 165 | const e = new RangeError('Row length does not match headers') 166 | this.emit('error', e) 167 | } else { 168 | if (!skip) { 169 | const byteOffset = this.bytesRead - buffer.length + start 170 | this.writeRow(cells, byteOffset) 171 | } 172 | } 173 | } 174 | 175 | parseValue (buffer, start, end) { 176 | if (this.options.raw) { 177 | return buffer.slice(start, end) 178 | } 179 | 180 | return buffer.toString('utf-8', start, end) 181 | } 182 | 183 | writeRow (cells, byteOffset) { 184 | const headers = (this.headers === false) ? cells.map((value, index) => index) : this.headers 185 | 186 | const row = cells.reduce((o, cell, index) => { 187 | const header = headers[index] 188 | if (header === null) return o // skip columns 189 | if (header !== undefined) { 190 | o[header] = cell 191 | } else { 192 | o[`_${index}`] = cell 193 | } 194 | return o 195 | }, {}) 196 | 197 | if (this.options.outputByteOffset) { 198 | this.push({ row, byteOffset }) 199 | } else { 200 | this.push(row) 201 | } 202 | } 203 | 204 | _flush (cb) { 205 | if (this.state.escaped || !this._prev) return cb() 206 | this.parseLine(this._prev, this.state.previousEnd, this._prev.length + 1) // plus since online -1s 207 | cb() 208 | } 209 | 210 | _transform (data, enc, cb) { 211 | if (typeof data === 'string') { 212 | data = Buffer.from(data) 213 | } 214 | 215 | const { escape, quote } = this.options 216 | let start = 0 217 | let buffer = data 218 | this.bytesRead += data.byteLength 219 | 220 | if (this._prev) { 221 | start = this._prev.length 222 | buffer = Buffer.concat([this._prev, data]) 223 | this._prev = null 224 | } 225 | 226 | const bufferLength = buffer.length 227 | 228 | for (let i = start; i < bufferLength; i++) { 229 | const chr = buffer[i] 230 | const nextChr = i + 1 < bufferLength ? buffer[i + 1] : null 231 | 232 | this.state.rowLength++ 233 | if (this.state.rowLength > this.options.maxRowBytes) { 234 | return cb(new Error('Row exceeds the maximum size')) 235 | } 236 | 237 | if (!this.state.escaped && chr === escape && nextChr === quote && i !== start) { 238 | this.state.escaped = true 239 | continue 240 | } else if (chr === quote) { 241 | if (this.state.escaped) { 242 | this.state.escaped = false 243 | // non-escaped quote (quoting the cell) 244 | } else { 245 | this.state.quoted = !this.state.quoted 246 | } 247 | continue 248 | } 249 | 250 | if (!this.state.quoted) { 251 | if (this.state.first && !this.options.customNewline) { 252 | if (chr === nl) { 253 | this.options.newline = nl 254 | } else if (chr === cr) { 255 | if (nextChr !== nl) { 256 | this.options.newline = cr 257 | } 258 | } 259 | } 260 | 261 | if (chr === this.options.newline) { 262 | this.parseLine(buffer, this.state.previousEnd, i + 1) 263 | this.state.previousEnd = i + 1 264 | this.state.rowLength = 0 265 | } 266 | } 267 | } 268 | 269 | if (this.state.previousEnd === bufferLength) { 270 | this.state.previousEnd = 0 271 | return cb() 272 | } 273 | 274 | if (bufferLength - this.state.previousEnd < data.length) { 275 | this._prev = data 276 | this.state.previousEnd -= (bufferLength - data.length) 277 | return cb() 278 | } 279 | 280 | this._prev = buffer 281 | cb() 282 | } 283 | } 284 | 285 | module.exports = (opts) => new CsvParser(opts) 286 | -------------------------------------------------------------------------------- /index.test-d.ts: -------------------------------------------------------------------------------- 1 | import { expectType } from 'tsd'; 2 | import csvParser = require('.'); 3 | 4 | const options: csvParser.Options = {}; 5 | 6 | expectType(csvParser(['Name', 'Age'])); 7 | expectType(csvParser({ escape: '"' })); 8 | expectType(csvParser({ headers: ['Name', 'Age'] })); 9 | expectType(csvParser({ headers: false })); 10 | expectType( 11 | csvParser({ 12 | mapHeaders: ({ header, index }) => { 13 | expectType(header); 14 | expectType(index); 15 | return header.toLowerCase(); 16 | }, 17 | }) 18 | ); 19 | expectType(csvParser({ mapHeaders: ({ header, index }) => null })); 20 | expectType( 21 | csvParser({ 22 | mapValues: ({ header, index, value }) => { 23 | expectType(header); 24 | expectType(index); 25 | expectType(value); 26 | 27 | return value.toLowerCase(); 28 | }, 29 | }) 30 | ); 31 | expectType(csvParser({ mapValues: ({ header, index, value }) => null })); 32 | expectType(csvParser({ newline: '\n' })); 33 | expectType(csvParser({ quote: '"' })); 34 | expectType(csvParser({ raw: true })); 35 | expectType(csvParser({ separator: ',' })); 36 | expectType(csvParser({ skipComments: true })); 37 | expectType(csvParser({ skipComments: '#' })); 38 | expectType(csvParser({ skipLines: 1 })); 39 | expectType(csvParser({ maxRowBytes: 1 })); 40 | expectType(csvParser({ strict: true })); 41 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "csv-parser", 3 | "version": "3.2.0", 4 | "description": "Streaming CSV parser that aims for maximum speed as well as compatibility with the csv-spectrum test suite", 5 | "license": "MIT", 6 | "repository": "mafintosh/csv-parser", 7 | "author": "mafintosh", 8 | "maintainers": [ 9 | "Andrew Powell " 10 | ], 11 | "homepage": "https://github.com/mafintosh/csv-parser", 12 | "bugs": "https://github.com/mafintosh/csv-parser/issues", 13 | "bin": { 14 | "csv-parser": "./bin/csv-parser" 15 | }, 16 | "main": "index.js", 17 | "files": [ 18 | "bin/csv-parser", 19 | "index.js", 20 | "index.d.ts" 21 | ], 22 | "engines": { 23 | "node": ">= 10" 24 | }, 25 | "scripts": { 26 | "bench": "bin/bench", 27 | "commitlint": "commitlint", 28 | "coverage": "nyc npm run test && nyc report --reporter=text-lcov > coverage.lcov", 29 | "lint": "eslint .", 30 | "lint-staged": "lint-staged", 31 | "security": "npm audit", 32 | "test": "ava && tsd" 33 | }, 34 | "devDependencies": { 35 | "@commitlint/cli": "^8.2.0", 36 | "@commitlint/config-conventional": "^8.0.0", 37 | "@types/node": "^12.0.0", 38 | "ava": "^3.0.0", 39 | "bops": "^1.0.0", 40 | "chalk": "^2.4.2", 41 | "concat-stream": "^2.0.0", 42 | "csv-spectrum": "^1.0.0", 43 | "eslint": "^6.4.0", 44 | "eslint-config-standard": "^14.1.0", 45 | "eslint-plugin-import": "^2.18.2", 46 | "eslint-plugin-node": "^10.0.0", 47 | "eslint-plugin-promise": "^4.1.1", 48 | "eslint-plugin-standard": "^4.0.0", 49 | "execa": "^2.1.0", 50 | "globby": "^10.0.1", 51 | "husky": "^3.0.0", 52 | "lint-staged": "^9.0.2", 53 | "loud-rejection": "^2.1.0", 54 | "nyc": "^14.1.1", 55 | "pre-commit": "^1.2.2", 56 | "strip-ansi": "^5.2.0", 57 | "text-table": "^0.2.0", 58 | "time-span": "^3.1.0", 59 | "tsd": "^0.8.0" 60 | }, 61 | "directories": { 62 | "example": "examples", 63 | "test": "test" 64 | }, 65 | "keywords": [ 66 | "csv", 67 | "parser", 68 | "fast", 69 | "json" 70 | ], 71 | "ava": { 72 | "files": [ 73 | "!**/fixtures/**", 74 | "!**/helpers/**" 75 | ] 76 | }, 77 | "husky": { 78 | "hooks": { 79 | "commit-msg": "commitlint -e $HUSKY_GIT_PARAMS" 80 | } 81 | }, 82 | "lint-staged": { 83 | "*.js": [ 84 | "eslint --fix", 85 | "git add" 86 | ] 87 | }, 88 | "pre-commit": "lint-staged" 89 | } 90 | -------------------------------------------------------------------------------- /test/byteOffset.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const bops = require('bops') 3 | const csv = require('..') 4 | 5 | const { collect } = require('./helpers/helper') 6 | 7 | test.cb('simple csv with byte offset', (t) => { 8 | const verify = (err, lines) => { 9 | t.false(err, 'no err') 10 | t.snapshot(lines[0], 'first row') 11 | t.is(lines.length, 1, '1 row') 12 | t.end() 13 | } 14 | 15 | collect('basic', { outputByteOffset: true }, verify) 16 | }) 17 | 18 | test.cb('large dataset with byte offset', (t) => { 19 | const verify = (err, lines) => { 20 | t.false(err, 'no err') 21 | t.snapshot(lines[0], 'first row') 22 | t.snapshot(lines[1], 'second row') 23 | t.is(lines.length, 7268, '7268 rows') 24 | t.end() 25 | } 26 | 27 | collect('large-dataset', { outputByteOffset: true }, verify) 28 | }) 29 | 30 | test.cb('multibyte character set with byte offset', (t) => { 31 | const headers = bops.from('a\n') 32 | const cell1 = bops.from('this ʤ is multibyte1\n') 33 | const cell2 = bops.from('this ʤ is multibyte2\n') 34 | const expected1 = 'this ʤ is multibyte1' 35 | const expected2 = 'this ʤ is multibyte2' 36 | const parser = csv({ outputByteOffset: true }) 37 | 38 | parser.write(headers) 39 | parser.write(cell1) 40 | parser.write(cell2) 41 | parser.end() 42 | 43 | const lines = [] 44 | parser.on('data', (data) => { 45 | lines.push(data) 46 | }) 47 | parser.on('end', () => { 48 | t.is(lines.length, 2, '2 rows') 49 | t.is(lines[0].row.a, expected1, 'multibyte character is preserved') 50 | t.is(lines[0].byteOffset, 2, 'byte offset is correct') 51 | t.is(lines[1].row.a, expected2, 'multibyte character is preserved') 52 | t.is(lines[1].byteOffset, 24, 'byte offset is correct') 53 | t.end() 54 | }) 55 | }) 56 | -------------------------------------------------------------------------------- /test/escape.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('headers: false, numeric column names', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.snapshot(lines, 'lines') 9 | t.is(lines.length, 2, '2 rows') 10 | t.end() 11 | } 12 | 13 | collect('basic', { headers: false }, verify) 14 | }) 15 | -------------------------------------------------------------------------------- /test/fixtures/backtick.csv: -------------------------------------------------------------------------------- 1 | pokemon_id`p_desc 2 | 1`Bulbasaur can be seen napping 3 | 2`There is a bud on this 4 | -------------------------------------------------------------------------------- /test/fixtures/bad-data.csv: -------------------------------------------------------------------------------- 1 | ,somejunk, 2 | ,nope, 3 | yes,yup,yeah 4 | ok,ok,ok! 5 | -------------------------------------------------------------------------------- /test/fixtures/basic.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | -------------------------------------------------------------------------------- /test/fixtures/bench/latin.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/fixtures/bench/latin.csv -------------------------------------------------------------------------------- /test/fixtures/bench/mac-newlines.csv: -------------------------------------------------------------------------------- 1 | a,b,c 1,2,3 "Once upon a time",5,6 -------------------------------------------------------------------------------- /test/fixtures/bench/utf16-big.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/fixtures/bench/utf16-big.csv -------------------------------------------------------------------------------- /test/fixtures/bench/utf16.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/fixtures/bench/utf16.csv -------------------------------------------------------------------------------- /test/fixtures/bench/utf8.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5,ʤ -------------------------------------------------------------------------------- /test/fixtures/comma-in-quote.csv: -------------------------------------------------------------------------------- 1 | first,last,address,city,zip 2 | John,Doe,120 any st.,"Anytown, WW",08123 -------------------------------------------------------------------------------- /test/fixtures/comment.csv: -------------------------------------------------------------------------------- 1 | # comment 2 | a,b,c 3 | 1,2,3 4 | -------------------------------------------------------------------------------- /test/fixtures/empty-columns.csv: -------------------------------------------------------------------------------- 1 | 2007-01-01,, 2 | 2007-01-02,, 3 | -------------------------------------------------------------------------------- /test/fixtures/escape-quotes.csv: -------------------------------------------------------------------------------- 1 | a,b 2 | 1,"ha ""ha"" ha" 3 | 2,"""""" 4 | 3,4 5 | -------------------------------------------------------------------------------- /test/fixtures/geojson.csv: -------------------------------------------------------------------------------- 1 | id,prop0,prop1,geojson 2 | ,value0,,"{""type"": ""Point"", ""coordinates"": [102.0, 0.5]}" 3 | ,value0,0.0,"{""type"": ""LineString"", ""coordinates"": [[102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0]]}" 4 | ,value0,{u'this': u'that'},"{""type"": ""Polygon"", ""coordinates"": [[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]]}" 5 | -------------------------------------------------------------------------------- /test/fixtures/headers.csv: -------------------------------------------------------------------------------- 1 | 1,2,3 2 | 4,5,6 3 | 7,8,9 4 | -------------------------------------------------------------------------------- /test/fixtures/newlines.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | "Once upon 4 | a time",5,6 5 | 7,8,9 6 | -------------------------------------------------------------------------------- /test/fixtures/no-headers.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5,6 4 | 7,8,9,10 5 | -------------------------------------------------------------------------------- /test/fixtures/option-comment.csv: -------------------------------------------------------------------------------- 1 | ~ comment 2 | a,b,c 3 | 1,2,3 4 | -------------------------------------------------------------------------------- /test/fixtures/option-escape.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,"some \"escaped\" value",2 3 | 3,"\"\"",4 4 | 5,6,7 5 | -------------------------------------------------------------------------------- /test/fixtures/option-newline.csv: -------------------------------------------------------------------------------- 1 | a,b,cX1,2,3X"X-Men",5,6X7,8,9 -------------------------------------------------------------------------------- /test/fixtures/option-quote-escape.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,'some \'escaped\' value',2 3 | 3,'\'\'',4 4 | 5,6,7 5 | -------------------------------------------------------------------------------- /test/fixtures/option-quote-many.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,'some ''escaped'' value',2 3 | 3,'''''',4 4 | 5,6,7 5 | -------------------------------------------------------------------------------- /test/fixtures/option-quote.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,'some value',2 3 | 3,4,5 4 | -------------------------------------------------------------------------------- /test/fixtures/quotes+newlines.csv: -------------------------------------------------------------------------------- 1 | a,b 2 | 1,"ha 3 | ""ha"" 4 | ha" 5 | 2," 6 | """" 7 | " 8 | 3,4 9 | -------------------------------------------------------------------------------- /test/fixtures/strict+skipLines.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | h1,h2,h3 3 | 1,2,3 4 | 4,5,6 5 | 7,8,9 6 | -------------------------------------------------------------------------------- /test/fixtures/strict-false-less-columns.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5 4 | 6,7,8 5 | -------------------------------------------------------------------------------- /test/fixtures/strict-false-more-columns.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5,6,7 4 | 8,9,10 5 | -------------------------------------------------------------------------------- /test/fixtures/strict.csv: -------------------------------------------------------------------------------- 1 | a,b,c 2 | 1,2,3 3 | 4,5,6 4 | 7,8,9,10 5 | -------------------------------------------------------------------------------- /test/headers.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('custom escape character', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.snapshot(lines[0], 'first row') 9 | t.snapshot(lines[1], 'second row') 10 | t.snapshot(lines[2], 'third row') 11 | t.is(lines.length, 3, '3 rows') 12 | t.end() 13 | } 14 | 15 | collect('option-escape', { escape: '\\' }, verify) 16 | }) 17 | 18 | test.cb('headers: false', (t) => { 19 | const verify = (err, lines) => { 20 | t.false(err, 'no err') 21 | t.snapshot(lines) 22 | t.end() 23 | } 24 | 25 | collect('no-headers', { headers: false }, verify) 26 | }) 27 | 28 | test.cb('headers option', (t) => { 29 | const verify = (err, lines) => { 30 | t.false(err, 'no err') 31 | t.snapshot(lines) 32 | t.end() 33 | } 34 | 35 | collect('headers', { headers: ['a', 'b', 'c'] }, verify) 36 | }) 37 | -------------------------------------------------------------------------------- /test/helpers/helper.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | 4 | const csv = require('../..') 5 | 6 | const read = fs.createReadStream 7 | 8 | // helpers 9 | function fixture (name) { 10 | return path.join(__dirname, '../fixtures', name) 11 | } 12 | 13 | function collect (file, opts, cb) { 14 | if (typeof opts === 'function') { 15 | return collect(file, null, opts) 16 | } 17 | const data = read(fixture(`${file}.csv`)) 18 | const lines = [] 19 | const parser = csv(opts) 20 | data 21 | .pipe(parser) 22 | .on('data', (line) => { 23 | lines.push(line) 24 | }) 25 | .on('error', (err) => { 26 | cb(err, lines) 27 | }) 28 | .on('end', () => { 29 | // eslint-disable-next-line standard/no-callback-literal 30 | cb(false, lines) 31 | }) 32 | return parser 33 | } 34 | 35 | module.exports = { collect, fixture } 36 | -------------------------------------------------------------------------------- /test/issues.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('backtick separator (#105)', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.snapshot(lines, 'lines') 9 | t.is(lines.length, 2, '2 rows') 10 | t.end() 11 | } 12 | 13 | collect('backtick', { separator: '`' }, verify) 14 | }) 15 | 16 | test.cb('strict + skipLines (#136)', (t) => { 17 | const verify = (err, lines) => { 18 | t.false(err, 'no err') 19 | t.snapshot(lines, 'lines') 20 | t.is(lines.length, 3, '4 rows') 21 | t.end() 22 | } 23 | 24 | collect('strict+skipLines', { strict: true, skipLines: 1 }, verify) 25 | }) 26 | -------------------------------------------------------------------------------- /test/mapHeaders.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('rename columns', (t) => { 6 | const headers = { a: 'x', b: 'y', c: 'z' } 7 | const mapHeaders = ({ header, index }) => { 8 | return headers[header] 9 | } 10 | const verify = (err, lines) => { 11 | t.false(err, 'no err') 12 | t.snapshot(lines[0], 'first row') 13 | t.is(lines.length, 1, '1 row') 14 | t.end() 15 | } 16 | 17 | collect('basic', { mapHeaders }, verify) 18 | }) 19 | 20 | test.cb('skip columns a and c', (t) => { 21 | const mapHeaders = ({ header, index }) => { 22 | if (['a', 'c'].indexOf(header) > -1) { 23 | return null 24 | } 25 | return header 26 | } 27 | 28 | const verify = (err, lines) => { 29 | t.false(err, 'no err') 30 | t.snapshot(lines[0], 'first row') 31 | t.is(lines.length, 1, '1 row') 32 | t.end() 33 | } 34 | 35 | collect('basic', { mapHeaders }, verify) 36 | }) 37 | -------------------------------------------------------------------------------- /test/mapValues.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('map values', (t) => { 6 | const headers = [] 7 | const indexes = [] 8 | const mapValues = ({ header, index, value }) => { 9 | headers.push(header) 10 | indexes.push(index) 11 | return parseInt(value, 10) 12 | } 13 | 14 | const verify = (err, lines) => { 15 | t.false(err, 'no err') 16 | t.snapshot(lines[0], 'first row') 17 | t.is(lines.length, 1, '1 row') 18 | t.snapshot(headers, 'headers') 19 | t.snapshot(indexes, 'indexes') 20 | t.end() 21 | } 22 | 23 | collect('basic', { mapValues }, verify) 24 | }) 25 | 26 | test.cb('map last empty value', (t) => { 27 | const mapValues = ({ value }) => { 28 | return value === '' ? null : value 29 | } 30 | 31 | const verify = (err, lines) => { 32 | t.false(err, 'no err') 33 | t.is(lines.length, 2, '2 rows') 34 | t.is(lines[0].name, null, 'name is mapped') 35 | t.is(lines[0].location, null, 'last value mapped') 36 | t.end() 37 | } 38 | 39 | collect('empty-columns', { mapValues, headers: ['date', 'name', 'location'] }, verify) 40 | }) 41 | -------------------------------------------------------------------------------- /test/maxRowBytes.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('maxRowBytes', (t) => { 6 | const verify = (err, lines) => { 7 | t.is(err.message, 'Row exceeds the maximum size', 'strict row size') 8 | t.is(lines.length, 4576, '4576 rows before error') 9 | t.end() 10 | } 11 | 12 | collect('option-maxRowBytes', { maxRowBytes: 200 }, verify) 13 | }) 14 | -------------------------------------------------------------------------------- /test/newline.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('newline', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.snapshot(lines[0], 'first row') 9 | t.snapshot(lines[1], 'second row') 10 | t.snapshot(lines[2], 'third row') 11 | t.is(lines.length, 3, '3 rows') 12 | t.end() 13 | } 14 | 15 | collect('option-newline', { newline: 'X' }, verify) 16 | }) 17 | -------------------------------------------------------------------------------- /test/quote.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('custom quote character', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.snapshot(lines[0], 'first row') 9 | t.snapshot(lines[1], 'second row') 10 | t.is(lines.length, 2, '2 rows') 11 | t.end() 12 | } 13 | 14 | collect('option-quote', { quote: "'" }, verify) 15 | }) 16 | 17 | test.cb('custom quote and escape character', (t) => { 18 | const verify = (err, lines) => { 19 | t.false(err, 'no err') 20 | t.snapshot(lines[0], 'first row') 21 | t.snapshot(lines[1], 'second row') 22 | t.snapshot(lines[2], 'third row') 23 | t.is(lines.length, 3, '3 rows') 24 | t.end() 25 | } 26 | 27 | collect('option-quote-escape', { quote: "'", escape: '\\' }, verify) 28 | }) 29 | 30 | test.cb('quote many', (t) => { 31 | const verify = (err, lines) => { 32 | t.false(err, 'no err') 33 | t.snapshot(lines[0], 'first row') 34 | t.snapshot(lines[1], 'second row') 35 | t.snapshot(lines[2], 'third row') 36 | t.is(lines.length, 3, '3 rows') 37 | t.end() 38 | } 39 | 40 | collect('option-quote-many', { quote: "'" }, verify) 41 | }) 42 | -------------------------------------------------------------------------------- /test/skipComments.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('comment', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.snapshot(lines) 9 | t.is(lines.length, 1, '1 row') 10 | t.end() 11 | } 12 | 13 | collect('comment', { skipComments: true }, verify) 14 | }) 15 | 16 | test.cb('custom comment', (t) => { 17 | const verify = (err, lines) => { 18 | t.false(err, 'no err') 19 | t.snapshot(lines) 20 | t.is(lines.length, 1, '1 row') 21 | t.end() 22 | } 23 | 24 | collect('option-comment', { skipComments: '~' }, verify) 25 | }) 26 | -------------------------------------------------------------------------------- /test/skipLines.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('skip lines', (t) => { 6 | const verify = (err, lines) => { 7 | t.false(err, 'no err') 8 | t.is(lines.length, 1, '1 row') 9 | t.is(JSON.stringify(lines[0]), JSON.stringify({ yes: 'ok', yup: 'ok', yeah: 'ok!' })) 10 | t.end() 11 | } 12 | 13 | collect('bad-data', { skipLines: 2 }, verify) 14 | }) 15 | 16 | test.cb('skip lines with headers', (t) => { 17 | const verify = (err, lines) => { 18 | t.false(err, 'no err') 19 | t.is(lines.length, 2, '2 rows') 20 | t.is(JSON.stringify(lines[0]), JSON.stringify({ s: 'yes', p: 'yup', h: 'yeah' })) 21 | t.is(JSON.stringify(lines[1]), JSON.stringify({ s: 'ok', p: 'ok', h: 'ok!' })) 22 | t.end() 23 | } 24 | 25 | collect('bad-data', { headers: ['s', 'p', 'h'], skipLines: 2 }, verify) 26 | }) 27 | -------------------------------------------------------------------------------- /test/snapshots/byteOffset.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/byteOffset.test.js` 2 | 3 | The actual snapshot is saved in `byteOffset.test.js.snap`. 4 | 5 | Generated by [AVA](https://avajs.dev). 6 | 7 | ## simple csv with byte offset 8 | 9 | > first row 10 | 11 | { 12 | byteOffset: 6, 13 | row: { 14 | a: '1', 15 | b: '2', 16 | c: '3', 17 | }, 18 | } 19 | 20 | ## large dataset with byte offset 21 | 22 | > first row 23 | 24 | { 25 | byteOffset: 85, 26 | row: { 27 | depth: '100', 28 | dmin: '', 29 | gap: '', 30 | id: 'ak12293661', 31 | latitude: '59.9988', 32 | longitude: '-152.7191', 33 | mag: '3', 34 | magType: 'ml', 35 | net: 'ak', 36 | nst: '', 37 | place: '54km S of Redoubt Volcano, Alaska', 38 | rms: '0.54', 39 | time: '2015-12-22T18:45:11.000Z', 40 | type: 'earthquake', 41 | updated: '2015-12-22T19:09:29.736Z', 42 | }, 43 | } 44 | 45 | > second row 46 | 47 | { 48 | byteOffset: 231, 49 | row: { 50 | depth: '65.4', 51 | dmin: '', 52 | gap: '', 53 | id: 'ak12293651', 54 | latitude: '62.9616', 55 | longitude: '-148.7532', 56 | mag: '1.9', 57 | magType: 'ml', 58 | net: 'ak', 59 | nst: '', 60 | place: '48km SSE of Cantwell, Alaska', 61 | rms: '0.51', 62 | time: '2015-12-22T18:38:34.000Z', 63 | type: 'earthquake', 64 | updated: '2015-12-22T18:47:23.287Z', 65 | }, 66 | } 67 | -------------------------------------------------------------------------------- /test/snapshots/byteOffset.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/byteOffset.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/escape.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/escape.test.js` 2 | 3 | The actual snapshot is saved in `escape.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## headers: false, numeric column names 8 | 9 | > lines 10 | 11 | [ 12 | { 13 | 0: 'a', 14 | 1: 'b', 15 | 2: 'c', 16 | }, 17 | { 18 | 0: '1', 19 | 1: '2', 20 | 2: '3', 21 | }, 22 | ] 23 | -------------------------------------------------------------------------------- /test/snapshots/escape.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/escape.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/headers.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/headers.test.js` 2 | 3 | The actual snapshot is saved in `headers.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## custom escape character 8 | 9 | > first row 10 | 11 | { 12 | a: '1', 13 | b: 'some "escaped" value', 14 | c: '2', 15 | } 16 | 17 | > second row 18 | 19 | { 20 | a: '3', 21 | b: '""', 22 | c: '4', 23 | } 24 | 25 | > third row 26 | 27 | { 28 | a: '5', 29 | b: '6', 30 | c: '7', 31 | } 32 | 33 | ## headers option 34 | 35 | > Snapshot 1 36 | 37 | [ 38 | { 39 | a: '1', 40 | b: '2', 41 | c: '3', 42 | }, 43 | { 44 | a: '4', 45 | b: '5', 46 | c: '6', 47 | }, 48 | { 49 | a: '7', 50 | b: '8', 51 | c: '9', 52 | }, 53 | ] 54 | 55 | ## headers: false 56 | 57 | > Snapshot 1 58 | 59 | [ 60 | { 61 | 0: 'a', 62 | 1: 'b', 63 | 2: 'c', 64 | }, 65 | { 66 | 0: '1', 67 | 1: '2', 68 | 2: '3', 69 | }, 70 | { 71 | 0: '4', 72 | 1: '5', 73 | 2: '6', 74 | }, 75 | { 76 | 0: '7', 77 | 1: '8', 78 | 2: '9', 79 | 3: '10', 80 | }, 81 | ] 82 | -------------------------------------------------------------------------------- /test/snapshots/headers.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/headers.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/issues.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/issues.test.js` 2 | 3 | The actual snapshot is saved in `issues.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## backtick separator (#105) 8 | 9 | > lines 10 | 11 | [ 12 | { 13 | p_desc: 'Bulbasaur can be seen napping', 14 | pokemon_id: '1', 15 | }, 16 | { 17 | p_desc: 'There is a bud on this', 18 | pokemon_id: '2', 19 | }, 20 | ] 21 | 22 | ## strict + skipLines (#136) 23 | 24 | > lines 25 | 26 | [ 27 | { 28 | h1: '1', 29 | h2: '2', 30 | h3: '3', 31 | }, 32 | { 33 | h1: '4', 34 | h2: '5', 35 | h3: '6', 36 | }, 37 | { 38 | h1: '7', 39 | h2: '8', 40 | h3: '9', 41 | }, 42 | ] 43 | -------------------------------------------------------------------------------- /test/snapshots/issues.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/issues.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/mapHeaders.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/mapHeaders.test.js` 2 | 3 | The actual snapshot is saved in `mapHeaders.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## rename columns 8 | 9 | > first row 10 | 11 | { 12 | x: '1', 13 | y: '2', 14 | z: '3', 15 | } 16 | 17 | ## skip columns a and c 18 | 19 | > first row 20 | 21 | { 22 | b: '2', 23 | } 24 | -------------------------------------------------------------------------------- /test/snapshots/mapHeaders.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/mapHeaders.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/mapValues.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/mapValues.test.js` 2 | 3 | The actual snapshot is saved in `mapValues.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## map values 8 | 9 | > first row 10 | 11 | { 12 | a: 1, 13 | b: 2, 14 | c: 3, 15 | } 16 | 17 | > headers 18 | 19 | [ 20 | 'a', 21 | 'b', 22 | 'c', 23 | ] 24 | 25 | > indexes 26 | 27 | [ 28 | 0, 29 | 1, 30 | 2, 31 | ] 32 | -------------------------------------------------------------------------------- /test/snapshots/mapValues.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/mapValues.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/newline.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/newline.test.js` 2 | 3 | The actual snapshot is saved in `newline.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## newline 8 | 9 | > first row 10 | 11 | { 12 | a: '1', 13 | b: '2', 14 | c: '3', 15 | } 16 | 17 | > second row 18 | 19 | { 20 | a: 'X-Men', 21 | b: '5', 22 | c: '6', 23 | } 24 | 25 | > third row 26 | 27 | { 28 | a: '7', 29 | b: '8', 30 | c: '9', 31 | } 32 | -------------------------------------------------------------------------------- /test/snapshots/newline.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/newline.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/quote.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/quote.test.js` 2 | 3 | The actual snapshot is saved in `quote.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## custom quote and escape character 8 | 9 | > first row 10 | 11 | { 12 | a: '1', 13 | b: 'some \'escaped\' value', 14 | c: '2', 15 | } 16 | 17 | > second row 18 | 19 | { 20 | a: '3', 21 | b: '\'\'', 22 | c: '4', 23 | } 24 | 25 | > third row 26 | 27 | { 28 | a: '5', 29 | b: '6', 30 | c: '7', 31 | } 32 | 33 | ## custom quote character 34 | 35 | > first row 36 | 37 | { 38 | a: '1', 39 | b: 'some value', 40 | c: '2', 41 | } 42 | 43 | > second row 44 | 45 | { 46 | a: '3', 47 | b: '4', 48 | c: '5', 49 | } 50 | 51 | ## quote many 52 | 53 | > first row 54 | 55 | { 56 | a: '1', 57 | b: 'some \'escaped\' value', 58 | c: '2', 59 | } 60 | 61 | > second row 62 | 63 | { 64 | a: '3', 65 | b: '\'\'', 66 | c: '4', 67 | } 68 | 69 | > third row 70 | 71 | { 72 | a: '5', 73 | b: '6', 74 | c: '7', 75 | } 76 | -------------------------------------------------------------------------------- /test/snapshots/quote.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/quote.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/skipComments.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/skipComments.test.js` 2 | 3 | The actual snapshot is saved in `skipComments.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## comment 8 | 9 | > Snapshot 1 10 | 11 | [ 12 | { 13 | a: '1', 14 | b: '2', 15 | c: '3', 16 | }, 17 | ] 18 | 19 | ## custom comment 20 | 21 | > Snapshot 1 22 | 23 | [ 24 | { 25 | a: '1', 26 | b: '2', 27 | c: '3', 28 | }, 29 | ] 30 | -------------------------------------------------------------------------------- /test/snapshots/skipComments.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/skipComments.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/strict.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/strict.test.js` 2 | 3 | The actual snapshot is saved in `strict.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## strict 8 | 9 | > first row 10 | 11 | { 12 | a: '1', 13 | b: '2', 14 | c: '3', 15 | } 16 | 17 | > second row 18 | 19 | { 20 | a: '4', 21 | b: '5', 22 | c: '6', 23 | } 24 | 25 | ## strict 26 | 27 | > first row 28 | 29 | undefined 30 | 31 | > second row 32 | 33 | undefined 34 | 35 | > first row 36 | 37 | undefined 38 | 39 | > second row 40 | 41 | undefined 42 | -------------------------------------------------------------------------------- /test/snapshots/strict.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/strict.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/strictNo.test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/strictNo.test.js` 2 | 3 | The actual snapshot is saved in `strictNo.test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## strict: false - less columns 8 | 9 | > first row 10 | 11 | { 12 | a: '1', 13 | b: '2', 14 | c: '3', 15 | } 16 | 17 | > broken row 18 | 19 | { 20 | a: '4', 21 | b: '5', 22 | } 23 | 24 | > last row 25 | 26 | { 27 | a: '6', 28 | b: '7', 29 | c: '8', 30 | } 31 | 32 | ## strict: false - more columns 33 | 34 | > first row 35 | 36 | { 37 | a: '1', 38 | b: '2', 39 | c: '3', 40 | } 41 | 42 | > broken row 43 | 44 | { 45 | _3: '7', 46 | a: '4', 47 | b: '5', 48 | c: '6', 49 | } 50 | 51 | > last row 52 | 53 | { 54 | a: '8', 55 | b: '9', 56 | c: '10', 57 | } 58 | -------------------------------------------------------------------------------- /test/snapshots/strictNo.test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/strictNo.test.js.snap -------------------------------------------------------------------------------- /test/snapshots/test.js.md: -------------------------------------------------------------------------------- 1 | # Snapshot report for `test/test.js` 2 | 3 | The actual snapshot is saved in `test.js.snap`. 4 | 5 | Generated by [AVA](https://ava.li). 6 | 7 | ## binary stanity 8 | 9 | > Snapshot 1 10 | 11 | '{"a":"1"}' 12 | 13 | ## csv-spectrum 14 | 15 | > comma_in_quotes 16 | 17 | [ 18 | { 19 | address: '120 any st.', 20 | city: 'Anytown, WW', 21 | first: 'John', 22 | last: 'Doe', 23 | zip: '08123', 24 | }, 25 | ] 26 | 27 | > empty 28 | 29 | [ 30 | { 31 | a: '1', 32 | b: '', 33 | c: '', 34 | }, 35 | { 36 | a: '2', 37 | b: '3', 38 | c: '4', 39 | }, 40 | ] 41 | 42 | > empty_crlf 43 | 44 | [ 45 | { 46 | a: '1', 47 | b: '', 48 | c: '', 49 | }, 50 | { 51 | a: '2', 52 | b: '3', 53 | c: '4', 54 | }, 55 | ] 56 | 57 | > escaped_quotes 58 | 59 | [ 60 | { 61 | a: '1', 62 | b: 'ha "ha" ha', 63 | }, 64 | { 65 | a: '3', 66 | b: '4', 67 | }, 68 | ] 69 | 70 | > json 71 | 72 | [ 73 | { 74 | key: '1', 75 | val: '{"type": "Point", "coordinates": [102.0, 0.5]}', 76 | }, 77 | ] 78 | 79 | > newlines 80 | 81 | [ 82 | { 83 | a: '1', 84 | b: '2', 85 | c: '3', 86 | }, 87 | { 88 | a: `Once upon ␊ 89 | a time`, 90 | b: '5', 91 | c: '6', 92 | }, 93 | { 94 | a: '7', 95 | b: '8', 96 | c: '9', 97 | }, 98 | ] 99 | 100 | > newlines_crlf 101 | 102 | [ 103 | { 104 | a: '1', 105 | b: '2', 106 | c: '3', 107 | }, 108 | { 109 | a: `Once upon ␍␊ 110 | a time`, 111 | b: '5', 112 | c: '6', 113 | }, 114 | { 115 | a: '7', 116 | b: '8', 117 | c: '9', 118 | }, 119 | ] 120 | 121 | > quotes_and_newlines 122 | 123 | [ 124 | { 125 | a: '1', 126 | b: `ha ␊ 127 | "ha" ␊ 128 | ha`, 129 | }, 130 | { 131 | a: '3', 132 | b: '4', 133 | }, 134 | ] 135 | 136 | > simple 137 | 138 | [ 139 | { 140 | a: '1', 141 | b: '2', 142 | c: '3', 143 | }, 144 | ] 145 | 146 | > simple_crlf 147 | 148 | [ 149 | { 150 | a: '1', 151 | b: '2', 152 | c: '3', 153 | }, 154 | ] 155 | 156 | > utf8 157 | 158 | [ 159 | { 160 | a: '1', 161 | b: '2', 162 | c: '3', 163 | }, 164 | { 165 | a: '4', 166 | b: '5', 167 | c: 'ʤ', 168 | }, 169 | ] 170 | 171 | ## newlines in a cell 172 | 173 | > first row 174 | 175 | { 176 | a: '1', 177 | b: '2', 178 | c: '3', 179 | } 180 | 181 | > second row 182 | 183 | { 184 | a: `Once upon ␊ 185 | a time`, 186 | b: '5', 187 | c: '6', 188 | } 189 | 190 | > fourth row 191 | 192 | { 193 | a: '7', 194 | b: '8', 195 | c: '9', 196 | } 197 | 198 | ## raw escaped quotes 199 | 200 | > first row 201 | 202 | { 203 | a: '1', 204 | b: 'ha "ha" ha', 205 | } 206 | 207 | > second row 208 | 209 | { 210 | a: '2', 211 | b: '""', 212 | } 213 | 214 | > third row 215 | 216 | { 217 | a: '3', 218 | b: '4', 219 | } 220 | 221 | ## raw escaped quotes and newlines 222 | 223 | > first row 224 | 225 | { 226 | a: '1', 227 | b: `ha ␊ 228 | "ha" ␊ 229 | ha`, 230 | } 231 | 232 | > second row 233 | 234 | { 235 | a: '2', 236 | b: ` ␊ 237 | "" ␊ 238 | `, 239 | } 240 | 241 | > third row 242 | 243 | { 244 | a: '3', 245 | b: '4', 246 | } 247 | 248 | ## simple csv 249 | 250 | > first row 251 | 252 | { 253 | a: '1', 254 | b: '2', 255 | c: '3', 256 | } 257 | 258 | ## supports strings 259 | 260 | > Snapshot 1 261 | 262 | { 263 | hello: 'world', 264 | } 265 | -------------------------------------------------------------------------------- /test/snapshots/test.js.snap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mafintosh/csv-parser/885501eba3e7a02afda2eef9dace21ba4282fd3f/test/snapshots/test.js.snap -------------------------------------------------------------------------------- /test/strict.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('strict', (t) => { 6 | const verify = (err, lines) => { 7 | t.is(err.name, 'RangeError', 'err name') 8 | t.is(err.message, 'Row length does not match headers', 'strict row length') 9 | t.snapshot(lines[0], 'first row') 10 | t.snapshot(lines[1], 'second row') 11 | t.is(lines.length, 2, '2 rows before error') 12 | t.end() 13 | } 14 | 15 | collect('strict', { strict: true }, verify) 16 | }) 17 | -------------------------------------------------------------------------------- /test/strictNo.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | 3 | const { collect } = require('./helpers/helper') 4 | 5 | test.cb('strict: false - more columns', (t) => { 6 | const verify = (err, lines) => { 7 | const headersFirstLine = Object.keys(lines[0]) 8 | const headersBrokenLine = Object.keys(lines[1]) 9 | const headersLastLine = Object.keys(lines[2]) 10 | t.false(err, 'no err') 11 | t.deepEqual(headersFirstLine, headersLastLine) 12 | t.deepEqual(headersBrokenLine, ['a', 'b', 'c', '_3']) 13 | t.snapshot(lines[0], 'first row') 14 | t.snapshot(lines[1], 'broken row') 15 | t.snapshot(lines[2], 'last row') 16 | t.is(lines.length, 3, '3 rows') 17 | t.is(headersBrokenLine.length, 4, '4 columns') 18 | t.end() 19 | } 20 | 21 | collect('strict-false-more-columns', { strict: false }, verify) 22 | }) 23 | 24 | test.cb('strict: false - less columns', (t) => { 25 | const verify = (err, lines) => { 26 | const headersFirstLine = Object.keys(lines[0]) 27 | const headersBrokenLine = Object.keys(lines[1]) 28 | const headersLastLine = Object.keys(lines[2]) 29 | t.false(err, 'no err') 30 | t.deepEqual(headersFirstLine, headersLastLine) 31 | t.deepEqual(headersBrokenLine, ['a', 'b']) 32 | t.snapshot(lines[0], 'first row') 33 | t.snapshot(lines[1], 'broken row') 34 | t.snapshot(lines[2], 'last row') 35 | t.is(lines.length, 3, '3 rows') 36 | t.is(headersBrokenLine.length, 2, '2 columns') 37 | t.end() 38 | } 39 | 40 | collect('strict-false-less-columns', { strict: false }, verify) 41 | }) 42 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | 3 | const test = require('ava') 4 | const bops = require('bops') 5 | const spectrum = require('csv-spectrum') 6 | const concat = require('concat-stream') 7 | const execa = require('execa') 8 | 9 | const csv = require('..') 10 | 11 | const { collect } = require('./helpers/helper') 12 | 13 | const eol = '\n' 14 | 15 | test.cb('simple csv', (t) => { 16 | const verify = (err, lines) => { 17 | t.false(err, 'no err') 18 | t.snapshot(lines[0], 'first row') 19 | t.is(lines.length, 1, '1 row') 20 | t.end() 21 | } 22 | 23 | collect('basic', verify) 24 | }) 25 | 26 | test.cb('supports strings', (t) => { 27 | const parser = csv() 28 | 29 | parser.on('data', (data) => { 30 | t.snapshot(data) 31 | t.end() 32 | }) 33 | 34 | parser.write('hello\n') 35 | parser.write('world\n') 36 | parser.end() 37 | }) 38 | 39 | test.cb('newlines in a cell', (t) => { 40 | const verify = (err, lines) => { 41 | t.false(err, 'no err') 42 | t.snapshot(lines[0], 'first row') 43 | t.snapshot(lines[1], 'second row') 44 | t.snapshot(lines[2], 'fourth row') 45 | t.is(lines.length, 3, '3 rows') 46 | t.end() 47 | } 48 | 49 | collect('newlines', verify) 50 | }) 51 | 52 | test.cb('raw escaped quotes', (t) => { 53 | const verify = (err, lines) => { 54 | t.false(err, 'no err') 55 | t.snapshot(lines[0], 'first row') 56 | t.snapshot(lines[1], 'second row') 57 | t.snapshot(lines[2], 'third row') 58 | t.is(lines.length, 3, '3 rows') 59 | t.end() 60 | } 61 | 62 | collect('escape-quotes', verify) 63 | }) 64 | 65 | test.cb('raw escaped quotes and newlines', (t) => { 66 | const verify = (err, lines) => { 67 | t.false(err, 'no err') 68 | t.snapshot(lines[0], 'first row') 69 | t.snapshot(lines[1], 'second row') 70 | t.snapshot(lines[2], 'third row') 71 | t.is(lines.length, 3, '3 rows') 72 | t.end() 73 | } 74 | 75 | collect('quotes+newlines', verify) 76 | }) 77 | 78 | test.cb('line with comma in quotes', (t) => { 79 | const headers = bops.from('a,b,c,d,e\n') 80 | const line = bops.from('John,Doe,120 any st.,"Anytown, WW",08123\n') 81 | const correct = JSON.stringify({ 82 | a: 'John', 83 | b: 'Doe', 84 | c: '120 any st.', 85 | d: 'Anytown, WW', 86 | e: '08123' 87 | }) 88 | const parser = csv() 89 | 90 | parser.write(headers) 91 | parser.write(line) 92 | parser.end() 93 | 94 | parser.once('data', (data) => { 95 | t.is(JSON.stringify(data), correct) 96 | t.end() 97 | }) 98 | }) 99 | 100 | test.cb('line with newline in quotes', (t) => { 101 | const headers = bops.from('a,b,c\n') 102 | const line = bops.from(`1,"ha ${eol}""ha"" ${eol}ha",3\n`) 103 | const correct = JSON.stringify({ a: '1', b: `ha ${eol}"ha" ${eol}ha`, c: '3' }) 104 | const parser = csv() 105 | 106 | parser.write(headers) 107 | parser.write(line) 108 | parser.end() 109 | 110 | parser.once('data', (data) => { 111 | t.is(JSON.stringify(data), correct) 112 | t.end() 113 | }) 114 | }) 115 | 116 | test.cb('cell with comma in quotes', (t) => { 117 | const headers = bops.from('a\n') 118 | const cell = bops.from('"Anytown, WW"\n') 119 | const correct = 'Anytown, WW' 120 | const parser = csv() 121 | 122 | parser.write(headers) 123 | parser.write(cell) 124 | parser.end() 125 | 126 | parser.once('data', (data) => { 127 | t.is(data.a, correct) 128 | t.end() 129 | }) 130 | }) 131 | 132 | test.cb('cell with newline', (t) => { 133 | const headers = bops.from('a\n') 134 | const cell = bops.from(`"why ${eol}hello ${eol}there"\n`) 135 | const correct = `why ${eol}hello ${eol}there` 136 | const parser = csv() 137 | 138 | parser.write(headers) 139 | parser.write(cell) 140 | parser.end() 141 | 142 | parser.once('data', (data) => { 143 | t.is(data.a, correct) 144 | t.end() 145 | }) 146 | }) 147 | 148 | test.cb('cell with escaped quote in quotes', (t) => { 149 | const headers = bops.from('a\n') 150 | const cell = bops.from('"ha ""ha"" ha"\n') 151 | const correct = 'ha "ha" ha' 152 | const parser = csv() 153 | 154 | parser.write(headers) 155 | parser.write(cell) 156 | parser.end() 157 | 158 | parser.once('data', (data) => { 159 | t.is(data.a, correct) 160 | t.end() 161 | }) 162 | }) 163 | 164 | test.cb('cell with multibyte character', (t) => { 165 | const headers = bops.from('a\n') 166 | const cell = bops.from('this ʤ is multibyte\n') 167 | const correct = 'this ʤ is multibyte' 168 | const parser = csv() 169 | 170 | parser.write(headers) 171 | parser.write(cell) 172 | parser.end() 173 | 174 | parser.once('data', (data) => { 175 | t.is(data.a, correct, 'multibyte character is preserved') 176 | t.end() 177 | }) 178 | }) 179 | 180 | test.cb('geojson', (t) => { 181 | const verify = (err, lines) => { 182 | t.false(err, 'no err') 183 | const lineObj = { 184 | type: 'LineString', 185 | coordinates: [[102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0]] 186 | } 187 | t.deepEqual(JSON.parse(lines[1].geojson), lineObj, 'linestrings match') 188 | t.end() 189 | } 190 | 191 | collect('geojson', verify) 192 | }) 193 | 194 | test.cb('empty columns', (t) => { 195 | const verify = (err, lines) => { 196 | t.false(err, 'no err') 197 | function testLine (row) { 198 | t.is(Object.keys(row).length, 3, 'Split into three columns') 199 | t.truthy(/^2007-01-0\d$/.test(row.a), 'First column is a date') 200 | t.truthy(row.b !== undefined, 'Empty column is in line') 201 | t.is(row.b.length, 0, 'Empty column is empty') 202 | t.truthy(row.c !== undefined, 'Empty column is in line') 203 | t.is(row.c.length, 0, 'Empty column is empty') 204 | } 205 | lines.forEach(testLine) 206 | t.end() 207 | } 208 | 209 | collect('empty-columns', ['a', 'b', 'c'], verify) 210 | }) 211 | 212 | test.cb('csv-spectrum', (t) => { 213 | spectrum((err, data) => { 214 | if (err) throw err 215 | let pending = data.length 216 | data.map((d) => { 217 | const parser = csv() 218 | const collector = concat((objs) => { 219 | t.snapshot(objs, d.name) 220 | done() 221 | }) 222 | parser.pipe(collector) 223 | parser.write(d.csv) 224 | parser.end() 225 | }) 226 | function done () { 227 | pending-- 228 | if (pending === 0) t.end() 229 | } 230 | }) 231 | }) 232 | 233 | test.cb('process all rows', (t) => { 234 | const verify = (err, lines) => { 235 | t.false(err, 'no err') 236 | t.is(lines.length, 7268, '7268 rows') 237 | t.end() 238 | } 239 | 240 | collect('large-dataset', {}, verify) 241 | }) 242 | 243 | test('binary stanity', async (t) => { 244 | const binPath = path.resolve(__dirname, '../bin/csv-parser') 245 | const { stdout } = await execa(`echo "a\n1" | ${process.execPath} ${binPath}`, { shell: true }) 246 | 247 | t.snapshot(stdout) 248 | }) 249 | --------------------------------------------------------------------------------