├── .gitignore ├── .travis.yml ├── LICENCE ├── examples └── pretty.js ├── index.js ├── package-lock.json ├── package.json ├── readme.markdown └── test ├── options.asynct.js ├── partitioned_unicode.js ├── split.asynct.js └── try_catch.asynct.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | node_modules/* 3 | npm_debug.log 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Dominic Tarr 2 | 3 | Permission is hereby granted, free of charge, 4 | to any person obtaining a copy of this software and 5 | associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 20 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /examples/pretty.js: -------------------------------------------------------------------------------- 1 | 2 | var inspect = require('util').inspect 3 | var es = require('event-stream') //load event-stream 4 | var split = require('../') 5 | 6 | if(!module.parent) { 7 | es.pipe( //pipe joins streams together 8 | process.openStdin(), //open stdin 9 | split(), //split stream to break on newlines 10 | es.map(function (data, callback) {//turn this async function into a stream 11 | var j 12 | try { 13 | j = JSON.parse(data) //try to parse input into json 14 | } catch (err) { 15 | return callback(null, data) //if it fails just pass it anyway 16 | } 17 | callback(null, inspect(j)) //render it nicely 18 | }), 19 | process.stdout // pipe it to stdout ! 20 | ) 21 | } 22 | 23 | // run this 24 | // 25 | // curl -sS registry.npmjs.org/event-stream | node pretty.js 26 | // 27 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | //filter will reemit the data if cb(err,pass) pass is truthy 2 | 3 | // reduce is more tricky 4 | // maybe we want to group the reductions or emit progress updates occasionally 5 | // the most basic reduce just emits one 'data' event after it has received 'end' 6 | 7 | 8 | var through = require('through') 9 | var Decoder = require('string_decoder').StringDecoder 10 | 11 | module.exports = split 12 | 13 | //TODO pass in a function to map across the lines. 14 | 15 | function split (matcher, mapper, options) { 16 | var decoder = new Decoder() 17 | var soFar = '' 18 | var maxLength = options && options.maxLength; 19 | var trailing = options && options.trailing === false ? false : true 20 | if('function' === typeof matcher) 21 | mapper = matcher, matcher = null 22 | if (!matcher) 23 | matcher = /\r?\n/ 24 | 25 | function emit(stream, piece) { 26 | if(mapper) { 27 | try { 28 | piece = mapper(piece) 29 | } 30 | catch (err) { 31 | return stream.emit('error', err) 32 | } 33 | if('undefined' !== typeof piece) 34 | stream.queue(piece) 35 | } 36 | else 37 | stream.queue(piece) 38 | } 39 | 40 | function next (stream, buffer) { 41 | var pieces = ((soFar != null ? soFar : '') + buffer).split(matcher) 42 | soFar = pieces.pop() 43 | 44 | if (maxLength && soFar.length > maxLength) 45 | return stream.emit('error', new Error('maximum buffer reached')) 46 | 47 | for (var i = 0; i < pieces.length; i++) { 48 | var piece = pieces[i] 49 | emit(stream, piece) 50 | } 51 | } 52 | 53 | return through(function (b) { 54 | next(this, decoder.write(b)) 55 | }, 56 | function () { 57 | if(decoder.end) 58 | next(this, decoder.end()) 59 | if(trailing && soFar != null) 60 | emit(this, soFar) 61 | this.queue(null) 62 | }) 63 | } 64 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "split", 3 | "version": "1.0.1", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "assertions": { 8 | "version": "2.3.4", 9 | "resolved": "https://registry.npmjs.org/assertions/-/assertions-2.3.4.tgz", 10 | "integrity": "sha1-qUM87R/OV8yZmvCWXRAI6WwnluY=", 11 | "dev": true, 12 | "requires": { 13 | "fomatto": "git://github.com/BonsaiDen/Fomatto.git#468666f600b46f9067e3da7200fd9df428923ea6", 14 | "render": "0.1.4", 15 | "traverser": "1.0.0" 16 | }, 17 | "dependencies": { 18 | "traverser": { 19 | "version": "1.0.0", 20 | "resolved": "https://registry.npmjs.org/traverser/-/traverser-1.0.0.tgz", 21 | "integrity": "sha1-b1nlgTdZruqzZGuPRRP9SmLk/iA=", 22 | "dev": true, 23 | "requires": { 24 | "curry": "0.0.4" 25 | } 26 | } 27 | } 28 | }, 29 | "asynct": { 30 | "version": "1.1.0", 31 | "resolved": "https://registry.npmjs.org/asynct/-/asynct-1.1.0.tgz", 32 | "integrity": "sha1-O7LFB9OhPyAAugWGmhp0DCGICRM=", 33 | "dev": true, 34 | "requires": { 35 | "ctrlflow": "0.0.3", 36 | "test-cmd": "1.7.0" 37 | } 38 | }, 39 | "ctrlflow": { 40 | "version": "0.0.3", 41 | "resolved": "https://registry.npmjs.org/ctrlflow/-/ctrlflow-0.0.3.tgz", 42 | "integrity": "sha1-6WwYCJDcBcKztDc19SmnEpPhOcA=", 43 | "dev": true, 44 | "requires": { 45 | "curry": "0.0.4" 46 | } 47 | }, 48 | "curry": { 49 | "version": "0.0.4", 50 | "resolved": "https://registry.npmjs.org/curry/-/curry-0.0.4.tgz", 51 | "integrity": "sha1-F1DVGNkZxE89N/9E7caT3h8NX8s=", 52 | "dev": true 53 | }, 54 | "d-utils": { 55 | "version": "2.1.1", 56 | "resolved": "https://registry.npmjs.org/d-utils/-/d-utils-2.1.1.tgz", 57 | "integrity": "sha1-NZxqACBJUW8ELbnkneMCIE4f7Dw=", 58 | "dev": true 59 | }, 60 | "duplexer": { 61 | "version": "0.1.1", 62 | "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", 63 | "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=", 64 | "dev": true 65 | }, 66 | "event-stream": { 67 | "version": "3.0.20", 68 | "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.0.20.tgz", 69 | "integrity": "sha1-A4u7LqnqkDhbJvvBhU0LU58qvqM=", 70 | "dev": true, 71 | "requires": { 72 | "duplexer": "0.1.1", 73 | "from": "0.1.7", 74 | "map-stream": "0.0.7", 75 | "pause-stream": "0.0.11", 76 | "split": "0.2.10", 77 | "stream-combiner": "0.0.4", 78 | "through": "2.3.8" 79 | } 80 | }, 81 | "fomatto": { 82 | "version": "git://github.com/BonsaiDen/Fomatto.git#468666f600b46f9067e3da7200fd9df428923ea6", 83 | "dev": true 84 | }, 85 | "from": { 86 | "version": "0.1.7", 87 | "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", 88 | "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=", 89 | "dev": true 90 | }, 91 | "inherits": { 92 | "version": "2.0.3", 93 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 94 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", 95 | "dev": true 96 | }, 97 | "it-is": { 98 | "version": "1.0.3", 99 | "resolved": "https://registry.npmjs.org/it-is/-/it-is-1.0.3.tgz", 100 | "integrity": "sha1-RpcAgZg4pGVu1PF2lT9b/PU/SDs=", 101 | "dev": true, 102 | "requires": { 103 | "assertions": "2.3.4", 104 | "render": "0.1.4", 105 | "style": "0.1.1", 106 | "traverser": "0.0.5", 107 | "trees": "0.0.4" 108 | } 109 | }, 110 | "ls-r": { 111 | "version": "0.2.1", 112 | "resolved": "https://registry.npmjs.org/ls-r/-/ls-r-0.2.1.tgz", 113 | "integrity": "sha1-sjmgH6p9QxSIdKZP6xHgrey2b6A=", 114 | "dev": true, 115 | "requires": { 116 | "ctrlflow": "4.1.3", 117 | "d-utils": "2.3.3", 118 | "optimist": "0.2.8" 119 | }, 120 | "dependencies": { 121 | "ctrlflow": { 122 | "version": "4.1.3", 123 | "resolved": "https://registry.npmjs.org/ctrlflow/-/ctrlflow-4.1.3.tgz", 124 | "integrity": "sha1-MCi+xE8BocET/ZIG/E+ZgjN4QoY=", 125 | "dev": true, 126 | "requires": { 127 | "curry": "0.0.4", 128 | "d-utils": "1.4.0" 129 | }, 130 | "dependencies": { 131 | "d-utils": { 132 | "version": "1.4.0", 133 | "resolved": "https://registry.npmjs.org/d-utils/-/d-utils-1.4.0.tgz", 134 | "integrity": "sha1-RHbSSGduWDLG0Pxp4u0NEMFMn/c=", 135 | "dev": true 136 | } 137 | } 138 | }, 139 | "d-utils": { 140 | "version": "2.3.3", 141 | "resolved": "https://registry.npmjs.org/d-utils/-/d-utils-2.3.3.tgz", 142 | "integrity": "sha1-utm7TWjxLr1VJvU1UI2D4/skFz0=", 143 | "dev": true 144 | } 145 | } 146 | }, 147 | "macgyver": { 148 | "version": "1.6.3", 149 | "resolved": "https://registry.npmjs.org/macgyver/-/macgyver-1.6.3.tgz", 150 | "integrity": "sha1-2pcZQt8MRDV53KhhcyAgqQXfsrg=", 151 | "dev": true, 152 | "requires": { 153 | "assertions": "2.3.4" 154 | } 155 | }, 156 | "map-stream": { 157 | "version": "0.0.7", 158 | "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", 159 | "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=", 160 | "dev": true 161 | }, 162 | "optimist": { 163 | "version": "0.2.8", 164 | "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.2.8.tgz", 165 | "integrity": "sha1-6YGrfiaLRXlIWTtVZ0wJmoFcrDE=", 166 | "dev": true, 167 | "requires": { 168 | "wordwrap": "0.0.3" 169 | } 170 | }, 171 | "pause-stream": { 172 | "version": "0.0.11", 173 | "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", 174 | "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", 175 | "dev": true, 176 | "requires": { 177 | "through": "2.3.8" 178 | } 179 | }, 180 | "render": { 181 | "version": "0.1.4", 182 | "resolved": "https://registry.npmjs.org/render/-/render-0.1.4.tgz", 183 | "integrity": "sha1-z7M6NOJgaFkdQYRp4j2Mxc4c7/U=", 184 | "dev": true, 185 | "requires": { 186 | "traverser": "0.0.5" 187 | } 188 | }, 189 | "should": { 190 | "version": "11.2.1", 191 | "resolved": "https://registry.npmjs.org/should/-/should-11.2.1.tgz", 192 | "integrity": "sha1-kPVRRVUtAc/CAGZuToGKHJZw7aI=", 193 | "dev": true, 194 | "requires": { 195 | "should-equal": "1.0.1", 196 | "should-format": "3.0.3", 197 | "should-type": "1.4.0", 198 | "should-type-adaptors": "1.0.1", 199 | "should-util": "1.0.0" 200 | } 201 | }, 202 | "should-equal": { 203 | "version": "1.0.1", 204 | "resolved": "https://registry.npmjs.org/should-equal/-/should-equal-1.0.1.tgz", 205 | "integrity": "sha1-C26VFvJgGp+wuy3MNpr6HH4gCvc=", 206 | "dev": true, 207 | "requires": { 208 | "should-type": "1.4.0" 209 | } 210 | }, 211 | "should-format": { 212 | "version": "3.0.3", 213 | "resolved": "https://registry.npmjs.org/should-format/-/should-format-3.0.3.tgz", 214 | "integrity": "sha1-m/yPdPo5IFxT04w01xcwPidxJPE=", 215 | "dev": true, 216 | "requires": { 217 | "should-type": "1.4.0", 218 | "should-type-adaptors": "1.0.1" 219 | } 220 | }, 221 | "should-type": { 222 | "version": "1.4.0", 223 | "resolved": "https://registry.npmjs.org/should-type/-/should-type-1.4.0.tgz", 224 | "integrity": "sha1-B1bYzoRt/QmEOmlHcZ36DUz/XPM=", 225 | "dev": true 226 | }, 227 | "should-type-adaptors": { 228 | "version": "1.0.1", 229 | "resolved": "https://registry.npmjs.org/should-type-adaptors/-/should-type-adaptors-1.0.1.tgz", 230 | "integrity": "sha1-7+VVPN9oz/ZuXF9RtxLcNRx3vqo=", 231 | "dev": true, 232 | "requires": { 233 | "should-type": "1.4.0", 234 | "should-util": "1.0.0" 235 | } 236 | }, 237 | "should-util": { 238 | "version": "1.0.0", 239 | "resolved": "https://registry.npmjs.org/should-util/-/should-util-1.0.0.tgz", 240 | "integrity": "sha1-yYzaN0qmsZDfi6h8mInCtNtiAGM=", 241 | "dev": true 242 | }, 243 | "split": { 244 | "version": "0.2.10", 245 | "resolved": "https://registry.npmjs.org/split/-/split-0.2.10.tgz", 246 | "integrity": "sha1-Zwl8YB1pfOE2j0GPBs0gHPBSGlc=", 247 | "dev": true, 248 | "requires": { 249 | "through": "2.3.8" 250 | } 251 | }, 252 | "stream-combiner": { 253 | "version": "0.0.4", 254 | "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", 255 | "integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=", 256 | "dev": true, 257 | "requires": { 258 | "duplexer": "0.1.1" 259 | } 260 | }, 261 | "stream-spec": { 262 | "version": "0.2.1", 263 | "resolved": "https://registry.npmjs.org/stream-spec/-/stream-spec-0.2.1.tgz", 264 | "integrity": "sha1-vuEVU/JRJDk1yHN10CErUX15f3I=", 265 | "dev": true, 266 | "requires": { 267 | "assertions": "2.3.4", 268 | "macgyver": "1.6.3" 269 | } 270 | }, 271 | "string-to-stream": { 272 | "version": "1.0.1", 273 | "resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.0.1.tgz", 274 | "integrity": "sha1-0oKOiV+49m+MovFoLww3PIbM95Y=", 275 | "dev": true, 276 | "requires": { 277 | "inherits": "2.0.3" 278 | } 279 | }, 280 | "style": { 281 | "version": "0.1.1", 282 | "resolved": "https://registry.npmjs.org/style/-/style-0.1.1.tgz", 283 | "integrity": "sha1-4vq2WxuB0+AOvK2FTMWEuiMfJW0=", 284 | "dev": true, 285 | "requires": { 286 | "curry": "0.0.4" 287 | } 288 | }, 289 | "test-cmd": { 290 | "version": "1.7.0", 291 | "resolved": "https://registry.npmjs.org/test-cmd/-/test-cmd-1.7.0.tgz", 292 | "integrity": "sha1-nldMSfOUcweeLyj+cWF4EtFH4xA=", 293 | "dev": true, 294 | "requires": { 295 | "ctrlflow": "4.1.3", 296 | "d-utils": "2.1.1", 297 | "ls-r": "0.2.1", 298 | "optimist": "0.2.8", 299 | "test-report": "1.1.2", 300 | "test-report-view": "1.0.0" 301 | }, 302 | "dependencies": { 303 | "ctrlflow": { 304 | "version": "4.1.3", 305 | "resolved": "https://registry.npmjs.org/ctrlflow/-/ctrlflow-4.1.3.tgz", 306 | "integrity": "sha1-MCi+xE8BocET/ZIG/E+ZgjN4QoY=", 307 | "dev": true, 308 | "requires": { 309 | "curry": "0.0.4", 310 | "d-utils": "1.4.0" 311 | }, 312 | "dependencies": { 313 | "d-utils": { 314 | "version": "1.4.0", 315 | "resolved": "https://registry.npmjs.org/d-utils/-/d-utils-1.4.0.tgz", 316 | "integrity": "sha1-RHbSSGduWDLG0Pxp4u0NEMFMn/c=", 317 | "dev": true 318 | } 319 | } 320 | } 321 | } 322 | }, 323 | "test-report": { 324 | "version": "1.1.2", 325 | "resolved": "https://registry.npmjs.org/test-report/-/test-report-1.1.2.tgz", 326 | "integrity": "sha1-nCHcPuk/qpMwTfnl1Zo+ccMBxaQ=", 327 | "dev": true 328 | }, 329 | "test-report-view": { 330 | "version": "1.0.0", 331 | "resolved": "https://registry.npmjs.org/test-report-view/-/test-report-view-1.0.0.tgz", 332 | "integrity": "sha1-/Z8/hA/Hsp0/dhNweJ93gFsZ440=", 333 | "dev": true 334 | }, 335 | "through": { 336 | "version": "2.3.8", 337 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 338 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" 339 | }, 340 | "traverser": { 341 | "version": "0.0.5", 342 | "resolved": "https://registry.npmjs.org/traverser/-/traverser-0.0.5.tgz", 343 | "integrity": "sha1-xm84xFagwhqIAUsSI1gMfr4GMes=", 344 | "dev": true, 345 | "requires": { 346 | "curry": "0.0.4" 347 | } 348 | }, 349 | "trees": { 350 | "version": "0.0.4", 351 | "resolved": "https://registry.npmjs.org/trees/-/trees-0.0.4.tgz", 352 | "integrity": "sha1-L1H9m19khvcyyX4R7nVmcja+HVk=", 353 | "dev": true, 354 | "requires": { 355 | "should": "11.2.1", 356 | "style": "0.1.1", 357 | "traverser": "0.0.5" 358 | } 359 | }, 360 | "ubelt": { 361 | "version": "2.9.0", 362 | "resolved": "https://registry.npmjs.org/ubelt/-/ubelt-2.9.0.tgz", 363 | "integrity": "sha1-GVDembjnvb1Tzr2gAkpGjM3mtPY=", 364 | "dev": true 365 | }, 366 | "wordwrap": { 367 | "version": "0.0.3", 368 | "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", 369 | "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", 370 | "dev": true 371 | } 372 | } 373 | } 374 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "split", 3 | "version": "1.0.1", 4 | "license": "MIT", 5 | "description": "split a Text Stream into a Line Stream", 6 | "homepage": "http://github.com/dominictarr/split", 7 | "repository": { 8 | "type": "git", 9 | "url": "git://github.com/dominictarr/split.git" 10 | }, 11 | "dependencies": { 12 | "through": "2" 13 | }, 14 | "devDependencies": { 15 | "asynct": "*", 16 | "event-stream": "~3.0.2", 17 | "it-is": "1", 18 | "stream-spec": "~0.2", 19 | "ubelt": "~2.9", 20 | "string-to-stream": "~1.0.0" 21 | }, 22 | "scripts": { 23 | "test": "asynct test/" 24 | }, 25 | "author": "Dominic Tarr (http://bit.ly/dominictarr)", 26 | "optionalDependencies": {}, 27 | "engines": { 28 | "node": "*" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /readme.markdown: -------------------------------------------------------------------------------- 1 | # Split (matcher) 2 | 3 | [![build status](https://secure.travis-ci.org/dominictarr/split.png)](http://travis-ci.org/dominictarr/split) 4 | 5 | Break up a stream and reassemble it so that each line is a chunk. matcher may be a `String`, or a `RegExp` 6 | 7 | Example, read every line in a file ... 8 | 9 | ``` js 10 | fs.createReadStream(file) 11 | .pipe(split()) 12 | .on('data', function (line) { 13 | //each chunk now is a separate line! 14 | }) 15 | 16 | ``` 17 | 18 | `split` takes the same arguments as `string.split` except it defaults to '/\r?\n/' instead of ',', and the optional `limit` parameter is ignored. 19 | [String#split](https://developer.mozilla.org/en/JavaScript/Reference/Global_Objects/String/split) 20 | 21 | `split` takes an optional options object on its third argument. 22 | 23 | ``` js 24 | split(matcher, mapper, options) 25 | ``` 26 | 27 | Valid options: 28 | 29 | * maxLength - The maximum buffer length without seeing a newline or `matcher`, 30 | if a single line exceeds this, the split stream will emit an error. 31 | 32 | ``` js 33 | split(JSON.parse, null, { maxLength: 2}) 34 | ``` 35 | 36 | * trailing - By default the last buffer not delimited by a newline or `matcher` will be emitted. To prevent this set `options.trailing` to `false`. 37 | 38 | ``` js 39 | split(JSON.parse, null, { trailing: false }) 40 | ``` 41 | 42 | ## keep matched splitter 43 | 44 | As with `String#split`, if you split by a regular expression with a matching group, 45 | the matches will be retained in the collection. 46 | 47 | ``` 48 | stdin 49 | .pipe(split(/(\r?\n)/)) 50 | ... //lines + separators. 51 | ``` 52 | 53 | 54 | # NDJ - Newline Delimited Json 55 | 56 | `split` accepts a function which transforms each line. 57 | 58 | ``` js 59 | fs.createReadStream(file) 60 | .pipe(split(JSON.parse)) 61 | .on('data', function (obj) { 62 | //each chunk now is a a js object 63 | }) 64 | .on('error', function (err) { 65 | //syntax errors will land here 66 | //note, this ends the stream. 67 | }) 68 | ``` 69 | 70 | # License 71 | 72 | MIT 73 | -------------------------------------------------------------------------------- /test/options.asynct.js: -------------------------------------------------------------------------------- 1 | var it = require('it-is').style('colour') 2 | , split = require('..') 3 | 4 | exports ['maximum buffer limit'] = function (test) { 5 | var s = split(JSON.parse, null, { 6 | maxLength: 2 7 | }) 8 | , caughtError = false 9 | , rows = [] 10 | 11 | s.on('error', function (err) { 12 | caughtError = true 13 | }) 14 | 15 | s.on('data', function (row) { rows.push(row) }) 16 | 17 | s.write('{"a":1}\n{"') 18 | s.write('{ "') 19 | it(caughtError).equal(true) 20 | 21 | s.end() 22 | test.done() 23 | } 24 | 25 | exports ['ignore trailing buffers'] = function (test) { 26 | var s = split(JSON.parse, null, { 27 | trailing: false 28 | }) 29 | , caughtError = false 30 | , rows = [] 31 | 32 | s.on('error', function (err) { 33 | caughtError = true 34 | }) 35 | 36 | s.on('data', function (row) { rows.push(row) }) 37 | 38 | s.write('{"a":1}\n{"') 39 | s.write('{ "') 40 | s.end() 41 | 42 | it(caughtError).equal(false) 43 | it(rows).deepEqual([ { a: 1 } ]) 44 | 45 | test.done() 46 | } 47 | -------------------------------------------------------------------------------- /test/partitioned_unicode.js: -------------------------------------------------------------------------------- 1 | var it = require('it-is').style('colour') 2 | , split = require('..') 3 | 4 | exports ['split data with partitioned unicode character'] = function (test) { 5 | var s = split(/,/g) 6 | , caughtError = false 7 | , rows = [] 8 | 9 | s.on('error', function (err) { 10 | caughtError = true 11 | }) 12 | 13 | s.on('data', function (row) { rows.push(row) }) 14 | 15 | var x = 'テスト試験今日とても,よい天気で' 16 | unicodeData = new Buffer(x); 17 | 18 | // partition of 日 19 | piece1 = unicodeData.slice(0, 20); 20 | piece2 = unicodeData.slice(20, unicodeData.length); 21 | 22 | s.write(piece1); 23 | s.write(piece2); 24 | 25 | s.end() 26 | 27 | it(caughtError).equal(false) 28 | 29 | it(rows).deepEqual(['テスト試験今日とても', 'よい天気で']); 30 | 31 | it(rows).deepEqual(x.split(',')) 32 | 33 | test.done() 34 | } 35 | -------------------------------------------------------------------------------- /test/split.asynct.js: -------------------------------------------------------------------------------- 1 | var es = require('event-stream') 2 | , it = require('it-is').style('colour') 3 | , d = require('ubelt') 4 | , split = require('..') 5 | , join = require('path').join 6 | , fs = require('fs') 7 | , Stream = require('stream').Stream 8 | , Readable = require('stream').Readable 9 | , spec = require('stream-spec') 10 | , through = require('through') 11 | , stringStream = require('string-to-stream') 12 | 13 | exports ['split() works like String#split'] = function (test) { 14 | var readme = join(__filename) 15 | , expected = fs.readFileSync(readme, 'utf-8').split('\n') 16 | , cs = split() 17 | , actual = [] 18 | , ended = false 19 | , x = spec(cs).through() 20 | 21 | var a = new Stream () 22 | 23 | a.write = function (l) { 24 | actual.push(l.trim()) 25 | } 26 | a.end = function () { 27 | 28 | ended = true 29 | expected.forEach(function (v,k) { 30 | //String.split will append an empty string '' 31 | //if the string ends in a split pattern. 32 | //es.split doesn't which was breaking this test. 33 | //clearly, appending the empty string is correct. 34 | //tests are passing though. which is the current job. 35 | if(v) 36 | it(actual[k]).like(v) 37 | }) 38 | //give the stream time to close 39 | process.nextTick(function () { 40 | test.done() 41 | x.validate() 42 | }) 43 | } 44 | a.writable = true 45 | 46 | fs.createReadStream(readme, {flags: 'r'}).pipe(cs) 47 | cs.pipe(a) 48 | 49 | } 50 | 51 | exports ['split() takes mapper function'] = function (test) { 52 | var readme = join(__filename) 53 | , expected = fs.readFileSync(readme, 'utf-8').split('\n') 54 | , cs = split(function (line) { return line.toUpperCase() }) 55 | , actual = [] 56 | , ended = false 57 | , x = spec(cs).through() 58 | 59 | var a = new Stream () 60 | 61 | a.write = function (l) { 62 | actual.push(l.trim()) 63 | } 64 | a.end = function () { 65 | 66 | ended = true 67 | expected.forEach(function (v,k) { 68 | //String.split will append an empty string '' 69 | //if the string ends in a split pattern. 70 | //es.split doesn't which was breaking this test. 71 | //clearly, appending the empty string is correct. 72 | //tests are passing though. which is the current job. 73 | if(v) 74 | it(actual[k]).equal(v.trim().toUpperCase()) 75 | }) 76 | //give the stream time to close 77 | process.nextTick(function () { 78 | test.done() 79 | x.validate() 80 | }) 81 | } 82 | a.writable = true 83 | 84 | fs.createReadStream(readme, {flags: 'r'}).pipe(cs) 85 | cs.pipe(a) 86 | 87 | } 88 | 89 | exports ['split() works with empty string chunks'] = function (test) { 90 | var str = ' foo' 91 | , expected = str.split(/[\s]*/).reduce(splitBy(/[\s]*/), []) 92 | , cs1 = split(/[\s]*/) 93 | , cs2 = split(/[\s]*/) 94 | , actual = [] 95 | , ended = false 96 | , x = spec(cs1).through() 97 | , y = spec(cs2).through() 98 | 99 | var a = new Stream () 100 | 101 | a.write = function (l) { 102 | actual.push(l.trim()) 103 | } 104 | a.end = function () { 105 | 106 | ended = true 107 | expected.forEach(function (v,k) { 108 | //String.split will append an empty string '' 109 | //if the string ends in a split pattern. 110 | //es.split doesn't which was breaking this test. 111 | //clearly, appending the empty string is correct. 112 | //tests are passing though. which is the current job. 113 | if(v) 114 | it(actual[k]).like(v) 115 | }) 116 | //give the stream time to close 117 | process.nextTick(function () { 118 | test.done() 119 | x.validate() 120 | y.validate() 121 | }) 122 | } 123 | a.writable = true 124 | 125 | cs1.pipe(cs2) 126 | cs2.pipe(a) 127 | 128 | cs1.write(str) 129 | cs1.end() 130 | 131 | } 132 | 133 | function splitBy (delimiter) { 134 | return function (arr, piece) { 135 | return arr.concat(piece.split(delimiter)) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /test/try_catch.asynct.js: -------------------------------------------------------------------------------- 1 | var it = require('it-is').style('colour') 2 | , split = require('..') 3 | 4 | exports ['emit mapper exceptions as error events'] = function (test) { 5 | var s = split(JSON.parse) 6 | , caughtError = false 7 | , rows = [] 8 | 9 | s.on('error', function (err) { 10 | caughtError = true 11 | }) 12 | 13 | s.on('data', function (row) { rows.push(row) }) 14 | 15 | s.write('{"a":1}\n{"') 16 | it(caughtError).equal(false) 17 | it(rows).deepEqual([ { a: 1 } ]) 18 | 19 | s.write('b":2}\n{"c":}\n') 20 | it(caughtError).equal(true) 21 | it(rows).deepEqual([ { a: 1 }, { b: 2 } ]) 22 | 23 | s.end() 24 | test.done() 25 | } 26 | 27 | exports ['mapper error events on trailing chunks'] = function (test) { 28 | var s = split(JSON.parse) 29 | , caughtError = false 30 | , rows = [] 31 | 32 | s.on('error', function (err) { 33 | caughtError = true 34 | }) 35 | 36 | s.on('data', function (row) { rows.push(row) }) 37 | 38 | s.write('{"a":1}\n{"') 39 | it(caughtError).equal(false) 40 | it(rows).deepEqual([ { a: 1 } ]) 41 | 42 | s.write('b":2}\n{"c":}') 43 | it(caughtError).equal(false) 44 | it(rows).deepEqual([ { a: 1 }, { b: 2 } ]) 45 | 46 | s.end() 47 | it(caughtError).equal(true) 48 | it(rows).deepEqual([ { a: 1 }, { b: 2 } ]) 49 | 50 | test.done() 51 | } 52 | --------------------------------------------------------------------------------