├── .gitignore ├── .npmignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── lib └── batchflow.js ├── package.json └── test ├── batchflow-load.test.js ├── batchflow-par-array.test.js ├── batchflow-par-limit.test.js ├── batchflow-par-object.test.js ├── batchflow-seq-array.test.js ├── batchflow-seq-object.test.js ├── batchflow.test.js ├── batchflow_regression-file-filter.test.js ├── mocha.opts └── resources ├── .gitkeep └── regr-file-filter ├── config └── .gitkeep ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .gitignore 2 | .travis.yml 3 | test/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 0.10 4 | - 0.8 -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.4.0 / 2013-04-24 2 | ------------------ 3 | * fixed for Node v0.10 4 | 5 | 0.3.2 / 2013-02-05 6 | ------------------ 7 | * Fixed bug that if code in `each` was sync, then it'd eventually cause a RangeError of Maxium Stack Limit. 8 | 9 | 0.3.1 / 2013-01-22 10 | ------------------ 11 | * Fixed a nasty regression causing the each callback to be called more than the total. 12 | 13 | 0.3.0 / 2012-10-13 14 | ------------------ 15 | * Added `total` and `finished` fields. 16 | * Added `limit` parameter to `parallel()`. 17 | * Changed behavior that keeps batchflow processing if an error occurs instead of stopping before it iterates all items. 18 | * Major refactoring of tests and code. 19 | 20 | 0.2.0 / 2012-09-16 21 | ------------------ 22 | * When no parameter is passed to `done()/next()`, then `undefined` is not added to the `results` array of the `end()` callback. 23 | 24 | 0.1.0 / 2012-09-06 25 | ------------------ 26 | * Errors would not throw or show up if generated in `each` method and no error handler was set. 27 | * Fixed bug from `0.0.4` that would still cause an empty collection to execute at least one `each()`. 28 | 29 | 0.0.4 / 2012-09-05 30 | ------------------ 31 | * Fixed bug that would execute `each()` on an empty collection. 32 | 33 | 0.0.3 / 2012-08-15 34 | ------------------ 35 | * Fixed bug when array or object contained any synchronous data. 36 | 37 | 0.0.2 / 2012-08-14 38 | ------------------ 39 | * Added `error` method. 40 | * Made `series()` an alias for `sequential()`. 41 | 42 | 0.0.1 / 2012-08-14 43 | ------------------ 44 | * Initial release. 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2012, JP 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files 6 | (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, 7 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 11 | 12 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 13 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS 14 | OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 15 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | Node.js - batchflow 3 | =================== 4 | 5 | [![build status](https://secure.travis-ci.org/jprichardson/node-batchflow.png)](http://travis-ci.org/jprichardson/node-batchflow) 6 | 7 | Batch process collections in parallel or sequentially. 8 | 9 | 10 | Why? 11 | ---- 12 | 13 | I really got tired of writing the following patterns over and over again: 14 | 15 | **Sequential Iteration:** 16 | 17 | ```javascript 18 | var files = [... list of files ...] 19 | function again(x) { 20 | if (x < files.length) { 21 | fs.readFile(files[x], function(err, data) { 22 | //... do something with data ... 23 | again(x + 1) 24 | }) 25 | } else { 26 | console.log('Done.') 27 | } 28 | } 29 | 30 | again(0) 31 | ``` 32 | 33 | or.. 34 | 35 | **Parallel Iteration:** 36 | 37 | ```javascript 38 | var files = [... list of files ...] 39 | var pending = 0 40 | files.forEach(function(file, i) { 41 | pending += 1 42 | fs.readFile(file, function(err, data) { 43 | //... do something with data .... 44 | pending -= 1 45 | if (pending === 0 && i === files.length -1) { 46 | console.log('Done.') 47 | } 48 | }) 49 | }) 50 | ``` 51 | 52 | That's ugly. For more complicated examples it requires a bit more thinking. 53 | 54 | Why don't I use the wonderful library [async][1]? Well, `async` tries to do way too much and its `async` syntax is also very ugly. 55 | 56 | 57 | 58 | Installation 59 | ------------ 60 | 61 | npm install --save batchflow 62 | 63 | 64 | 65 | Examples 66 | -------- 67 | 68 | ### 50 Foot Overview 69 | 70 | These examples demonstrate how to process collections sequentially or in parallel. 71 | 72 | Simple Sequential Example: 73 | 74 | ```javascript 75 | var a = [ 76 | function(finished) { setTimeout(function(){finished()}, 1); }, //executes in 1 ms 77 | function(finished) { setTimeout(function(){finished()}, 20); }, //executes in 20 ms 78 | function(finished) { setTimeout(function(){finished()}, 2); } //executes in 2 ms 79 | ]; 80 | 81 | //sequential 82 | var results = [] 83 | batch(a).sequential() 84 | .each(function(i, fn, done) { 85 | fn(function(){ results.push(i) }) //executes sequentially after timeout 86 | }) 87 | .end(function() { 88 | console.dir(results) //[ 0, 1, 2 ] 89 | }) 90 | ``` 91 | 92 | Simple Parallel Example: 93 | 94 | ```javascript 95 | //parallel 96 | var results = [] 97 | batch(a).parallel() 98 | .each(function(i, item, done) { 99 | fn(function() { results.push(i) }) 100 | }).end(function() { 101 | console.dir(results) //[ 0, 2, 1 ] 102 | }) 103 | ``` 104 | 105 | ### Arrays 106 | 107 | Let's rewrite the previous file patterns mentioned in **Why?** into a sequential example: 108 | 109 | **Sequential:** 110 | 111 | ```javascript 112 | var batch = require('batchflow') 113 | 114 | var files = [... list of files ...]; 115 | batch(files).sequential() 116 | .each(function(i, item, next) { 117 | fs.readFile(item, function(err, data) { 118 | //do something with data 119 | next() 120 | }) 121 | }).end(function() { 122 | //analyze results 123 | }) 124 | ``` 125 | 126 | How about the parallel example? 127 | 128 | **Parallel:** 129 | 130 | ```javascript 131 | var batch = require('batchflow'); 132 | 133 | var files = [... list of files ...]; 134 | batch(files).parallel() 135 | .each(function(i, item, done) { 136 | fs.readFile(item, function(err, data) { 137 | //do something with data 138 | done(someResult); //<---- yes, you must still call done() in parallel, this way we can know when to trigger `end()`. 139 | }); 140 | }).end(function(results) { 141 | //analyze results 142 | }); 143 | ``` 144 | 145 | What's that, your data is not stored in an array? Oh, you say it's stored in an object? That's OK too... 146 | 147 | ### Objects 148 | 149 | **Sequential:** 150 | 151 | ```javascript 152 | var batch = require('batchflow'); 153 | 154 | var files = {'file1': 'path'.... 'filen': 'pathn'} 155 | batch(files).sequential() 156 | .each(function(key, val, next) { 157 | fs.readFile(val, function(err, data) { 158 | //do something with data 159 | next(someResult); 160 | }); 161 | }).end(function(results) { 162 | //analyze results 163 | }); 164 | ``` 165 | 166 | How about the parallel example? 167 | 168 | **Parallel:** 169 | 170 | ```javascript 171 | var batch = require('batchflow'); 172 | 173 | var files = {'file1': 'path'.... 'filen': 'pathn'} 174 | batch(files).parallel() 175 | .each(function(key, val, done) { 176 | fs.readFile(val, function(err, data) { 177 | //do something with data 178 | done(someResult); 179 | }); 180 | }).end(function(results) { 181 | //analyze results 182 | }); 183 | ``` 184 | 185 | ### Misc 186 | 187 | 1. Is `sequential()` or `parallel()` too long? Fine. `series()` and `seq()` are aliases for `sequential()`. `par()` is an alias for `parallel()`. 188 | 2. You don't like the fluent API? That's OK too: 189 | 190 | Non-fluent API BatchFlow 191 | 192 | ```javascript 193 | var batch = require('batchflow'); 194 | var bf = batch(files); 195 | bf.sequential() 196 | 197 | bf.each(function(i, file, next) { 198 | next(someResult); 199 | }); 200 | 201 | bf.end(function(results) { 202 | //blah blah 203 | }); 204 | ``` 205 | 206 | 207 | ### CoffeeScript Friendly 208 | 209 | ```coffee 210 | batch = require('batchflow') 211 | files = [... list of files ...] 212 | bf = batch(files).seq().each (i, file, done) -> 213 | fs.readFile file, done 214 | bf.error (err) -> 215 | console.log(err); 216 | bf.end (results) -> 217 | console.log fr.toString() for fr in results 218 | ``` 219 | 220 | 221 | ### Error Handling 222 | 223 | What's that, you want error handling? Well, you might as well call me Burger King... have it your way. 224 | 225 | Note that before version `0.3`, it would exit prematurely if an error happened. This was a boneheaded 226 | design decision. After `0.3`, it'll keep happily processing even if an error occured. 227 | 228 | Catch an error in the callback parameter... 229 | 230 | ```javascript 231 | var a = {'f': '/tmp/file_DOES_NOT_exist_hopefully' + Math.random()}; 232 | batch(a).parallel().each(function(i, item, done) { 233 | fs.readFile(item, done); 234 | }).error(function(err) { 235 | console.error(err); 236 | }).end(function(fileData) { 237 | //do something with file data 238 | }); 239 | ``` 240 | 241 | Catch an error in the function... 242 | 243 | ```javascript 244 | var a = ['/tmp/file_DOES_NOT_exist_hopefully' + Math.random()]; 245 | batch(a).series().each(function(i, item, done) { 246 | throw new Error('err'); 247 | }).error(function(err) { 248 | console.error(err); 249 | }).end(function() { 250 | //do something 251 | }); 252 | 253 | ``` 254 | 255 | 256 | ### Limits 257 | 258 | You can set a limit to how many items can be processed in parallel. In fact, `sequential` mode is the same as having the `limit` set to `1` and calling `parallel`. In other words: `batch(myArray).sequential() ....` is the same as `batch(myArray).parallel(1)`. 259 | 260 | To set the limit, just pass the limit as a parameter to `parallel()`. The **default is 2^53** which is the max integer size in JavaScript. 261 | 262 | Example: 263 | 264 | ```javascript 265 | batch(myArray).parallel(8) 266 | .each(function(i, item, done){ 267 | // ... code here ... 268 | }).end(function(results){ 269 | // ... code here ... 270 | }) 271 | ``` 272 | 273 | 274 | ### Difference between done() and next() 275 | 276 | So you noticed that in all of the examples where I was calling `sequential()` the third parameter is named `next` and in the examples where I was calling `parallel()` the third parameter is named `done`? This is really just a matter of convention. It could be named `fruitypebbles`. But in sequential processing, it makes sense for it to be `next` because you want to process the next one. However, in parallel processing, you want to alert the system that the callback is `done`. 277 | 278 | Sequential... 279 | 280 | ```javascript 281 | batch(myArray).sequential() 282 | .each(function(i, item, next) { 283 | // ... code here ... 284 | }).end(); 285 | ``` 286 | 287 | Parallel... 288 | 289 | ```javascript 290 | batch(myArray).parallel() 291 | .each(function(i, item, done) { 292 | // ... code here ... 293 | }).end(); 294 | ``` 295 | 296 | 297 | ### Progress 298 | 299 | You can keep track of progress by accessing the `finished` field. 300 | 301 | Compute percentage by this formula: `(this.finished / this.total) * 100.0`. 302 | 303 | Example: 304 | 305 | ```javascript 306 | var myar = {w: 'hi', b: 'hello', c: 'hola', z: 'gutentag'} 307 | batch(myar).sequential() 308 | .each(function(i, item, next) { 309 | console.log(this.finished) //the number finished. 310 | console.log(this.total) //4 311 | console.log((this.finished / this.total) * 100.0) //{percent complete} 312 | }) 313 | .end(); 314 | ``` 315 | 316 | 317 | 318 | License 319 | ------- 320 | 321 | (MIT License) 322 | 323 | Copyright 2012, JP 324 | 325 | 326 | 327 | [1]: https://github.com/caolan/async/ 328 | 329 | 330 | 331 | 332 | -------------------------------------------------------------------------------- /lib/batchflow.js: -------------------------------------------------------------------------------- 1 | if (!global.setImmediate) { //pre Node v0.10 2 | global.setImmediate = process.nextTick 3 | } 4 | 5 | function noop() {}; 6 | 7 | function BatchFlow(collection) { 8 | this.collection = collection; 9 | this.keys = null; 10 | this.eachCallback = null; 11 | this.errorCallback = function(err){ throw err; }; 12 | this.limit = 1; //default to sequential mode 13 | this.finished = 0; //for progress too 14 | this.total = 0; 15 | } 16 | 17 | BatchFlow.prototype.each = function(callback) { 18 | var self = this; 19 | this.eachCallback = function(){ 20 | try { 21 | callback.apply(self, arguments); 22 | } catch (err) { 23 | this.errorCallback(err); 24 | } 25 | } 26 | return this; 27 | } 28 | 29 | BatchFlow.prototype.error = function(callback) { 30 | this.errorCallback = callback; 31 | return this; 32 | } 33 | 34 | BatchFlow.prototype.end = function(endCallback) { 35 | var self = this 36 | , running = 0 37 | , started = 0 38 | , isArray = this.collection instanceof Array 39 | , results = [] 40 | 41 | this.keys = Object.keys(this.collection); //this allows outside modules to directly modify keys and collection. 42 | this.total = this.keys.length; 43 | endCallback = endCallback || noop; 44 | 45 | if (this.total === 0) 46 | return endCallback([]); 47 | 48 | function handleClientResult(err,result) { 49 | self.finished += 1; 50 | if (err instanceof Error) { 51 | self.errorCallback(err); 52 | } else 53 | result = err; //not an error, but the result 54 | 55 | if (result) 56 | results.push(result); 57 | } 58 | 59 | function fireNext(idx, callback) { 60 | //after process.nextTick was added to doneCallback below in 0.3.2, this line might be able to be removed, tests pass with and without it 61 | //if (running + self.finished >= self.total) return //this is critical to fix a regression bug, fireNext may get fired more times than needed 62 | 63 | running += 1; 64 | started += 1; 65 | var key = isArray ? parseInt(self.keys[idx], 10) : self.keys[idx]; 66 | self.eachCallback(key, self.collection[self.keys[idx]], callback); 67 | } 68 | 69 | function doneCallback(err, result) { 70 | setImmediate(function() { 71 | handleClientResult(err, result) 72 | running -= 1; 73 | 74 | if (self.finished < self.total) { 75 | if (started < self.total && running < self.limit) { 76 | fireNext(started, doneCallback) 77 | } 78 | } 79 | else if (self.finished === self.total) { 80 | endCallback(results); 81 | } 82 | }) 83 | } 84 | 85 | var max = self.limit > self.total ? self.total : self.limit; 86 | for (var i = 0; i < max; ++i) { 87 | //this is fired sequentially, but sometimes what's in the each callback is sync and fireNext is called again 88 | //in doneCallback before we even get to the next 'i' in this iteration thus causing fireNext to trigger more times than necessary 89 | fireNext(i, doneCallback) 90 | } 91 | 92 | return this; 93 | }; 94 | 95 | BatchFlow.prototype.parallel = function(limit) { 96 | this.limit = limit || Math.pow(2,53); //2^53 is the largest integer 97 | return this; 98 | } 99 | 100 | BatchFlow.prototype.sequential = function() { 101 | this.limit = 1; 102 | return this; 103 | }; 104 | 105 | BatchFlow.prototype.seq = BatchFlow.prototype.sequential; 106 | BatchFlow.prototype.series = BatchFlow.prototype.sequential; 107 | BatchFlow.prototype.par = BatchFlow.prototype.parallel; 108 | 109 | module.exports = function batch(collection) { 110 | return new BatchFlow(collection); 111 | } 112 | 113 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "batchflow", 3 | "version": "0.4.0", 4 | "description": "Batch process collections in parallel or sequentially.", 5 | "repository": { 6 | "type": "git", 7 | "url": "git@github.com:jprichardson/node-batchflow.git" 8 | }, 9 | "keywords": [ 10 | "batch", 11 | "flow", 12 | "parallel", 13 | "control", 14 | "sequence", 15 | "sequentially", 16 | "arrays", 17 | "async", 18 | "step", 19 | "seq" 20 | ], 21 | "author": "JP ", 22 | "licenses": [ 23 | { 24 | "type": "MIT", 25 | "url": "" 26 | } 27 | ], 28 | "dependencies": {}, 29 | "main": "./lib/batchflow.js", 30 | "scripts": { 31 | "test": "mocha test" 32 | }, 33 | "devDependencies": { 34 | "testutil": "~0.4.0", 35 | "mocha": "*", 36 | "autoresolve": "0.0.3", 37 | "string": "~1.2.0" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /test/batchflow-load.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | 5 | describe('batchflow', function() { 6 | describe('load testing', function() { 7 | it('should pass', function(done) { 8 | var a = [] 9 | for (var i = 0; i < 10000; ++i) 10 | a[i] = i 11 | batch(a).seq() 12 | .each(function(i, val, next) { 13 | next() //<--- before 0.3.2 this would fail 14 | }) 15 | .end(function() { 16 | done() 17 | }) 18 | }) 19 | }) 20 | }) -------------------------------------------------------------------------------- /test/batchflow-par-array.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | 5 | describe('batchflow', function() { 6 | describe('arrays', function() { 7 | var a = [ 8 | function(done) { setTimeout(function() { done(1) }, 1) }, //1 ms 9 | function(done) { setTimeout(function() { done(2) }, 20) }, //20 ms 10 | function(done) { setTimeout(function() { done(3) }, 5) } //2 ms 11 | ]; 12 | 13 | 14 | describe('parallel', function() { 15 | it('should execute async in parallel', function(done) { 16 | var indexes = []; 17 | 18 | batch(a).parallel() 19 | .each(function(i, item, next) { 20 | indexes.push(i); 21 | item(next); 22 | }).end(function(results) { 23 | console.dir(results) 24 | T (results[0] === 1) 25 | T (results[1] === 3) 26 | T (results[2] === 2) 27 | 28 | for (var i = 0; i < indexes.length; ++i) { 29 | T (indexes[i] === i); 30 | } 31 | 32 | T (indexes.length === 3) 33 | 34 | done() 35 | }); 36 | }) 37 | 38 | it('should execute sync in parallel', function(done) { 39 | var t = [1,2,3]; 40 | batch(t).parallel() 41 | .each(function(i, token, next) { 42 | next(token*2); 43 | }) 44 | .end(function(results){ 45 | T (results[0] === 2) 46 | T (results[1] === 4) 47 | T (results[2] === 6) 48 | done() 49 | }); 50 | }) 51 | 52 | it('should not execute if its empty', function(done) { 53 | batch([]).parallel().each(function(){ 54 | T (false) 55 | }).end(function(results){ 56 | done() 57 | }) 58 | }) 59 | 60 | it('should have an empty results array when nothing is passed to next', function(done) { 61 | batch(['hello']).parallel() 62 | .each(function(i, item, next) { 63 | next() 64 | }).end(function(results) { 65 | T (results.length === 0) 66 | done(); 67 | }); 68 | }) 69 | }) 70 | }) 71 | }) 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /test/batchflow-par-limit.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | 5 | describe('batchflow', function() { 6 | describe('parallel limits', function() { 7 | var a = [ 8 | function(done) { setTimeout(function() { done(1) }, 15) }, 9 | function(done) { setTimeout(function() { done(2) }, 30) }, 10 | function(done) { setTimeout(function() { done(3) }, 5) }, 11 | function(done) { setTimeout(function() { done(4) }, 12) } 12 | ]; 13 | 14 | 15 | describe('when no limit is set', function() { 16 | it('should execute async in parallel', function(done) { 17 | var indexes = []; 18 | 19 | batch(a).parallel() 20 | .each(function(i, item, next) { 21 | indexes.push(i); 22 | item(next); 23 | }).end(function(results) { 24 | T (results[0] === 3) 25 | T (results[1] === 4) 26 | T (results[2] === 1) 27 | T (results[3] === 2) 28 | 29 | for (var i = 0; i < indexes.length; ++i) { 30 | T (indexes[i] === i); 31 | } 32 | 33 | T (indexes.length === 4) 34 | 35 | done() 36 | }); 37 | 38 | }) 39 | }) 40 | 41 | describe('when a limit is set', function() { 42 | it('should only execute the number specified by the limit async in parallel', function(done) { 43 | var indexes = []; 44 | 45 | batch(a).parallel(2) 46 | .each(function(i, item, next) { 47 | indexes.push(i); 48 | item(next); 49 | }).end(function(results) { 50 | //console.log(results) 51 | //exit() 52 | T (results[0] === 1) 53 | T (results[1] === 3) 54 | T (results[2] === 2) 55 | T (results[3] === 4) 56 | 57 | for (var i = 0; i < indexes.length; ++i) { 58 | T (indexes[i] === i); 59 | } 60 | 61 | T (indexes.length === 4) 62 | 63 | done() 64 | }); 65 | 66 | }) 67 | }) 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /test/batchflow-par-object.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | 5 | describe('batchflow', function() { 6 | describe('objects', function() { 7 | var o = { 8 | 'a': function(done) { setTimeout(function() { done(1) }, 1) }, //1 ms 9 | 'b': function(done) { setTimeout(function() { done(2) }, 20) }, //20 ms 10 | 'c': function(done) { setTimeout(function() { done(3) }, 5) } //3 ms 11 | }; 12 | 13 | describe('parallel', function() { 14 | it('should execute async in parallel', function(done) { 15 | var keys = []; 16 | 17 | batch(o).parallel() 18 | .each(function(key, val, next) { 19 | keys.push(key); 20 | val(next); 21 | }).end(function(results) { 22 | T (results[0] === 1) 23 | T (results[1] === 3) 24 | T (results[2] === 2) 25 | 26 | T (keys[0] === 'a') 27 | T (keys[1] === 'b') 28 | T (keys[2] === 'c') 29 | 30 | T (results.length === 3) 31 | T (keys.length === 3) 32 | 33 | done(); 34 | }) 35 | 36 | }) 37 | 38 | it('should execute sync in parallel', function(done) { 39 | var t = {'a': 1, 'b': 2, 'c': 3}; 40 | 41 | batch(t).parallel() 42 | .each(function(i, token, next) { 43 | next(token*2) 44 | }) 45 | .end(function(results){ 46 | T (results[0] === 2) 47 | T (results[1] === 4) 48 | T (results[2] === 6) 49 | done() 50 | }) 51 | }) 52 | 53 | it('should not execute if its empty', function(done) { 54 | batch({}).parallel() 55 | .each(function(){ 56 | T (false) 57 | }).end(function(results){ 58 | done() 59 | }) 60 | }) 61 | 62 | it('should have an empty results array when nothing is passed to next', function(done) { 63 | batch({'a': 'hello'}).parallel() 64 | .each(function(i, item, next) { 65 | next(); 66 | }).end(function(results) { 67 | T (results.length === 0) 68 | done(); 69 | }) 70 | }) 71 | }) 72 | }) 73 | }) 74 | 75 | -------------------------------------------------------------------------------- /test/batchflow-seq-array.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | 5 | describe('batchflow', function() { 6 | describe('arrays', function() { 7 | var a = [ 8 | function(done) { setTimeout(function() { done(1) }, 1) }, //1 ms 9 | function(done) { setTimeout(function() { done(2) }, 20) }, //20 ms 10 | function(done) { setTimeout(function() { done(3) }, 2) } //2 ms 11 | ]; 12 | 13 | describe('sequential', function() { 14 | it('should execute async sequentially', function(done) { 15 | var indexes = []; 16 | 17 | batch(a).sequential() 18 | .each(function(i, item, next) { 19 | indexes.push(i); 20 | item(next); 21 | }).end(function(results) { 22 | T (results[0] === 1) 23 | T (results[1] === 2) 24 | T (results[2] === 3) 25 | 26 | for (var i = 0; i < indexes.length; ++i) { 27 | T (indexes[i] === i) 28 | } 29 | 30 | T (indexes.length === 3) 31 | done() 32 | }); 33 | }) 34 | 35 | it('should execute sync sequentially', function(done) { 36 | var t = [1,2,3]; 37 | 38 | batch(t).sequential() 39 | .each(function(i, token, next) { 40 | next(token*2); 41 | }) 42 | .end(function(results){ 43 | T (results[0] === 2) 44 | T (results[1] === 4) 45 | T (results[2] === 6) 46 | done(); 47 | }); 48 | }) 49 | 50 | it('should not execute if its empty', function(done) { 51 | batch([]).sequential() 52 | .each(function(){ 53 | T (false) 54 | }).end(function(results){ 55 | done(); 56 | }) 57 | }) 58 | 59 | it('should have an empty results array when nothing is passed to next', function(done) { 60 | batch(['hello']).sequential() 61 | .each(function(i, item, next) { 62 | next(); 63 | }).end(function(results) { 64 | T (results.length === 0) 65 | done(); 66 | }); 67 | }) 68 | }) 69 | }) 70 | }) 71 | 72 | 73 | -------------------------------------------------------------------------------- /test/batchflow-seq-object.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | 5 | describe('batchflow', function() { 6 | describe('objects', function() { 7 | var o = { 8 | 'a': function(done) { setTimeout(function() { done(1) }, 1) }, 9 | 'b': function(done) { setTimeout(function() { done(2) }, 20) }, 10 | 'c': function(done) { setTimeout(function() { done(3) }, 2) } 11 | }; 12 | 13 | describe('sequential', function() { 14 | it('should execute async sequentially', function(done) { 15 | var keys = []; 16 | 17 | batch(o).sequential() 18 | .each(function(key, val, next) { 19 | keys.push(key); 20 | val(next); 21 | }).end(function(results) { 22 | T (results[0] === 1) 23 | T (results[1] === 2) 24 | T (results[2] === 3) 25 | 26 | T (keys[0] === 'a') 27 | T (keys[1] === 'b') 28 | T (keys[2] === 'c') 29 | 30 | T (results.length === 3) 31 | T (keys.length === 3) 32 | 33 | done(); 34 | }); 35 | 36 | }) 37 | 38 | it('should execute sync sequentially', function(done) { 39 | var t = {'a': 1, 'b': 2, 'c': 3}; 40 | 41 | batch(t).sequential() 42 | .each(function(i, token, next) { 43 | next(token*2); 44 | }) 45 | .end(function(results){ 46 | T (results[0] === 2); 47 | T (results[1] === 4); 48 | T (results[2] === 6); 49 | done() 50 | }) 51 | 52 | }) 53 | 54 | it('should not execute if its empty', function(done) { 55 | batch({}).sequential() 56 | .each(function(){ 57 | T (false) 58 | }).end(function(results){ 59 | done() 60 | }) 61 | 62 | }) 63 | 64 | it('should have an empty results array when nothing is passed to next', function(done) { 65 | batch({'a': 'hello'}).sequential() 66 | .each(function(i, item, next) { 67 | next(); 68 | }).end(function(results) { 69 | T (results.length === 0) 70 | done(); 71 | }); 72 | }) 73 | 74 | }) 75 | }) 76 | }) 77 | 78 | -------------------------------------------------------------------------------- /test/batchflow.test.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | , testutil = require('testutil') 3 | , batch = require('../lib/batchflow') 4 | , fs = require('fs'); 5 | 6 | describe('batchflow', function() { 7 | 8 | 9 | describe('error handling', function() { 10 | it('should call the series-array error handler', function(done) { 11 | var a = ['/tmp/file_DOES_NOT_exist_hopefully' + Math.random()]; 12 | batch(a).series().each(function(i, item, done) { 13 | fs.readFile(item, done); 14 | }).error(function(err) { 15 | assert(err); 16 | done(); 17 | }).end(function() { 18 | //assert(false); //<--- shouldn't get here 19 | }); 20 | }); 21 | 22 | it('should call the parallel-array error handler', function(done) { 23 | var a = ['/tmp/file_DOES_NOT_exist_hopefully' + Math.random()]; 24 | batch(a).parallel().each(function(i, item, done) { 25 | fs.readFile(item, done); 26 | }).error(function(err) { 27 | assert(err); 28 | done(); 29 | }).end(function() { 30 | //assert(false); //<--- shouldn't get here 31 | }); 32 | }); 33 | 34 | it('should call the series-obj error handler', function(done) { 35 | var a = {'f': '/tmp/file_DOES_NOT_exist_hopefully' + Math.random()}; 36 | batch(a).series().each(function(i, item, done) { 37 | fs.readFile(item, done); 38 | }).error(function(err) { 39 | assert(err); 40 | done(); 41 | }).end(function() { 42 | //assert(false); //<--- shouldn't get here 43 | }); 44 | }); 45 | 46 | it('should call the parallel-obj error handler', function(done) { 47 | var a = {'f': '/tmp/file_DOES_NOT_exist_hopefully' + Math.random()}; 48 | batch(a).parallel().each(function(i, item, done) { 49 | fs.readFile(item, done); 50 | }).error(function(err) { 51 | assert(err); 52 | done(); 53 | }).end(function() { 54 | //assert(false); //<--- shouldn't get here 55 | }); 56 | }); 57 | 58 | it('should throw the series-array error handler', function(done) { 59 | var a = ['/tmp/file_DOES_NOT_exist_hopefully' + Math.random()]; 60 | batch(a).series().each(function(i, item, done) { 61 | throw new Error('err'); 62 | }).error(function(err) { 63 | assert(err); 64 | done(); 65 | }).end(function() { 66 | //assert(false); //<--- shouldn't get here 67 | }); 68 | }); 69 | 70 | it('should throw the parallel-array error handler', function(done) { 71 | var a = ['/tmp/file_DOES_NOT_exist_hopefully' + Math.random()]; 72 | batch(a).parallel().each(function(i, item, done) { 73 | throw new Error('err'); 74 | }).error(function(err) { 75 | assert(err); 76 | done(); 77 | }).end(function() { 78 | //assert(false); //<--- shouldn't get here 79 | }); 80 | }); 81 | 82 | it('should throw the series-obj error handler', function(done) { 83 | var a = {'f': '/tmp/file_DOES_NOT_exist_hopefully' + Math.random()}; 84 | batch(a).series().each(function(i, item, done) { 85 | throw new Error('err'); 86 | }).error(function(err) { 87 | assert(err); 88 | done(); 89 | }).end(function() { 90 | //assert(false); //<--- shouldn't get here 91 | }); 92 | }); 93 | 94 | it('should throw the parallel-obj error handler', function(done) { 95 | var a = {'f': '/tmp/file_DOES_NOT_exist_hopefully' + Math.random()}; 96 | batch(a).parallel().each(function(i, item, done) { 97 | throw new Error('err'); 98 | }).error(function(err) { 99 | assert(err); 100 | done(); 101 | }).end(function() { 102 | //assert(false); //<--- shouldn't get here 103 | }); 104 | }); 105 | }); 106 | 107 | 108 | }); 109 | 110 | 111 | -------------------------------------------------------------------------------- /test/batchflow_regression-file-filter.test.js: -------------------------------------------------------------------------------- 1 | var testutil = require('testutil') 2 | , fs = require('fs') 3 | , batch = require('../lib/batchflow') 4 | , path = require('path') 5 | , P = require('autoresolve') 6 | , S = require('string') 7 | 8 | describe('batchflow', function() { 9 | describe('regression', function() { 10 | describe('> when filtering a directory', function() { 11 | it('should return the file', function(done) { 12 | var dir = P('test/resources/regr-file-filter') 13 | var files = fs.readdirSync(dir).map(function(file) { return path.join(dir, file) }) 14 | 15 | batch(files).par().each(function(i, file, next) { 16 | if (path.extname(file) === '.json') 17 | fs.lstat(file, function (err, stats) { 18 | if (stats.isFile()) 19 | next(file); 20 | }) 21 | else 22 | next(); //is a directory or does not have .json extension 23 | }) 24 | .error(done) 25 | .end(function(results){ 26 | //console.dir(results) 27 | EQ (results.length, 1) 28 | T (S(results[0]).endsWith('package.json')) 29 | done() 30 | }) 31 | }) 32 | }) 33 | }) 34 | }) 35 | 36 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | --ui bdd 3 | --timeout 5000 -------------------------------------------------------------------------------- /test/resources/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprichardson/node-batchflow/ee78cc2009bc5dd70d608b21ef1bd7f975518ef2/test/resources/.gitkeep -------------------------------------------------------------------------------- /test/resources/regr-file-filter/config/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprichardson/node-batchflow/ee78cc2009bc5dd70d608b21ef1bd7f975518ef2/test/resources/regr-file-filter/config/.gitkeep -------------------------------------------------------------------------------- /test/resources/regr-file-filter/index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprichardson/node-batchflow/ee78cc2009bc5dd70d608b21ef1bd7f975518ef2/test/resources/regr-file-filter/index.js -------------------------------------------------------------------------------- /test/resources/regr-file-filter/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jprichardson/node-batchflow/ee78cc2009bc5dd70d608b21ef1bd7f975518ef2/test/resources/regr-file-filter/package.json --------------------------------------------------------------------------------