├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── lib └── index.js ├── package.json └── test ├── index.js └── lib └── index.js /.gitignore: -------------------------------------------------------------------------------- 1 | out 2 | 3 | # Logs 4 | logs 5 | *.log 6 | 7 | # Runtime data 8 | pids 9 | *.pid 10 | *.seed 11 | 12 | # Directory for instrumented libs generated by jscoverage/JSCover 13 | lib-cov 14 | 15 | # Coverage directory used by tools like istanbul 16 | coverage 17 | 18 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 19 | .grunt 20 | 21 | # Compiled binary addons (http://nodejs.org/api/addons.html) 22 | build/Release 23 | 24 | # Dependency directory 25 | # Commenting this out is preferred by some people, see 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 27 | node_modules 28 | 29 | # Users Environment Variables 30 | .lock-wscript 31 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | tests 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Bradley Meck 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # generator-runner 2 | 3 | A function scheduler to allow generators to perform asynchronous task scheduling. 4 | 5 | ```javascript 6 | require('generator-runner')(function* task(step) { 7 | // 8 | // yield to async tasks ... 9 | // 10 | return 'ret'; 11 | }, function (err, ret) { 12 | // err is truthy if an error was thrown 13 | // ret is our generator's return value 14 | }); 15 | ``` 16 | 17 | In order to perform async tasks we use `yield $task`. 18 | 19 | ```javascript 20 | require('generator-runner')(function* task(step) { 21 | let async_task_result = yield async_task; 22 | }); 23 | ``` 24 | 25 | If an async task produces an exception we throw an error at that point and we 26 | can use a `try`/`catch` block to handle asynchronous errors. 27 | 28 | ```javascript 29 | require('generator-runner')(function* task(step) { 30 | try { 31 | let async_task_result = yield async_task; 32 | } 33 | catch (e) { 34 | if (cant_recover(e)) throw e; 35 | else { 36 | recover_from(e); 37 | } 38 | } 39 | }, function (e) { 40 | // any thrown error will stop execution and invoke this callback 41 | }); 42 | ``` 43 | 44 | This is more important than merely `try`/`catch` because we can achieve finally 45 | blocks for cleanup. 46 | 47 | ```javascript 48 | let file = 'test.txt'; 49 | require('generator-runner')(function* task(step) { 50 | let fd = open(file); 51 | try { 52 | // the life of our resource is the length of async_task_with(fd) 53 | yield async_task_with(fd); 54 | } 55 | // we don't need a catch 56 | finally { 57 | // this gets called regardless of success or failure 58 | close(fd); 59 | } 60 | 61 | }); 62 | ``` 63 | 64 | ## function runner 65 | 66 | This performs a node style callback async function. 67 | The function is invoked with a callback to continue the generator. 68 | If the first parameter of the callback is truthy an error is thrown. 69 | Otherwise, the second value of the callback is treated as a return value. 70 | 71 | ```javascript 72 | value = yield function (next/*(err, value)*/) { 73 | // perform async and call next 74 | } 75 | ``` 76 | 77 | * causes an error at the point of yield if `err` is truthy 78 | * returns the `value` for the yield expression 79 | 80 | ## disjoint function runner 81 | 82 | This uses the `step` function passed into the generator to continue the 83 | generator. 84 | This is very important for evented style programming where an event listener 85 | has been setup previously and we are just waiting for an event to come in. 86 | 87 | ```javascript 88 | setTimeout(step, 0); 89 | yield step // tell the runner to wait on step to be called 90 | // setTimeout fires after the yield, and the generator continues 91 | ``` 92 | 93 | ## promise runner 94 | 95 | Promises map almost directly onto generator's `.next` and `.throw` methods. 96 | Promises do not start any work since they are a data structure only; 97 | instead they will cause an error if rejected or return a value if fulfilled. 98 | 99 | ``` 100 | yield my_promise; 101 | ``` 102 | 103 | * throw an error on rejection 104 | * return a value on fulfillment 105 | 106 | ## generator instance running 107 | 108 | Generator instances are a series of tasks to be performed. 109 | We can temporarily give control to another generator to perform tasks 110 | for our current generator. 111 | 112 | This is not the same as using `yield*` because it produces a step guard 113 | (see below). 114 | 115 | ```javascript 116 | function* subtask() { 117 | yield function (next) { setTimeout(next); }; 118 | } 119 | yield subtask(); 120 | ``` 121 | 122 | ## parallel array runner 123 | 124 | Yielding an array will result in all the values being run in parallel. 125 | The values follow all of the runners listed here. 126 | Order of values is preserved so you do not need to worry about which task ends 127 | first. 128 | 129 | ```javascript 130 | let [timeout, promise_result] = yield [ 131 | _ => setTimeout(_, 1e3), // wait a second 132 | my_promise 133 | ]; 134 | ``` 135 | 136 | 137 | ## parallel object runner 138 | 139 | Similar to arrays when an object is yielded it will produce parallel tasks. 140 | 141 | ```javascript 142 | let timeouts = yield { 143 | timeout1: _ => setTimeout(_) 144 | timeout2: _ => setTimeout(_) 145 | }; 146 | ``` 147 | 148 | ## nesting 149 | 150 | *Any* valid value can be placed inside of our parallel object runners; 151 | we can setup specific parallelism using nesting. 152 | 153 | ```javascript 154 | let [timeouts, promises] = yield [ 155 | [ 156 | _ => setTimeout(_), 157 | _ => setTimeout(_) 158 | ], 159 | [ 160 | my_promise1, 161 | my_promise2 162 | ] 163 | ]; 164 | ``` 165 | 166 | ## racing 167 | 168 | Although not complex, the ability to race for whatever the first task to finish 169 | is a common work flow. 170 | The race function will let you do this. 171 | It accepts either Arrays or Objects, though it will not preserve which index was 172 | the one to finish first. 173 | 174 | ```javascript 175 | let race = require('generator-runner').race; 176 | yield race({ 177 | timeout: _ => setTimeout(() => _(new Error('timeout!'), 120 * 1000), 178 | exit: _ => child.on('exit', _), 179 | error: _ => child.on('error', _) 180 | }); 181 | ``` 182 | 183 | ## limited concurrency 184 | 185 | Doing limited concurrency is actually a lot of book keeping; 186 | we provide a simple function for making your parallel tasks easier. 187 | 188 | **NOTE** this only works with array style parallelism. 189 | 190 | ```javascript 191 | let concurrency = require('generator-runner').concurrent; 192 | yield concurrency(2, [ 193 | _ => setTimeout(_), 194 | _ => setTimeout(_), 195 | _ => setTimeout(_), 196 | _ => setTimeout(_), 197 | _ => setTimeout(_), 198 | _ => setTimeout(_), 199 | _ => setTimeout(_) 200 | ]); 201 | ``` 202 | 203 | ## step guards 204 | 205 | While an async task or parallel task is running any attempt at disjoint 206 | stepping will result in an error to prevent race conditions. 207 | 208 | ```javascript 209 | // this will fire before our async task 210 | setTimeout(function () { 211 | try { 212 | // this throws an error because we are not waiting on `step` 213 | step(); 214 | } 215 | catch (e) { 216 | console.log(`I'm busy~`); 217 | } 218 | }, 0); 219 | yield _ => setTimeout(_, 1e3); 220 | ``` 221 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | function toCallback(value, options) { 2 | if (value && typeof value.next == 'function') { 3 | return function (next) { 4 | runner(function (step) { 5 | return value; 6 | }, next); 7 | } 8 | } 9 | if (typeof value === 'function') { 10 | return value; 11 | } 12 | else if (value && typeof value.then === 'function') { 13 | return function (next) { 14 | value.then( 15 | function (v) {next(null, v)}, 16 | function (e) {next(e, null)} 17 | ) 18 | }; 19 | } 20 | else if (Array.isArray(value)) { 21 | var steps = value.map(toCallback); 22 | var results = new Array(steps.length); 23 | // GC cleanup 24 | value = null; 25 | var todo = steps.length; 26 | return function (next) { 27 | function tick(err) { 28 | if (todo <= 0) return; 29 | if (err) { 30 | todo = 0; 31 | next(err); 32 | return; 33 | } 34 | todo--; 35 | if (todo == 0) { 36 | next(null, results); 37 | } 38 | } 39 | steps.forEach(function (step, i) { 40 | var done = false; 41 | step(function (err, result) { 42 | if (done) return; 43 | done = true; 44 | results[i] = result; 45 | tick(err); 46 | }) 47 | }); 48 | } 49 | } 50 | else if (value && typeof value === 'object') { 51 | var steps = []; 52 | var results = Object.create(null); 53 | var todo = 0; 54 | for (var key in value) { 55 | !function (key) { 56 | todo += 1; 57 | steps.push(function (next) { 58 | toCallback(value[key])(function (err, value) { 59 | results[key] = value; 60 | next(err); 61 | }); 62 | }); 63 | }(key); 64 | } 65 | return function (next) { 66 | function tick(err) { 67 | if (todo <= 0) return; 68 | if (err) { 69 | todo = 0; 70 | next(err); 71 | return; 72 | } 73 | todo--; 74 | if (todo == 0) { 75 | next(null, results); 76 | } 77 | } 78 | 79 | for (var i = 0; i < steps.length; i++) { 80 | steps[i](tick); 81 | } 82 | } 83 | } 84 | else { 85 | throw new Error('cannot convert to callback'); 86 | } 87 | } 88 | function runner(generator_fn, cb) { 89 | var done; 90 | var first = true; 91 | var waiting_already = false; 92 | function abort(err) { 93 | if (done) return; 94 | if (!err) throw new Error('cannot abort without error'); 95 | done = true; 96 | first = false; 97 | waiting_already = true; 98 | setImmediate(function () { 99 | cb && cb(err) 100 | }); 101 | } 102 | function step(err, val) { 103 | var result; 104 | var value; 105 | // yes, throw. 106 | // we are running in other people's code at the time 107 | if (done) throw new Error('already done'); 108 | if (waiting_already) throw new Error('already waiting'); 109 | try { 110 | if (err) { 111 | result = generator.throw(err); 112 | } 113 | else { 114 | // lol es-discuss 115 | if (first) { 116 | first = false; 117 | result = generator.next(); 118 | } 119 | else result = generator.next(val); 120 | } 121 | // someone aborted 122 | if (done) return; 123 | value = result.value; 124 | done = result.done; 125 | } 126 | catch (e) { 127 | done = true; 128 | setImmediate(function () { 129 | cb && cb(e, null) 130 | }); 131 | return; 132 | } 133 | if (done) { 134 | setImmediate(function () { 135 | cb && cb(null, value) 136 | }); 137 | } 138 | else if (value == step) { 139 | // do nothing, we were told that the generator has stuff setup; 140 | } 141 | else if (value == null) { 142 | step(null, undefined); 143 | } 144 | else { 145 | waiting_already = true; 146 | var fn; 147 | try { 148 | fn = toCallback(value); 149 | } 150 | catch (e) { 151 | done = true; 152 | setImmediate(function () { 153 | cb && cb(e, null); 154 | }); 155 | return; 156 | } 157 | var performed = false; 158 | fn(function () { 159 | if (done) throw new Error('already done'); 160 | var args = Array.prototype.slice.call(arguments); 161 | if (performed) throw new Error('already performed this step'); 162 | performed = true; 163 | waiting_already = false; 164 | setImmediate(function () { 165 | step.apply(null, args) 166 | }); 167 | }); 168 | } 169 | } 170 | var generator = generator_fn(step, abort); 171 | setImmediate(step); 172 | } 173 | exports = module.exports = runner; 174 | 175 | function race(tasks) { 176 | return function (cb) { 177 | var done = false; 178 | function end() { 179 | if (done) return; 180 | done = true; 181 | cb.apply(this, arguments); 182 | } 183 | if (Array.isArray(tasks)) { 184 | for (var i = 0; i < tasks.length; i++) { 185 | tasks[i](end); 186 | } 187 | } 188 | else { 189 | for (var k in tasks) { 190 | tasks[k](end); 191 | } 192 | } 193 | } 194 | } 195 | module.exports.race = race; 196 | 197 | function concurrent(concurrency, tasks) { 198 | return function $concurrent(cb) { 199 | var i = 0; 200 | var done = false; 201 | var results = []; 202 | var queued = 0; 203 | function next() { 204 | if (i < tasks.length) { 205 | var task = tasks[i]; 206 | i += 1; 207 | queued += 1; 208 | task(function (err, value) { 209 | if (err) { 210 | done = true; 211 | setImmediate(function () { cb && cb(err, null); } ); 212 | } 213 | else { 214 | results[i] = value; 215 | queued -= 1; 216 | if (i < tasks.length) { 217 | next(); 218 | return; 219 | } 220 | if (queued) return; 221 | done = true; 222 | setImmediate(function () { cb && cb(null, results); }); 223 | } 224 | }); 225 | } 226 | } 227 | for (var ii = concurrency; ii > 0; ii--) { 228 | next(); 229 | } 230 | } 231 | } 232 | module.exports.concurrent = concurrent; 233 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "generator-runner", 3 | "version": "2.2.0", 4 | "description": "like a more generic task.js", 5 | "main": "lib/", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "devDependencies": { 10 | "6to5": "^3.3.3", 11 | "source-map-support": "^0.2.9" 12 | }, 13 | "scripts": { 14 | "pretest": "6to5 ./test/lib --out-dir ./test/out --source-maps", 15 | "test": "node test/" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "https://github.com/bmeck/generator-runner.git" 20 | }, 21 | "keywords": [ 22 | "runner", 23 | "task", 24 | "generator", 25 | "async", 26 | "control", 27 | "flow" 28 | ], 29 | "author": "bradleymeck", 30 | "license": "ISC", 31 | "bugs": { 32 | "url": "https://github.com/bmeck/generator-runner/issues" 33 | }, 34 | "homepage": "https://github.com/bmeck/generator-runner" 35 | } 36 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | require('source-map-support'); 2 | require('6to5/polyfill'); 3 | require('./out'); 4 | -------------------------------------------------------------------------------- /test/lib/index.js: -------------------------------------------------------------------------------- 1 | var runner = require('../../'); 2 | var assert = require('assert'); 3 | 4 | var counted = {}; 5 | function counter(k, fn, ...prefixargs) { 6 | counted[k] = counted[k] || 0; 7 | counted[k]++; 8 | console.log(k); 9 | return fn.apply(this, prefixargs); 10 | } 11 | 12 | var ret; 13 | function* task(step, abort) { 14 | let disjoint_ret = {}; 15 | setTimeout(_ => counter('disjoint', step, null, disjoint_ret)); 16 | let disjoint_out = yield step; 17 | assert(disjoint_ret == disjoint_out); 18 | 19 | setTimeout(_ => counter('disjoint', step, null, disjoint_ret)); 20 | disjoint_out = yield step; 21 | assert(disjoint_ret == disjoint_out); 22 | 23 | let callback_ret = {}; 24 | let callback_out = yield _ => setTimeout($ => counter('callback', _, null, callback_ret)); 25 | assert(callback_out == callback_ret); 26 | 27 | 28 | let parallel_arr_ret = [{},{}]; 29 | let parallel_arr_out = yield [ 30 | _ => setTimeout($ => counter('parallel_arr', _, null, parallel_arr_ret[0])), 31 | _ => setTimeout($ => counter('parallel_arr', _, null, parallel_arr_ret[1])) 32 | ] 33 | assert(parallel_arr_ret.length == parallel_arr_out.length); 34 | parallel_arr_out.forEach((v,i) => { 35 | assert(v == parallel_arr_ret[i]); 36 | }); 37 | 38 | let parallel_key_ret = { 39 | a: {}, 40 | b: {} 41 | }; 42 | let parallel_key_out = yield { 43 | a: _ => setTimeout($ => counter('parallel_key', _, null, parallel_key_ret.a)), 44 | b: _ => setTimeout($ => counter('parallel_key', _, null, parallel_key_ret.b)), 45 | c: _ => setTimeout($ => counter('parallel_key', _, null, parallel_key_ret.c)) 46 | } 47 | for (var k in parallel_key_ret) { 48 | assert(parallel_key_ret[k] == parallel_key_out[k], k + ' to be the same'); 49 | } 50 | // need to check both (easier than checking keys are the same in case of prototype changes) 51 | for (var k in parallel_key_out) { 52 | assert(parallel_key_ret[k] == parallel_key_out[k], k + ' to be the same'); 53 | } 54 | 55 | let subtask_ret = {}; 56 | let subtask = function* () { 57 | yield _ => setTimeout($ => counter('subtask', _, null)); 58 | yield _ => setTimeout($ => counter('subtask', _, null)); 59 | yield _ => setTimeout($ => counter('subtask', _, null)); 60 | yield _ => setTimeout($ => counter('subtask', _, null)); 61 | return subtask_ret; 62 | } 63 | let subtask_out = yield subtask(); 64 | assert(subtask_out == subtask_ret); 65 | 66 | // check the guard 67 | yield _ => { 68 | try { 69 | step(); 70 | } 71 | catch (e) { 72 | assert(e); 73 | counter('guard', _); 74 | return; 75 | } 76 | assert(false); 77 | }; 78 | 79 | // test skipping 80 | yield null; 81 | yield undefined; 82 | 83 | // test timing consistency 84 | let done = 0 85 | yield [ 86 | _ => {done+=1;_()}, 87 | _ => {done+=1;_()} 88 | ] 89 | assert(done == 2); 90 | 91 | // test concurrency runner 92 | let running = 0; 93 | let concurrency = 2; 94 | 95 | function run(_) { 96 | running += 1; 97 | assert(running <= concurrency); 98 | setTimeout($ => { 99 | running -= 1; 100 | counter('concurrent', _, null); 101 | }); 102 | } 103 | yield runner.concurrent(concurrency, [ 104 | run, 105 | run, 106 | run, 107 | run, 108 | run, 109 | run, 110 | run, 111 | run, 112 | run, 113 | run, 114 | run 115 | ]); 116 | 117 | let race_turtle = new Promise((f,r) => setTimeout($ => counter('race', f), 100)); 118 | let race_ret = {}; 119 | let race_out = yield runner.race({ 120 | fast: _ => setTimeout($ => counter('race', _, null, race_ret)), 121 | slow: _ => race_turtle, 122 | never_called: _ => {} 123 | }); 124 | assert(race_ret == race_out); 125 | yield race_turtle; 126 | 127 | return ret; 128 | } 129 | var expected = { 130 | disjoint: 2, 131 | callback: 1, 132 | parallel_arr: 2, 133 | parallel_key: 3, 134 | subtask: 4, 135 | guard: 1, 136 | concurrent: 11, 137 | race: 2 138 | } 139 | var ran = 0; 140 | process.on('exit', _ => assert(ran == 2), 'the test ran'); 141 | // make it run! 142 | runner(task, (err, val) => { 143 | ran += 1; 144 | assert(!err, 'there is not an error: ' + err); 145 | assert(val == ret, 'the return value is the same reference'); 146 | var expectedKeys = Object.keys(expected).sort(); 147 | var countedKeys = Object.keys(counted).sort(); 148 | assert(expectedKeys.length == countedKeys.length, 'the keys have the same length for expected and counted'); 149 | assert(JSON.stringify(expectedKeys) == JSON.stringify(countedKeys), 'the keys for expected and counted are the same values'); 150 | expectedKeys.forEach(k => { 151 | assert(expected[k] == counted[k], 'the ' + k + ' key is the same value for expected and counted'); 152 | }); 153 | countedKeys.forEach(k => { 154 | assert(expected[k] == counted[k], 'the ' + k + ' key is the same value for expected and counted'); 155 | }); 156 | }); 157 | 158 | let abort_ret = {}; 159 | runner(function* abort_task(step, abort) { 160 | try { 161 | abort(); 162 | assert(false, 'abort without a value should throw'); 163 | } 164 | catch (e) { 165 | // haha need a value 166 | assert(e, 'aborting without a value should throw'); 167 | } 168 | let done = false; 169 | setImmediate(_ => { 170 | done = true; 171 | abort(abort_ret); 172 | }); 173 | setImmediate(_ => { 174 | assert(done, 'we should be done already'); 175 | try { 176 | // aborting when we are done is not an error 177 | abort(abort_ret); 178 | } 179 | catch (e) { 180 | assert(false, 'multiple aborts are allowed'); 181 | } 182 | }); 183 | yield _ => setImmediate($ => { 184 | try { 185 | _(); 186 | } 187 | catch (e) { 188 | assert(e, 'continuing after an abort is not possible'); 189 | } 190 | }); 191 | }, function (err) { 192 | ran += 1; 193 | assert(err == abort_ret, 'aborting causes an error'); 194 | }); 195 | --------------------------------------------------------------------------------