├── .gitignore ├── .gitmodules ├── .npmignore ├── CHANGES.md ├── Jenkinsfile ├── LICENSE ├── Makefile ├── Makefile.targ ├── README.md ├── examples ├── barrier-basic.js ├── barrier-readdir.js ├── foreach-parallel.js ├── foreach-pipeline.js ├── nofail.js ├── parallel.js ├── pipeline.js ├── queue-serializer.js ├── queue-stat.js ├── waterfall.js └── whilst.js ├── jsl.node.conf ├── lib └── vasync.js ├── package.json └── tests ├── compat.js ├── compat_tryEach.js ├── filter.js ├── issue-21.js ├── pipeline.js ├── queue.js ├── queue_concurrency.js ├── waterfall.js └── whilst.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "deps/jsstyle"] 2 | path = deps/jsstyle 3 | url = https://github.com/joyent/jsstyle.git 4 | [submodule "deps/javascriptlint"] 5 | path = deps/javascriptlint 6 | url = https://github.com/joyent/javascriptlint.git 7 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | deps 3 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Not yet released 4 | 5 | None yet. 6 | 7 | ## v2.2.0 8 | 9 | * #37 want whilst 10 | 11 | ## v2.1.0 12 | 13 | * #33 want filter, filterLimit, and filterSeries 14 | * #35 pipeline does not pass rv-object to final callback 15 | 16 | 17 | ## v2.0.0 18 | 19 | ** WARNING 20 | 21 | Do not use this version (v2.0.0), as it has broken pipeline and forEachPipeline 22 | functions. 23 | 24 | **Breaking Changes:** 25 | 26 | * The `waterfall` function's terminating callback no longer receives a 27 | status-object as its second argument. This is the behavior of `node-async` 28 | and we wish to match it as closely as possible. If you used the second 29 | argument of waterfall's terminating callback (instead of waterfall's return 30 | value) to extract job-statuses, this will break you. More specifically, this 31 | is only true if you called `waterfall` on an empty array of function. 32 | 33 | **Other Changes:** 34 | 35 | * #32 Would like a tryEach function. 36 | 37 | ## v1 and earlier 38 | 39 | Major version 1 and earlier did not have their changes logged in a changelog. 40 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | @Library('jenkins-joylib@v1.0.8') _ 2 | 3 | pipeline { 4 | 5 | agent none 6 | 7 | options { 8 | buildDiscarder(logRotator(numToKeepStr: '30')) 9 | timestamps() 10 | } 11 | 12 | stages { 13 | stage('top') { 14 | parallel { 15 | stage('v0.10.48-zone') { 16 | agent { 17 | label joyCommonLabels(image_ver: '15.4.1') 18 | } 19 | tools { 20 | nodejs 'sdcnode-v0.10.48-zone' 21 | } 22 | stages { 23 | stage('check') { 24 | steps{ 25 | sh('make check') 26 | } 27 | } 28 | stage('test') { 29 | steps{ 30 | sh('make all test') 31 | } 32 | } 33 | } 34 | } 35 | 36 | stage('v4-zone') { 37 | agent { 38 | label joyCommonLabels(image_ver: '15.4.1') 39 | } 40 | tools { 41 | nodejs 'sdcnode-v4-zone' 42 | } 43 | stages { 44 | stage('check') { 45 | steps{ 46 | sh('make check') 47 | } 48 | } 49 | stage('test') { 50 | steps{ 51 | sh('make all test') 52 | } 53 | } 54 | } 55 | } 56 | 57 | stage('v6-zone64') { 58 | agent { 59 | label joyCommonLabels(image_ver: '18.4.0') 60 | } 61 | tools { 62 | nodejs 'sdcnode-v6-zone64' 63 | } 64 | stages { 65 | stage('check') { 66 | steps{ 67 | sh('make check') 68 | } 69 | } 70 | stage('test') { 71 | steps{ 72 | sh('make all test') 73 | } 74 | } 75 | } 76 | } 77 | } 78 | } 79 | } 80 | 81 | post { 82 | always { 83 | joySlackNotifications() 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Joyent, Inc. All rights reserved. 2 | Compatibility tests copyright (c) 2010-2014 Caolan McMahon. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) 2021, Joyent, Inc. All rights reserved. 3 | # 4 | # Makefile: top-level Makefile 5 | # 6 | # This Makefile contains only repo-specific logic and uses included makefiles 7 | # to supply common targets (javascriptlint, jsstyle, restdown, etc.), which are 8 | # used by other repos as well. 9 | # 10 | 11 | # 12 | # Files 13 | # 14 | JS_FILES := $(shell find lib tests -name '*.js' -not -name compat\*.js) 15 | JSL_FILES_NODE = $(JS_FILES) 16 | JSSTYLE_FILES = $(JS_FILES) 17 | JSL_CONF_NODE = jsl.node.conf 18 | 19 | all: 20 | npm install 21 | 22 | test: all 23 | npm test 24 | 25 | include ./Makefile.targ 26 | -------------------------------------------------------------------------------- /Makefile.targ: -------------------------------------------------------------------------------- 1 | # -*- mode: makefile -*- 2 | # 3 | # Copyright (c) 2021, Joyent, Inc. All rights reserved. 4 | # 5 | # Makefile.targ: common targets. 6 | # 7 | # NOTE: This makefile comes from the "eng" repo. It's designed to be dropped 8 | # into other repos as-is without requiring any modifications. If you find 9 | # yourself changing this file, you should instead update the original copy in 10 | # eng.git and then update your repo to use the new version. 11 | # 12 | # This Makefile defines several useful targets and rules. You can use it by 13 | # including it from a Makefile that specifies some of the variables below. 14 | # 15 | # Targets defined in this Makefile: 16 | # 17 | # check Checks JavaScript files for lint and style 18 | # Checks bash scripts for syntax 19 | # Checks SMF manifests for validity against the SMF DTD 20 | # 21 | # clean Removes built files 22 | # 23 | # docs Builds restdown documentation in docs/ 24 | # 25 | # prepush Depends on "check" and "test" 26 | # 27 | # test Does nothing (you should override this) 28 | # 29 | # xref Generates cscope (source cross-reference index) 30 | # 31 | # For details on what these targets are supposed to do, see the Joyent 32 | # Engineering Guide. 33 | # 34 | # To make use of these targets, you'll need to set some of these variables. Any 35 | # variables left unset will simply not be used. 36 | # 37 | # BASH_FILES Bash scripts to check for syntax 38 | # (paths relative to top-level Makefile) 39 | # 40 | # CLEAN_FILES Files to remove as part of the "clean" target. Note 41 | # that files generated by targets in this Makefile are 42 | # automatically included in CLEAN_FILES. These include 43 | # restdown-generated HTML and JSON files. 44 | # 45 | # DOC_FILES Restdown (documentation source) files. These are 46 | # assumed to be contained in "docs/", and must NOT 47 | # contain the "docs/" prefix. 48 | # 49 | # JSL_CONF_NODE Specify JavaScriptLint configuration files 50 | # JSL_CONF_WEB (paths relative to top-level Makefile) 51 | # 52 | # Node.js and Web configuration files are separate 53 | # because you'll usually want different global variable 54 | # configurations. If no file is specified, none is given 55 | # to jsl, which causes it to use a default configuration, 56 | # which probably isn't what you want. 57 | # 58 | # JSL_FILES_NODE JavaScript files to check with Node config file. 59 | # JSL_FILES_WEB JavaScript files to check with Web config file. 60 | # 61 | # You can also override these variables: 62 | # 63 | # BASH Path to bash (default: bash) 64 | # 65 | # CSCOPE_DIRS Directories to search for source files for the cscope 66 | # index. (default: ".") 67 | # 68 | # JSL Path to JavaScriptLint (default: "jsl") 69 | # 70 | # JSL_FLAGS_NODE Additional flags to pass through to JSL 71 | # JSL_FLAGS_WEB 72 | # JSL_FLAGS 73 | # 74 | # JSSTYLE Path to jsstyle (default: jsstyle) 75 | # 76 | # JSSTYLE_FLAGS Additional flags to pass through to jsstyle 77 | # 78 | 79 | # 80 | # Defaults for the various tools we use. 81 | # 82 | BASH ?= bash 83 | BASHSTYLE ?= tools/bashstyle 84 | CP ?= cp 85 | CSCOPE ?= cscope 86 | CSCOPE_DIRS ?= . 87 | JSL_EXEC ?= deps/javascriptlint/build/install/jsl 88 | JSL ?= $(JSL_EXEC) 89 | JSSTYLE ?= deps/jsstyle/jsstyle 90 | MKDIR ?= mkdir -p 91 | MV ?= mv 92 | RESTDOWN_FLAGS ?= 93 | RMTREE ?= rm -rf 94 | JSL_FLAGS ?= --nologo --nosummary 95 | 96 | ifeq ($(shell uname -s),SunOS) 97 | TAR ?= gtar 98 | else 99 | TAR ?= tar 100 | endif 101 | 102 | 103 | # 104 | # Defaults for other fixed values. 105 | # 106 | BUILD = build 107 | DISTCLEAN_FILES += $(BUILD) 108 | DOC_BUILD = $(BUILD)/docs/public 109 | 110 | # 111 | # Configure JSL_FLAGS_{NODE,WEB} based on JSL_CONF_{NODE,WEB}. 112 | # 113 | ifneq ($(origin JSL_CONF_NODE), undefined) 114 | JSL_FLAGS_NODE += --conf=$(JSL_CONF_NODE) 115 | endif 116 | 117 | ifneq ($(origin JSL_CONF_WEB), undefined) 118 | JSL_FLAGS_WEB += --conf=$(JSL_CONF_WEB) 119 | endif 120 | 121 | # 122 | # Targets. For descriptions on what these are supposed to do, see the 123 | # Joyent Engineering Guide. 124 | # 125 | 126 | # 127 | # Instruct make to keep around temporary files. We have rules below that 128 | # automatically update git submodules as needed, but they employ a deps/*/.git 129 | # temporary file. Without this directive, make tries to remove these .git 130 | # directories after the build has completed. 131 | # 132 | .SECONDARY: $($(wildcard deps/*):%=%/.git) 133 | 134 | # 135 | # This rule enables other rules that use files from a git submodule to have 136 | # those files depend on deps/module/.git and have "make" automatically check 137 | # out the submodule as needed. 138 | # 139 | deps/%/.git: 140 | git submodule update --init deps/$* 141 | 142 | # 143 | # These recipes make heavy use of dynamically-created phony targets. The parent 144 | # Makefile defines a list of input files like BASH_FILES. We then say that each 145 | # of these files depends on a fake target called filename.bashchk, and then we 146 | # define a pattern rule for those targets that runs bash in check-syntax-only 147 | # mode. This mechanism has the nice properties that if you specify zero files, 148 | # the rule becomes a noop (unlike a single rule to check all bash files, which 149 | # would invoke bash with zero files), and you can check individual files from 150 | # the command line with "make filename.bashchk". 151 | # 152 | .PHONY: check-bash 153 | check-bash: $(BASH_FILES:%=%.bashchk) $(BASH_FILES:%=%.bashstyle) 154 | 155 | %.bashchk: % 156 | $(BASH) -n $^ 157 | 158 | %.bashstyle: % 159 | $(BASHSTYLE) $^ 160 | 161 | $(JSL_EXEC): 162 | make -C deps/javascriptlint install 163 | 164 | .PHONY: check-jsl check-jsl-node check-jsl-web 165 | check-jsl: check-jsl-node check-jsl-web 166 | 167 | check-jsl-node: $(JSL_FILES_NODE:%=%.jslnodechk) 168 | 169 | check-jsl-web: $(JSL_FILES_WEB:%=%.jslwebchk) 170 | 171 | %.jslnodechk: % $(JSL_EXEC) 172 | $(JSL) $(JSL_FLAGS) $(JSL_FLAGS_NODE) $< 173 | 174 | %.jslwebchk: % $(JSL_EXEC) 175 | $(JSL) $(JSL_FLAGS) $(JSL_FLAGS_WEB) $< 176 | 177 | .PHONY: check-jsstyle 178 | check-jsstyle: $(JSSTYLE_FILES:%=%.jsstylechk) 179 | 180 | %.jsstylechk: % $(JSSTYLE_EXEC) 181 | $(JSSTYLE) $(JSSTYLE_FLAGS) $< 182 | 183 | .PHONY: check 184 | check: check-jsl check-jsstyle check-bash 185 | @echo check ok 186 | 187 | .PHONY: clean 188 | clean:: 189 | -$(RMTREE) $(CLEAN_FILES) 190 | 191 | .PHONY: distclean 192 | distclean:: clean 193 | -$(RMTREE) $(DISTCLEAN_FILES) 194 | 195 | CSCOPE_FILES = cscope.in.out cscope.out cscope.po.out 196 | CLEAN_FILES += $(CSCOPE_FILES) 197 | 198 | .PHONY: xref 199 | xref: cscope.files 200 | $(CSCOPE) -bqR 201 | 202 | .PHONY: cscope.files 203 | cscope.files: 204 | find $(CSCOPE_DIRS) -name '*.c' -o -name '*.h' -o -name '*.cc' \ 205 | -o -name '*.js' -o -name '*.s' -o -name '*.cpp' > $@ 206 | 207 | # 208 | # The "docs" target is complicated because we do several things here: 209 | # 210 | # (1) Use restdown to build HTML and JSON files from each of DOC_FILES. 211 | # 212 | # (2) Copy these files into $(DOC_BUILD) (build/docs/public), which 213 | # functions as a complete copy of the documentation that could be 214 | # mirrored or served over HTTP. 215 | # 216 | # (3) Then copy any directories and media from docs/media into 217 | # $(DOC_BUILD)/media. This allows projects to include their own media, 218 | # including files that will override same-named files provided by 219 | # restdown. 220 | # 221 | # Step (3) is the surprisingly complex part: in order to do this, we need to 222 | # identify the subdirectories in docs/media, recreate them in 223 | # $(DOC_BUILD)/media, then do the same with the files. 224 | # 225 | DOC_MEDIA_DIRS := $(shell find docs/media -type d 2>/dev/null | grep -v "^docs/media$$") 226 | DOC_MEDIA_DIRS := $(DOC_MEDIA_DIRS:docs/media/%=%) 227 | DOC_MEDIA_DIRS_BUILD := $(DOC_MEDIA_DIRS:%=$(DOC_BUILD)/media/%) 228 | 229 | DOC_MEDIA_FILES := $(shell find docs/media -type f 2>/dev/null) 230 | DOC_MEDIA_FILES := $(DOC_MEDIA_FILES:docs/media/%=%) 231 | DOC_MEDIA_FILES_BUILD := $(DOC_MEDIA_FILES:%=$(DOC_BUILD)/media/%) 232 | 233 | # 234 | # Like the other targets, "docs" just depends on the final files we want to 235 | # create in $(DOC_BUILD), leveraging other targets and recipes to define how 236 | # to get there. 237 | # 238 | .PHONY: docs 239 | docs: \ 240 | $(DOC_FILES:%.restdown=$(DOC_BUILD)/%.html) \ 241 | $(DOC_FILES:%.restdown=$(DOC_BUILD)/%.json) \ 242 | $(DOC_MEDIA_FILES_BUILD) 243 | 244 | # 245 | # We keep the intermediate files so that the next build can see whether the 246 | # files in DOC_BUILD are up to date. 247 | # 248 | .PRECIOUS: \ 249 | $(DOC_FILES:%.restdown=docs/%.html) \ 250 | $(DOC_FILES:%.restdown=docs/%json) 251 | 252 | # 253 | # We do clean those intermediate files, as well as all of DOC_BUILD. 254 | # 255 | CLEAN_FILES += \ 256 | $(DOC_BUILD) \ 257 | $(DOC_FILES:%.restdown=docs/%.html) \ 258 | $(DOC_FILES:%.restdown=docs/%.json) 259 | 260 | # 261 | # Before installing the files, we must make sure the directories exist. The | 262 | # syntax tells make that the dependency need only exist, not be up to date. 263 | # Otherwise, it might try to rebuild spuriously because the directory itself 264 | # appears out of date. 265 | # 266 | $(DOC_MEDIA_FILES_BUILD): | $(DOC_MEDIA_DIRS_BUILD) 267 | 268 | $(DOC_BUILD)/%: docs/% | $(DOC_BUILD) 269 | $(CP) $< $@ 270 | 271 | docs/%.json docs/%.html: docs/%.restdown | $(DOC_BUILD) $(RESTDOWN_EXEC) 272 | $(RESTDOWN) $(RESTDOWN_FLAGS) -m $(DOC_BUILD) $< 273 | 274 | $(DOC_BUILD): 275 | $(MKDIR) $@ 276 | 277 | $(DOC_MEDIA_DIRS_BUILD): 278 | $(MKDIR) $@ 279 | 280 | # 281 | # The default "test" target does nothing. This should usually be overridden by 282 | # the parent Makefile. It's included here so we can define "prepush" without 283 | # requiring the repo to define "test". 284 | # 285 | .PHONY: test 286 | test: 287 | 288 | .PHONY: prepush 289 | prepush: check test 290 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vasync: observable asynchronous control flow 2 | 3 | This module provides several functions for asynchronous control flow. There are 4 | many modules that do this already (notably async.js). This one's claim to fame 5 | is improved debuggability. 6 | 7 | 8 | ## Observability is important 9 | 10 | Working with Node's asynchronous, callback-based model is much easier with a 11 | handful of simple control-flow abstractions, like: 12 | 13 | * waterfalls and pipelines (which invoke a list of asynchronous callbacks 14 | sequentially) 15 | * parallel pipelines (which invoke a list of asynchronous callbacks in parallel 16 | and invoke a top-level callback when the last one completes). 17 | * queues 18 | * barriers 19 | 20 | But these structures also introduce new types of programming errors: failing to 21 | invoke the callback can cause the program to hang, and inadvertently invoking it 22 | twice can cause all kinds of mayhem that's very difficult to debug. 23 | 24 | The functions in this module keep track of what's going on so that you can 25 | figure out what happened when your program goes wrong. They generally return an 26 | object describing details of the current state. If your program goes wrong, you 27 | have several ways of getting at this state: 28 | 29 | * On illumos-based systems, use MDB to [find the status object](http://dtrace.org/blogs/bmc/2012/05/05/debugging-node-js-memory-leaks/) 30 | and then [print it out](http://dtrace.org/blogs/dap/2012/01/13/playing-with-nodev8-postmortem-debugging/). 31 | * Provide an HTTP API (or AMQP, or whatever) that returns these pending status 32 | objects as JSON (see [kang](https://github.com/davepacheco/kang)). 33 | * Incorporate a REPL into your program and print out the status object. 34 | * Use the Node debugger to print out the status object. 35 | 36 | ## Functions 37 | 38 | * [parallel](#parallel-invoke-n-functions-in-parallel): invoke N functions in 39 | parallel (and merge the results) 40 | * [forEachParallel](#foreachparallel-invoke-the-same-function-on-n-inputs-in-parallel): 41 | invoke the same function on N inputs in parallel 42 | * [pipeline](#pipeline-invoke-n-functions-in-series-and-stop-on-failure): invoke 43 | N functions in series (and stop on failure) 44 | * [tryEach](#tryeach-invoke-n-functions-in-series-and-stop-on-success): invoke 45 | N functions in series (and stop on success) 46 | * [forEachPipeline](#foreachpipeline-invoke-the-same-function-on-n-inputs-in-series-and-stop-on-failure): 47 | invoke the same function on N inputs in series (and stop on failure) 48 | * [filter/filterSeries/filterLimit](#filterfilterlimitfilterseries-filter-n-inputs-serially-or-concurrently): 49 | filter N inputs serially or concurrently 50 | * [whilst](#whilst-invoke-a-function-repeatedly-until-a-stopping-condition-is-met): 51 | invoke a function repeatedly until a stopping condition is met 52 | * [waterfall](#waterfall-invoke-n-functions-in-series-stop-on-failure-and-propagate-results): 53 | like pipeline, but propagating results between stages 54 | * [barrier](#barrier-coordinate-multiple-concurrent-operations): coordinate 55 | multiple concurrent operations 56 | * [queue/queuev](#queuequeuev-fixed-size-worker-queue): fixed-size worker queue 57 | 58 | ### parallel: invoke N functions in parallel 59 | 60 | Synopsis: `parallel(args, callback)` 61 | 62 | This function takes a list of input functions (specified by the "funcs" property 63 | of "args") and runs them all. These input functions are expected to be 64 | asynchronous: they get a "callback" argument and should invoke it as 65 | `callback(err, result)`. The error and result will be saved and made available 66 | to the original caller when all of these functions complete. 67 | 68 | This function returns the same "result" object it passes to the callback, and 69 | you can use the fields in this object to debug or observe progress: 70 | 71 | * `operations`: array corresponding to the input functions, with 72 | * `func`: input function, 73 | * `status`: "pending", "ok", or "fail", 74 | * `err`: returned "err" value, if any, and 75 | * `result`: returned "result" value, if any 76 | * `successes`: "result" field for each of "operations" where 77 | "status" == "ok" (in no particular order) 78 | * `ndone`: number of input operations that have completed 79 | * `nerrors`: number of input operations that have failed 80 | 81 | This status object lets you see in a debugger exactly which functions have 82 | completed, what they returned, and which ones are outstanding. 83 | 84 | All errors are combined into a single "err" parameter to the final callback (see 85 | below). 86 | 87 | Example usage: 88 | 89 | ```js 90 | console.log(mod_vasync.parallel({ 91 | 'funcs': [ 92 | function f1 (callback) { mod_dns.resolve('joyent.com', callback); }, 93 | function f2 (callback) { mod_dns.resolve('github.com', callback); }, 94 | function f3 (callback) { mod_dns.resolve('asdfaqsdfj.com', callback); } 95 | ] 96 | }, function (err, results) { 97 | console.log('error: %s', err.message); 98 | console.log('results: %s', mod_util.inspect(results, null, 3)); 99 | })); 100 | ``` 101 | 102 | In the first tick, this outputs: 103 | 104 | ```js 105 | status: { operations: 106 | [ { func: [Function: f1], status: 'pending' }, 107 | { func: [Function: f2], status: 'pending' }, 108 | { func: [Function: f3], status: 'pending' } ], 109 | successes: [], 110 | ndone: 0, 111 | nerrors: 0 } 112 | ``` 113 | 114 | showing that there are three operations pending and none has yet been started. 115 | When the program finishes, it outputs this error: 116 | 117 | error: first of 1 error: queryA ENOTFOUND 118 | 119 | which encapsulates all of the intermediate failures. This model allows you to 120 | write the final callback like you normally would: 121 | 122 | ```js 123 | if (err) 124 | return (callback(err)); 125 | ``` 126 | 127 | and still propagate useful information to callers that don't deal with multiple 128 | errors (i.e. most callers). 129 | 130 | The example also prints out the detailed final status, including all of the 131 | errors and return values: 132 | 133 | ```js 134 | results: { operations: 135 | [ { func: [Function: f1], 136 | funcname: 'f1', 137 | status: 'ok', 138 | err: null, 139 | result: [ '165.225.132.33' ] }, 140 | { func: [Function: f2], 141 | funcname: 'f2', 142 | status: 'ok', 143 | err: null, 144 | result: [ '207.97.227.239' ] }, 145 | { func: [Function: f3], 146 | funcname: 'f3', 147 | status: 'fail', 148 | err: { [Error: queryA ENOTFOUND] code: 'ENOTFOUND', 149 | errno: 'ENOTFOUND', syscall: 'queryA' }, 150 | result: undefined } ], 151 | successes: [ [ '165.225.132.33' ], [ '207.97.227.239' ] ], 152 | ndone: 3, 153 | nerrors: 1 } 154 | ``` 155 | 156 | You can use this if you want to handle all of the errors individually or to get 157 | at all of the individual return values. 158 | 159 | Note that "successes" is provided as a convenience and the order of items in 160 | that array may not correspond to the order of the inputs. To consume output in 161 | an ordered manner, you should iterate over "operations" and pick out the result 162 | from each item. 163 | 164 | 165 | ### forEachParallel: invoke the same function on N inputs in parallel 166 | 167 | Synopsis: `forEachParallel(args, callback)` 168 | 169 | This function is exactly like `parallel`, except that the input is specified as 170 | a *single* function ("func") and a list of inputs ("inputs"). The function is 171 | invoked on each input in parallel. 172 | 173 | This example is exactly equivalent to the one above: 174 | 175 | ```js 176 | console.log(mod_vasync.forEachParallel({ 177 | 'func': mod_dns.resolve, 178 | 'inputs': [ 'joyent.com', 'github.com', 'asdfaqsdfj.com' ] 179 | }, function (err, results) { 180 | console.log('error: %s', err.message); 181 | console.log('results: %s', mod_util.inspect(results, null, 3)); 182 | })); 183 | ``` 184 | 185 | ### pipeline: invoke N functions in series (and stop on failure) 186 | 187 | Synopsis: `pipeline(args, callback)` 188 | 189 | The named arguments (that go inside `args`) are: 190 | 191 | * `funcs`: input functions, to be invoked in series 192 | * `arg`: arbitrary argument that will be passed to each function 193 | 194 | The functions are invoked in order as `func(arg, callback)`, where "arg" is the 195 | user-supplied argument from "args" and "callback" should be invoked in the usual 196 | way. If any function emits an error, the whole pipeline stops. 197 | 198 | The return value and the arguments to the final callback are exactly the same as 199 | for `parallel`. The error object for the final callback is just the error 200 | returned by whatever pipeline function failed (if any). 201 | 202 | This example is similar to the one above, except that it runs the steps in 203 | sequence and stops early because `pipeline` stops on the first error: 204 | 205 | ```js 206 | console.log(mod_vasync.pipeline({ 207 | 'funcs': [ 208 | function f1 (_, callback) { mod_fs.stat('/tmp', callback); }, 209 | function f2 (_, callback) { mod_fs.stat('/noexist', callback); }, 210 | function f3 (_, callback) { mod_fs.stat('/var', callback); } 211 | ] 212 | }, function (err, results) { 213 | console.log('error: %s', err.message); 214 | console.log('results: %s', mod_util.inspect(results, null, 3)); 215 | })); 216 | ``` 217 | 218 | As a result, the status after the first tick looks like this: 219 | 220 | ```js 221 | { operations: 222 | [ { func: [Function: f1], status: 'pending' }, 223 | { func: [Function: f2], status: 'waiting' }, 224 | { func: [Function: f3], status: 'waiting' } ], 225 | successes: [], 226 | ndone: 0, 227 | nerrors: 0 } 228 | ``` 229 | 230 | Note that the second and third stages are now "waiting", rather than "pending" 231 | in the `parallel` case. The error and complete result look just like the 232 | parallel case. 233 | 234 | ### tryEach: invoke N functions in series (and stop on success) 235 | 236 | Synopsis: `tryEach(funcs, callback)` 237 | 238 | The `tryEach` function invokes each of the asynchronous functions in `funcs` 239 | serially. Each function takes a single argument: an interstitial-callback. 240 | `tryEach` will keep calling the functions until one of them succeeds (or they 241 | all fail). At the end, the terminating-callback is invoked with the error 242 | and/or results provided by the last function that was called (either the last 243 | one that failed or the first one that succeeded). 244 | 245 | This example is similar to the one above, except that it runs the steps in 246 | sequence and stops early because `tryEach` stops on the first success: 247 | 248 | ```js 249 | console.log(mod_vasync.tryEach([ 250 | function f1 (callback) { mod_fs.stat('/notreal', callback); }, 251 | function f2 (callback) { mod_fs.stat('/noexist', callback); }, 252 | function f3 (callback) { mod_fs.stat('/var', callback); }, 253 | function f4 (callback) { mod_fs.stat('/noexist', callback); } 254 | ], 255 | function (err, results) { 256 | console.log('error: %s', err); 257 | console.log('results: %s', mod_util.inspect(results)); 258 | })); 259 | 260 | ``` 261 | 262 | The above code will stop when it finishes f3, and we will only print a single 263 | result and no errors: 264 | 265 | ```js 266 | error: null 267 | results: { dev: 65760, 268 | mode: 16877, 269 | nlink: 41, 270 | uid: 0, 271 | gid: 3, 272 | rdev: -1, 273 | blksize: 2560, 274 | ino: 11, 275 | size: 41, 276 | blocks: 7, 277 | atime: Thu May 28 2015 16:21:25 GMT+0000 (UTC), 278 | mtime: Thu Jan 21 2016 22:08:50 GMT+0000 (UTC), 279 | ctime: Thu Jan 21 2016 22:08:50 GMT+0000 (UTC) } 280 | ``` 281 | 282 | If we comment out `f3`, we get the following output: 283 | 284 | ```js 285 | error: Error: ENOENT, stat '/noexist' 286 | results: undefined 287 | ``` 288 | 289 | Note that: there is a mismatch (inherited from `async`) between the semantics 290 | of the interstitial callback and the sematics of the terminating callback. See 291 | the following example: 292 | 293 | ```js 294 | console.log(mod_vasync.tryEach([ 295 | function f1 (callback) { callback(new Error()); }, 296 | function f2 (callback) { callback(new Error()); }, 297 | function f3 (callback) { callback(null, 1, 2, 3); }, 298 | function f4 (callback) { callback(null, 1); } 299 | ], 300 | function (err, results) { 301 | console.log('error: %s', err); 302 | console.log('results: %s', mod_util.inspect(results)); 303 | })); 304 | 305 | ``` 306 | 307 | We pass one or more results to the terminating-callback via the 308 | interstitial-callback's arglist -- `(err, res1, res2, ...)`. From the 309 | callback-implementor's perspective, the results get wrapped up in an array 310 | `(err, [res1, res2, ...])` -- unless there is only one result, which simply 311 | gets passed through as the terminating callback's second argument. This means 312 | that when we call the callback in `f3` above, the terminating callback receives 313 | the list `[1, 2, 3]` as its second argument. If, we comment out `f3`, we will 314 | end up calling the callback in `f4` which will end up invoking the terminating 315 | callback with a single result: `1`, instead of `[1]`. 316 | 317 | 318 | In short, be mindful that there is not always a 1:1 correspondence between the 319 | terminating callback that you define, and the interstitial callback that gets 320 | called from the function. 321 | 322 | 323 | 324 | ### forEachPipeline: invoke the same function on N inputs in series (and stop on failure) 325 | 326 | Synopsis: `forEachPipeline(args, callback)` 327 | 328 | This function is exactly like `pipeline`, except that the input is specified as 329 | a *single* function ("func") and a list of inputs ("inputs"). The function is 330 | invoked on each input in series. 331 | 332 | This example is exactly equivalent to the one above: 333 | 334 | ```js 335 | console.log(mod_vasync.forEachPipeline({ 336 | 'func': mod_dns.resolve, 337 | 'inputs': [ 'joyent.com', 'github.com', 'asdfaqsdfj.com' ] 338 | }, function (err, results) { 339 | console.log('error: %s', err.message); 340 | console.log('results: %s', mod_util.inspect(results, null, 3)); 341 | })); 342 | ``` 343 | 344 | ### waterfall: invoke N functions in series, stop on failure, and propagate results 345 | 346 | Synopsis: `waterfall(funcs, callback)` 347 | 348 | This function works like `pipeline` except for argument passing. 349 | 350 | Each function is passed any values emitted by the previous function (none for 351 | the first function), followed by the callback to invoke upon completion. This 352 | callback must be invoked exactly once, regardless of success or failure. As 353 | conventional in Node, the first argument to the callback indicates an error (if 354 | non-null). Subsequent arguments are passed to the next function in the "funcs" 355 | chain. 356 | 357 | If any function fails (i.e., calls its callback with an Error), then the 358 | remaining functions are not invoked and "callback" is invoked with the error. 359 | 360 | The only difference between waterfall() and pipeline() are the arguments passed 361 | to each function in the chain. pipeline() always passes the same argument 362 | followed by the callback, while waterfall() passes whatever values were emitted 363 | by the previous function followed by the callback. 364 | 365 | Here's an example: 366 | 367 | ```js 368 | mod_vasync.waterfall([ 369 | function func1(callback) { 370 | setImmediate(function () { 371 | callback(null, 37); 372 | }); 373 | }, 374 | function func2(extra, callback) { 375 | console.log('func2 got "%s" from func1', extra); 376 | callback(); 377 | } 378 | ], function () { 379 | console.log('done'); 380 | }); 381 | ``` 382 | 383 | This prints: 384 | 385 | ``` 386 | func2 got "37" from func1 387 | better stop early 388 | ``` 389 | 390 | ### filter/filterLimit/filterSeries: filter N inputs serially or concurrently 391 | 392 | Synopsis: `filter(inputs, filterFunc, callback)` 393 | 394 | Synopsis: `filterSeries(inputs, filterFunc, callback)` 395 | 396 | Synopsis: `filterLimit(inputs, limit, filterFunc, callback)` 397 | 398 | These functions take an array (of anything) and a function to call on each 399 | element of the array. The function must callback with a true or false value as 400 | the second argument or an error object as the first argument. False values 401 | will result in the element being filtered out of the results array. An error 402 | object passed as the first argument will cause the filter function to stop 403 | processing new elements and callback to the caller with the error immediately. 404 | Original input array order is maintained. 405 | 406 | `filter` and `filterSeries` are analogous to calling `filterLimit` with 407 | a limit of `Infinity` and `1` respectively. 408 | 409 | 410 | ```js 411 | var inputs = [ 412 | 'joyent.com', 413 | 'github.com', 414 | 'asdfaqsdfj.com' 415 | ]; 416 | function filterFunc(input, cb) { 417 | mod_dns.resolve(input, function (err, results) { 418 | if (err) { 419 | cb(null, false); 420 | } else { 421 | cb(null, true); 422 | } 423 | } 424 | } 425 | mod_vasync.filter(inputs, filterFunc, function (err, results) { 426 | // err => undefined 427 | // results => ['joyent.com', 'github.com'] 428 | }); 429 | ``` 430 | 431 | ### whilst: invoke a function repeatedly until a stopping condition is met 432 | 433 | Synopsis: `whilst(testFunc, iterateFunc, callback)` 434 | 435 | Repeatedly invoke `iterateFunc` while `testFunc` returns a true value. 436 | `iterateFunc` is an asychronous function that must call its callback (the first 437 | and only argument given to it) when it is finished with an optional error 438 | object as the first argument, and any other arbitrary arguments. If an error 439 | object is given as the first argument, `whilst` will finish and call `callback` 440 | with the error object. `testFunc` is a synchronous function that must return 441 | a value - if the value resolves to true `whilst` will invoke `iterateFunc`, if 442 | it resolves to false `whilst` will finish and invoke `callback` with the last 443 | set of arguments `iterateFunc` called back with. 444 | 445 | `whilst` also returns an object suitable for introspecting the current state of 446 | the specific `whilst` invocation which contains the following properties: 447 | 448 | * `finished`: boolean if this invocation has finished or is in progress 449 | * `iterations`: number of iterations performed (calls to `iterateFunc`) 450 | 451 | Compatible with `async.whilst` 452 | 453 | ```js 454 | var n = 0; 455 | 456 | var w = mod_vasync.whilst( 457 | function testFunc() { 458 | return (n < 5); 459 | }, 460 | function iterateFunc(cb) { 461 | n++; 462 | cb(null, {n: n}); 463 | }, 464 | function whilstDone(err, arg) { 465 | // err => undefined 466 | // arg => {n: 5} 467 | // w => {finished: true, iterations: 5} 468 | } 469 | ); 470 | 471 | // w => {finished: false, iterations: 0} 472 | ``` 473 | 474 | ### barrier: coordinate multiple concurrent operations 475 | 476 | Synopsis: `barrier([args])` 477 | 478 | Returns a new barrier object. Like `parallel`, barriers are useful for 479 | coordinating several concurrent operations, but instead of specifying a list of 480 | functions to invoke, you just say how many (and optionally which ones) are 481 | outstanding, and this object emits `'drain'` when they've all completed. This 482 | is syntactically lighter-weight, and more flexible. 483 | 484 | * Methods: 485 | 486 | * start(name): Indicates that the named operation began. The name must not 487 | match an operation which is already ongoing. 488 | * done(name): Indicates that the named operation ended. 489 | 490 | 491 | * Read-only public properties (for debugging): 492 | 493 | * pending: Set of pending operations. Keys are names passed to "start", and 494 | values are timestamps when the operation began. 495 | * recent: Array of recent completed operations. Each element is an object 496 | with a "name", "start", and "done" field. By default, 10 operations are 497 | remembered. 498 | 499 | 500 | * Options: 501 | 502 | * nrecent: number of recent operations to remember (for debugging) 503 | 504 | Example: printing sizes of files in a directory 505 | 506 | ```js 507 | var mod_fs = require('fs'); 508 | var mod_path = require('path'); 509 | var mod_vasync = require('../lib/vasync'); 510 | 511 | var barrier = mod_vasync.barrier(); 512 | 513 | barrier.on('drain', function () { 514 | console.log('all files checked'); 515 | }); 516 | 517 | barrier.start('readdir'); 518 | 519 | mod_fs.readdir(__dirname, function (err, files) { 520 | barrier.done('readdir'); 521 | 522 | if (err) 523 | throw (err); 524 | 525 | files.forEach(function (file) { 526 | barrier.start('stat ' + file); 527 | 528 | var path = mod_path.join(__dirname, file); 529 | 530 | mod_fs.stat(path, function (err2, stat) { 531 | barrier.done('stat ' + file); 532 | console.log('%s: %d bytes', file, stat['size']); 533 | }); 534 | }); 535 | }); 536 | ``` 537 | 538 | This emits: 539 | 540 | barrier-readdir.js: 602 bytes 541 | foreach-parallel.js: 358 bytes 542 | barrier-basic.js: 552 bytes 543 | nofail.js: 384 bytes 544 | pipeline.js: 490 bytes 545 | parallel.js: 481 bytes 546 | queue-serializer.js: 441 bytes 547 | queue-stat.js: 529 bytes 548 | all files checked 549 | 550 | 551 | ### queue/queuev: fixed-size worker queue 552 | 553 | Synopsis: `queue(worker, concurrency)` 554 | 555 | Synopsis: `queuev(args)` 556 | 557 | This function returns an object that allows up to a fixed number of tasks to be 558 | dispatched at any given time. The interface is compatible with that provided 559 | by the "async" Node library, except that the returned object's fields represent 560 | a public interface you can use to introspect what's going on. 561 | 562 | * Arguments 563 | 564 | * worker: a function invoked as `worker(task, callback)`, where `task` is a 565 | task dispatched to this queue and `callback` should be invoked when the 566 | task completes. 567 | * concurrency: a positive integer indicating the maximum number of tasks 568 | that may be dispatched at any time. With concurrency = 1, the queue 569 | serializes all operations. 570 | 571 | 572 | * Methods 573 | 574 | * push(task, [callback]): add a task (or array of tasks) to the queue, with 575 | an optional callback to be invoked when each task completes. If a list of 576 | tasks are added, the callback is invoked for each one. 577 | * length(): for compatibility with node-async. 578 | * close(): signal that no more tasks will be enqueued. Further attempts to 579 | enqueue tasks to this queue will throw. Once all pending and queued 580 | tasks are completed the object will emit the "end" event. The "end" 581 | event is the last event the queue will emit, and it will be emitted even 582 | if no tasks were ever enqueued. 583 | * kill(): clear enqueued tasks and implicitly close the queue. Several 584 | caveats apply when kill() is called: 585 | * The completion callback will _not_ be called for items purged from 586 | the queue. 587 | * The drain handler is cleared (for node-async compatibility) 588 | * Subsequent calls to kill() or close() are no-ops. 589 | * As with close(), it is not legal to call push() after kill(). 590 | 591 | 592 | * Read-only public properties (for debugging): 593 | 594 | * concurrency: for compatibility with node-async 595 | * worker: worker function, as passed into "queue"/"queuev" 596 | * worker\_name: worker function's "name" field 597 | * npending: the number of tasks currently being processed 598 | * pending: an object (*not* an array) describing the tasks currently being 599 | processed 600 | * queued: array of tasks currently queued for processing 601 | * closed: true when close() has been called on the queue 602 | * ended: true when all tasks have completed processing, and no more 603 | processing will occur 604 | * killed: true when kill() has been called on the queue 605 | 606 | 607 | * Hooks (for compatibility with node-async): 608 | 609 | * saturated 610 | * empty 611 | * drain 612 | 613 | * Events 614 | 615 | * 'end': see close() 616 | 617 | If the tasks are themselves simple objects, then the entire queue may be 618 | serialized (as via JSON.stringify) for debugging and monitoring tools. Using 619 | the above fields, you can see what this queue is doing (worker\_name), which 620 | tasks are queued, which tasks are being processed, and so on. 621 | 622 | ### Example 1: Stat several files 623 | 624 | Here's an example demonstrating the queue: 625 | 626 | ```js 627 | var mod_fs = require('fs'); 628 | var mod_vasync = require('../lib/vasync'); 629 | 630 | var queue; 631 | 632 | function doneOne() 633 | { 634 | console.log('task completed; queue state:\n%s\n', 635 | JSON.stringify(queue, null, 4)); 636 | } 637 | 638 | queue = mod_vasync.queue(mod_fs.stat, 2); 639 | 640 | console.log('initial queue state:\n%s\n', JSON.stringify(queue, null, 4)); 641 | 642 | queue.push('/tmp/file1', doneOne); 643 | queue.push('/tmp/file2', doneOne); 644 | queue.push('/tmp/file3', doneOne); 645 | queue.push('/tmp/file4', doneOne); 646 | 647 | console.log('all tasks dispatched:\n%s\n', JSON.stringify(queue, null, 4)); 648 | ``` 649 | 650 | The initial queue state looks like this: 651 | 652 | ```js 653 | initial queue state: 654 | { 655 | "nextid": 0, 656 | "worker_name": "anon", 657 | "npending": 0, 658 | "pending": {}, 659 | "queued": [], 660 | "concurrency": 2 661 | } 662 | ``` 663 | After four tasks have been pushed, we see that two of them have been dispatched 664 | and the remaining two are queued up: 665 | 666 | ```js 667 | all tasks pushed: 668 | { 669 | "nextid": 4, 670 | "worker_name": "anon", 671 | "npending": 2, 672 | "pending": { 673 | "1": { 674 | "id": 1, 675 | "task": "/tmp/file1" 676 | }, 677 | "2": { 678 | "id": 2, 679 | "task": "/tmp/file2" 680 | } 681 | }, 682 | "queued": [ 683 | { 684 | "id": 3, 685 | "task": "/tmp/file3" 686 | }, 687 | { 688 | "id": 4, 689 | "task": "/tmp/file4" 690 | } 691 | ], 692 | "concurrency": 2 693 | } 694 | ``` 695 | 696 | As they complete, we see tasks moving from "queued" to "pending", and completed 697 | tasks disappear: 698 | 699 | ```js 700 | task completed; queue state: 701 | { 702 | "nextid": 4, 703 | "worker_name": "anon", 704 | "npending": 1, 705 | "pending": { 706 | "3": { 707 | "id": 3, 708 | "task": "/tmp/file3" 709 | } 710 | }, 711 | "queued": [ 712 | { 713 | "id": 4, 714 | "task": "/tmp/file4" 715 | } 716 | ], 717 | "concurrency": 2 718 | } 719 | ``` 720 | 721 | When all tasks have completed, the queue state looks like it started: 722 | 723 | ```js 724 | task completed; queue state: 725 | { 726 | "nextid": 4, 727 | "worker_name": "anon", 728 | "npending": 0, 729 | "pending": {}, 730 | "queued": [], 731 | "concurrency": 2 732 | } 733 | ``` 734 | 735 | 736 | ### Example 2: A simple serializer 737 | 738 | You can use a queue with concurrency 1 and where the tasks are themselves 739 | functions to ensure that an arbitrary asynchronous function never runs 740 | concurrently with another one, no matter what each one does. Since the tasks 741 | are the actual functions to be invoked, the worker function just invokes each 742 | one: 743 | 744 | ```js 745 | var mod_vasync = require('../lib/vasync'); 746 | 747 | var queue = mod_vasync.queue( 748 | function (task, callback) { task(callback); }, 1); 749 | 750 | queue.push(function (callback) { 751 | console.log('first task begins'); 752 | setTimeout(function () { 753 | console.log('first task ends'); 754 | callback(); 755 | }, 500); 756 | }); 757 | 758 | queue.push(function (callback) { 759 | console.log('second task begins'); 760 | process.nextTick(function () { 761 | console.log('second task ends'); 762 | callback(); 763 | }); 764 | }); 765 | ``` 766 | 767 | This example outputs: 768 | 769 | $ node examples/queue-serializer.js 770 | first task begins 771 | first task ends 772 | second task begins 773 | second task ends 774 | -------------------------------------------------------------------------------- /examples/barrier-basic.js: -------------------------------------------------------------------------------- 1 | var mod_vasync = require('../lib/vasync'); 2 | 3 | var barrier = mod_vasync.barrier(); 4 | 5 | barrier.on('drain', function () { 6 | console.log('barrier drained!'); 7 | }); 8 | 9 | console.log('barrier', barrier); 10 | 11 | barrier.start('op1'); 12 | console.log('op1 started', barrier); 13 | 14 | barrier.start('op2'); 15 | console.log('op2 started', barrier); 16 | 17 | barrier.done('op2'); 18 | console.log('op2 done', barrier); 19 | 20 | barrier.done('op1'); 21 | console.log('op1 done', barrier); 22 | 23 | barrier.start('op3'); 24 | console.log('op3 started'); 25 | 26 | setTimeout(function () { 27 | barrier.done('op3'); 28 | console.log('op3 done'); 29 | }, 10); 30 | -------------------------------------------------------------------------------- /examples/barrier-readdir.js: -------------------------------------------------------------------------------- 1 | var mod_fs = require('fs'); 2 | var mod_path = require('path'); 3 | var mod_vasync = require('../lib/vasync'); 4 | 5 | var barrier = mod_vasync.barrier(); 6 | 7 | barrier.on('drain', function () { 8 | console.log('all files checked'); 9 | }); 10 | 11 | barrier.start('readdir'); 12 | 13 | mod_fs.readdir(__dirname, function (err, files) { 14 | barrier.done('readdir'); 15 | 16 | if (err) 17 | throw (err); 18 | 19 | files.forEach(function (file) { 20 | barrier.start('stat ' + file); 21 | 22 | var path = mod_path.join(__dirname, file); 23 | 24 | mod_fs.stat(path, function (err2, stat) { 25 | barrier.done('stat ' + file); 26 | console.log('%s: %d bytes', file, stat['size']); 27 | }); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /examples/foreach-parallel.js: -------------------------------------------------------------------------------- 1 | var mod_dns = require('dns'); 2 | var mod_util = require('util'); 3 | var mod_vasync = require('../lib/vasync'); 4 | 5 | console.log(mod_vasync.forEachParallel({ 6 | 'func': mod_dns.resolve, 7 | 'inputs': [ 'joyent.com', 'github.com', 'asdfaqsdfj.com' ] 8 | }, function (err, results) { 9 | console.log('error: %s', err.message); 10 | console.log('results: %s', mod_util.inspect(results, null, 3)); 11 | })); 12 | -------------------------------------------------------------------------------- /examples/foreach-pipeline.js: -------------------------------------------------------------------------------- 1 | var mod_dns = require('dns'); 2 | var mod_util = require('util'); 3 | var mod_vasync = require('../lib/vasync'); 4 | 5 | console.log(mod_vasync.forEachPipeline({ 6 | 'func': mod_dns.resolve, 7 | 'inputs': [ 'joyent.com', 'github.com', 'asdfaqsdfj.com' ] 8 | }, function (err, results) { 9 | console.log('error: %s', err.message); 10 | console.log('results: %s', mod_util.inspect(results, null, 3)); 11 | })); 12 | -------------------------------------------------------------------------------- /examples/nofail.js: -------------------------------------------------------------------------------- 1 | var mod_vasync = require('../lib/vasync'); 2 | var mod_util = require('util'); 3 | var mod_fs = require('fs'); 4 | 5 | var status = mod_vasync.parallel({ 6 | funcs: [ 7 | function f1 (callback) { mod_fs.stat('/tmp', callback); }, 8 | function f2 (callback) { mod_fs.stat('/var', callback); } 9 | ] 10 | }, function (err, results) { 11 | console.log(err); 12 | console.log(mod_util.inspect(results, false, 8)); 13 | }); 14 | -------------------------------------------------------------------------------- /examples/parallel.js: -------------------------------------------------------------------------------- 1 | var mod_dns = require('dns'); 2 | var mod_util = require('util'); 3 | var mod_vasync = require('../lib/vasync'); 4 | 5 | console.log(mod_vasync.parallel({ 6 | 'funcs': [ 7 | function f1 (callback) { mod_dns.resolve('joyent.com', callback); }, 8 | function f2 (callback) { mod_dns.resolve('github.com', callback); }, 9 | function f3 (callback) { mod_dns.resolve('asdfaqsdfj.com', callback); } 10 | ] 11 | }, function (err, results) { 12 | console.log('error: %s', err.message); 13 | console.log('results: %s', mod_util.inspect(results, null, 3)); 14 | })); 15 | -------------------------------------------------------------------------------- /examples/pipeline.js: -------------------------------------------------------------------------------- 1 | var mod_dns = require('dns'); 2 | var mod_util = require('util'); 3 | var mod_vasync = require('../lib/vasync'); 4 | 5 | console.log(mod_vasync.pipeline({ 6 | 'funcs': [ 7 | function f1 (_, callback) { mod_dns.resolve('joyent.com', callback); }, 8 | function f2 (_, callback) { mod_dns.resolve('github.com', callback); }, 9 | function f3 (_, callback) { mod_dns.resolve('asdfaqsdfj.com', callback); } 10 | ] 11 | }, function (err, results) { 12 | console.log('error: %s', err.message); 13 | console.log('results: %s', mod_util.inspect(results, null, 3)); 14 | })); 15 | -------------------------------------------------------------------------------- /examples/queue-serializer.js: -------------------------------------------------------------------------------- 1 | var mod_vasync = require('../lib/vasync'); 2 | 3 | var queue = mod_vasync.queue(function (task, callback) { task(callback); }, 1); 4 | 5 | queue.push(function (callback) { 6 | console.log('first task begins'); 7 | setTimeout(function () { 8 | console.log('first task ends'); 9 | callback(); 10 | }, 500); 11 | }); 12 | 13 | queue.push(function (callback) { 14 | console.log('second task begins'); 15 | process.nextTick(function () { 16 | console.log('second task ends'); 17 | callback(); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /examples/queue-stat.js: -------------------------------------------------------------------------------- 1 | var mod_fs = require('fs'); 2 | var mod_vasync = require('../lib/vasync'); 3 | 4 | var queue; 5 | 6 | function doneOne() 7 | { 8 | console.log('task completed; queue state:\n%s\n', 9 | JSON.stringify(queue, null, 4)); 10 | } 11 | 12 | queue = mod_vasync.queue(mod_fs.stat, 2); 13 | 14 | console.log('initial queue state:\n%s\n', JSON.stringify(queue, null, 4)); 15 | 16 | queue.push('/tmp/file1', doneOne); 17 | queue.push('/tmp/file2', doneOne); 18 | queue.push('/tmp/file3', doneOne); 19 | queue.push('/tmp/file4', doneOne); 20 | 21 | console.log('all tasks pushed:\n%s\n', JSON.stringify(queue, null, 4)); 22 | -------------------------------------------------------------------------------- /examples/waterfall.js: -------------------------------------------------------------------------------- 1 | /* 2 | * examples/waterfall.js: simple waterfall example 3 | */ 4 | var mod_vasync = require('..'); 5 | mod_vasync.waterfall([ 6 | function func1(callback) { 7 | setImmediate(function () { 8 | callback(null, 37); 9 | }); 10 | }, 11 | function func2(extra, callback) { 12 | console.log('func2 got "%s" from func1', extra); 13 | callback(); 14 | } 15 | ], function () { 16 | console.log('done'); 17 | }); 18 | -------------------------------------------------------------------------------- /examples/whilst.js: -------------------------------------------------------------------------------- 1 | var mod_vasync = require('../lib/vasync'); 2 | 3 | var n = 0; 4 | 5 | var w = mod_vasync.whilst( 6 | function testFunc() { 7 | return (n < 5); 8 | }, 9 | function iterateFunc(cb) { 10 | n++; 11 | cb(null, {n: n}); 12 | }, 13 | function whilstDone(err, arg) { 14 | console.log('err: %j', err); 15 | console.log('arg: %j', arg); 16 | console.log('w (end): %j', w); 17 | } 18 | ); 19 | 20 | console.log('w (start): %j', w); 21 | -------------------------------------------------------------------------------- /jsl.node.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration File for JavaScript Lint 3 | # 4 | # This configuration file can be used to lint a collection of scripts, or to enable 5 | # or disable warnings for scripts that are linted via the command line. 6 | # 7 | 8 | ### Warnings 9 | # Enable or disable warnings based on requirements. 10 | # Use "+WarningName" to display or "-WarningName" to suppress. 11 | # 12 | +ambiguous_else_stmt # the else statement could be matched with one of multiple if statements (use curly braces to indicate intent 13 | +ambiguous_nested_stmt # block statements containing block statements should use curly braces to resolve ambiguity 14 | +ambiguous_newline # unexpected end of line; it is ambiguous whether these lines are part of the same statement 15 | +anon_no_return_value # anonymous function does not always return value 16 | +assign_to_function_call # assignment to a function call 17 | -block_without_braces # block statement without curly braces 18 | +comma_separated_stmts # multiple statements separated by commas (use semicolons?) 19 | +comparison_type_conv # comparisons against null, 0, true, false, or an empty string allowing implicit type conversion (use === or !==) 20 | +default_not_at_end # the default case is not at the end of the switch statement 21 | +dup_option_explicit # duplicate "option explicit" control comment 22 | +duplicate_case_in_switch # duplicate case in switch statement 23 | +duplicate_formal # duplicate formal argument {name} 24 | +empty_statement # empty statement or extra semicolon 25 | +identifier_hides_another # identifer {name} hides an identifier in a parent scope 26 | -inc_dec_within_stmt # increment (++) and decrement (--) operators used as part of greater statement 27 | +incorrect_version # Expected /*jsl:content-type*/ control comment. The script was parsed with the wrong version. 28 | +invalid_fallthru # unexpected "fallthru" control comment 29 | +invalid_pass # unexpected "pass" control comment 30 | +jsl_cc_not_understood # couldn't understand control comment using /*jsl:keyword*/ syntax 31 | +leading_decimal_point # leading decimal point may indicate a number or an object member 32 | +legacy_cc_not_understood # couldn't understand control comment using /*@keyword@*/ syntax 33 | +meaningless_block # meaningless block; curly braces have no impact 34 | +mismatch_ctrl_comments # mismatched control comment; "ignore" and "end" control comments must have a one-to-one correspondence 35 | +misplaced_regex # regular expressions should be preceded by a left parenthesis, assignment, colon, or comma 36 | +missing_break # missing break statement 37 | +missing_break_for_last_case # missing break statement for last case in switch 38 | +missing_default_case # missing default case in switch statement 39 | +missing_option_explicit # the "option explicit" control comment is missing 40 | +missing_semicolon # missing semicolon 41 | +missing_semicolon_for_lambda # missing semicolon for lambda assignment 42 | +multiple_plus_minus # unknown order of operations for successive plus (e.g. x+++y) or minus (e.g. x---y) signs 43 | +nested_comment # nested comment 44 | +no_return_value # function {name} does not always return a value 45 | +octal_number # leading zeros make an octal number 46 | +parseint_missing_radix # parseInt missing radix parameter 47 | +partial_option_explicit # the "option explicit" control comment, if used, must be in the first script tag 48 | +redeclared_var # redeclaration of {name} 49 | +trailing_comma_in_array # extra comma is not recommended in array initializers 50 | +trailing_decimal_point # trailing decimal point may indicate a number or an object member 51 | +undeclared_identifier # undeclared identifier: {name} 52 | +unreachable_code # unreachable code 53 | -unreferenced_argument # argument declared but never referenced: {name} 54 | -unreferenced_function # function is declared but never referenced: {name} 55 | +unreferenced_variable # variable is declared but never referenced: {name} 56 | +unsupported_version # JavaScript {version} is not supported 57 | +use_of_label # use of label 58 | +useless_assign # useless assignment 59 | +useless_comparison # useless comparison; comparing identical expressions 60 | -useless_quotes # the quotation marks are unnecessary 61 | +useless_void # use of the void type may be unnecessary (void is always undefined) 62 | +var_hides_arg # variable {name} hides argument 63 | +want_assign_or_call # expected an assignment or function call 64 | +with_statement # with statement hides undeclared variables; use temporary variable instead 65 | 66 | 67 | ### Output format 68 | # Customize the format of the error message. 69 | # __FILE__ indicates current file path 70 | # __FILENAME__ indicates current file name 71 | # __LINE__ indicates current line 72 | # __COL__ indicates current column 73 | # __ERROR__ indicates error message (__ERROR_PREFIX__: __ERROR_MSG__) 74 | # __ERROR_NAME__ indicates error name (used in configuration file) 75 | # __ERROR_PREFIX__ indicates error prefix 76 | # __ERROR_MSG__ indicates error message 77 | # 78 | # For machine-friendly output, the output format can be prefixed with 79 | # "encode:". If specified, all items will be encoded with C-slashes. 80 | # 81 | # Visual Studio syntax (default): 82 | +output-format __FILE__(__LINE__): __ERROR__ 83 | # Alternative syntax: 84 | #+output-format __FILE__:__LINE__: __ERROR__ 85 | 86 | 87 | ### Context 88 | # Show the in-line position of the error. 89 | # Use "+context" to display or "-context" to suppress. 90 | # 91 | +context 92 | 93 | 94 | ### Control Comments 95 | # Both JavaScript Lint and the JScript interpreter confuse each other with the syntax for 96 | # the /*@keyword@*/ control comments and JScript conditional comments. (The latter is 97 | # enabled in JScript with @cc_on@). The /*jsl:keyword*/ syntax is preferred for this reason, 98 | # although legacy control comments are enabled by default for backward compatibility. 99 | # 100 | -legacy_control_comments 101 | 102 | 103 | ### Defining identifiers 104 | # By default, "option explicit" is enabled on a per-file basis. 105 | # To enable this for all files, use "+always_use_option_explicit" 106 | -always_use_option_explicit 107 | 108 | # Define certain identifiers of which the lint is not aware. 109 | # (Use this in conjunction with the "undeclared identifier" warning.) 110 | # 111 | # Common uses for webpages might be: 112 | +define __dirname 113 | +define clearInterval 114 | +define clearTimeout 115 | +define console 116 | +define exports 117 | +define global 118 | +define process 119 | +define require 120 | +define setImmediate 121 | +define setInterval 122 | +define setTimeout 123 | +define Buffer 124 | +define JSON 125 | +define Math 126 | 127 | ### JavaScript Version 128 | # To change the default JavaScript version: 129 | #+default-type text/javascript;version=1.5 130 | #+default-type text/javascript;e4x=1 131 | 132 | ### Files 133 | # Specify which files to lint 134 | # Use "+recurse" to enable recursion (disabled by default). 135 | # To add a set of files, use "+process FileName", "+process Folder\Path\*.js", 136 | # or "+process Folder\Path\*.htm". 137 | # 138 | 139 | -------------------------------------------------------------------------------- /lib/vasync.js: -------------------------------------------------------------------------------- 1 | /* 2 | * vasync.js: utilities for observable asynchronous control flow 3 | */ 4 | 5 | var mod_assert = require('assert'); 6 | var mod_events = require('events'); 7 | var mod_util = require('util'); 8 | var mod_verror = require('verror'); 9 | 10 | /* 11 | * Public interface 12 | */ 13 | exports.parallel = parallel; 14 | exports.forEachParallel = forEachParallel; 15 | exports.pipeline = pipeline; 16 | exports.tryEach = tryEach; 17 | exports.forEachPipeline = forEachPipeline; 18 | exports.filter = filter; 19 | exports.filterLimit = filterLimit; 20 | exports.filterSeries = filterSeries; 21 | exports.whilst = whilst; 22 | exports.queue = queue; 23 | exports.queuev = queuev; 24 | exports.barrier = barrier; 25 | exports.waterfall = waterfall; 26 | 27 | if (!global.setImmediate) { 28 | global.setImmediate = function (func) { 29 | var args = Array.prototype.slice.call(arguments, 1); 30 | args.unshift(0); 31 | args.unshift(func); 32 | setTimeout.apply(this, args); 33 | }; 34 | } 35 | 36 | /* 37 | * This is incorporated here from jsprim because jsprim ends up pulling in a lot 38 | * of dependencies. If we end up needing more from jsprim, though, we should 39 | * add it back and rip out this function. 40 | */ 41 | function isEmpty(obj) 42 | { 43 | var key; 44 | for (key in obj) 45 | return (false); 46 | return (true); 47 | } 48 | 49 | /* 50 | * Given a set of functions that complete asynchronously using the standard 51 | * callback(err, result) pattern, invoke them all and merge the results. See 52 | * README.md for details. 53 | */ 54 | function parallel(args, callback) 55 | { 56 | var funcs, rv, doneOne, i; 57 | 58 | mod_assert.equal(typeof (args), 'object', '"args" must be an object'); 59 | mod_assert.ok(Array.isArray(args['funcs']), 60 | '"args.funcs" must be specified and must be an array'); 61 | mod_assert.equal(typeof (callback), 'function', 62 | 'callback argument must be specified and must be a function'); 63 | 64 | funcs = args['funcs'].slice(0); 65 | 66 | rv = { 67 | 'operations': new Array(funcs.length), 68 | 'successes': [], 69 | 'ndone': 0, 70 | 'nerrors': 0 71 | }; 72 | 73 | if (funcs.length === 0) { 74 | setImmediate(function () { callback(null, rv); }); 75 | return (rv); 76 | } 77 | 78 | doneOne = function (entry) { 79 | return (function (err, result) { 80 | mod_assert.equal(entry['status'], 'pending'); 81 | 82 | entry['err'] = err; 83 | entry['result'] = result; 84 | entry['status'] = err ? 'fail' : 'ok'; 85 | 86 | if (err) 87 | rv['nerrors']++; 88 | else 89 | rv['successes'].push(result); 90 | 91 | if (++rv['ndone'] < funcs.length) 92 | return; 93 | 94 | var errors = rv['operations'].filter(function (ent) { 95 | return (ent['status'] == 'fail'); 96 | }).map(function (ent) { return (ent['err']); }); 97 | 98 | if (errors.length > 0) 99 | callback(new mod_verror.MultiError(errors), rv); 100 | else 101 | callback(null, rv); 102 | }); 103 | }; 104 | 105 | for (i = 0; i < funcs.length; i++) { 106 | rv['operations'][i] = { 107 | 'func': funcs[i], 108 | 'funcname': funcs[i].name || '(anon)', 109 | 'status': 'pending' 110 | }; 111 | 112 | funcs[i](doneOne(rv['operations'][i])); 113 | } 114 | 115 | return (rv); 116 | } 117 | 118 | /* 119 | * Exactly like parallel, except that the input is specified as a single 120 | * function to invoke on N different inputs (rather than N functions). "args" 121 | * must have the following fields: 122 | * 123 | * func asynchronous function to invoke on each input value 124 | * 125 | * inputs array of input values 126 | */ 127 | function forEachParallel(args, callback) 128 | { 129 | var func, funcs; 130 | 131 | mod_assert.equal(typeof (args), 'object', '"args" must be an object'); 132 | mod_assert.equal(typeof (args['func']), 'function', 133 | '"args.func" must be specified and must be a function'); 134 | mod_assert.ok(Array.isArray(args['inputs']), 135 | '"args.inputs" must be specified and must be an array'); 136 | 137 | func = args['func']; 138 | funcs = args['inputs'].map(function (input) { 139 | return (function (subcallback) { 140 | return (func(input, subcallback)); 141 | }); 142 | }); 143 | 144 | return (parallel({ 'funcs': funcs }, callback)); 145 | } 146 | 147 | /* 148 | * Like parallel, but invokes functions in sequence rather than in parallel 149 | * and aborts if any function exits with failure. Arguments include: 150 | * 151 | * funcs invoke the functions in parallel 152 | * 153 | * arg first argument to each pipeline function 154 | */ 155 | function pipeline(args, callback) 156 | { 157 | mod_assert.equal(typeof (args), 'object', '"args" must be an object'); 158 | mod_assert.ok(Array.isArray(args['funcs']), 159 | '"args.funcs" must be specified and must be an array'); 160 | 161 | var opts = { 162 | 'funcs': args['funcs'].slice(0), 163 | 'callback': callback, 164 | 'args': { impl: 'pipeline', uarg: args['arg'] }, 165 | 'stop_when': 'error', 166 | 'res_type': 'rv' 167 | }; 168 | return (waterfall_impl(opts)); 169 | } 170 | 171 | function tryEach(funcs, callback) 172 | { 173 | mod_assert.ok(Array.isArray(funcs), 174 | '"funcs" must be specified and must be an array'); 175 | mod_assert.ok(arguments.length == 1 || typeof (callback) == 'function', 176 | '"callback" must be a function'); 177 | var opts = { 178 | 'funcs': funcs.slice(0), 179 | 'callback': callback, 180 | 'args': { impl: 'tryEach' }, 181 | 'stop_when': 'success', 182 | 'res_type': 'array' 183 | }; 184 | return (waterfall_impl(opts)); 185 | } 186 | 187 | /* 188 | * Exactly like pipeline, except that the input is specified as a single 189 | * function to invoke on N different inputs (rather than N functions). "args" 190 | * must have the following fields: 191 | * 192 | * func asynchronous function to invoke on each input value 193 | * 194 | * inputs array of input values 195 | */ 196 | function forEachPipeline(args, callback) { 197 | mod_assert.equal(typeof (args), 'object', '"args" must be an object'); 198 | mod_assert.equal(typeof (args['func']), 'function', 199 | '"args.func" must be specified and must be a function'); 200 | mod_assert.ok(Array.isArray(args['inputs']), 201 | '"args.inputs" must be specified and must be an array'); 202 | mod_assert.equal(typeof (callback), 'function', 203 | 'callback argument must be specified and must be a function'); 204 | 205 | var func = args['func']; 206 | 207 | var funcs = args['inputs'].map(function (input) { 208 | return (function (_, subcallback) { 209 | return (func(input, subcallback)); 210 | }); 211 | }); 212 | 213 | return (pipeline({'funcs': funcs}, callback)); 214 | } 215 | 216 | /* 217 | * async.js compatible filter, filterLimit, and filterSeries. Takes an input 218 | * array, optionally a limit, and a single function to filter an array and will 219 | * callback with a new filtered array. This is effectively an asynchronous 220 | * version of Array.prototype.filter. 221 | */ 222 | function filter(inputs, filterFunc, callback) { 223 | return (filterLimit(inputs, Infinity, filterFunc, callback)); 224 | } 225 | 226 | function filterSeries(inputs, filterFunc, callback) { 227 | return (filterLimit(inputs, 1, filterFunc, callback)); 228 | } 229 | 230 | function filterLimit(inputs, limit, filterFunc, callback) { 231 | mod_assert.ok(Array.isArray(inputs), 232 | '"inputs" must be specified and must be an array'); 233 | mod_assert.equal(typeof (limit), 'number', 234 | '"limit" must be a number'); 235 | mod_assert.equal(isNaN(limit), false, 236 | '"limit" must be a number'); 237 | mod_assert.equal(typeof (filterFunc), 'function', 238 | '"filterFunc" must be specified and must be a function'); 239 | mod_assert.equal(typeof (callback), 'function', 240 | '"callback" argument must be specified as a function'); 241 | 242 | var errors = []; 243 | var q = queue(processInput, limit); 244 | var results = []; 245 | 246 | function processInput(input, cb) { 247 | /* 248 | * If the errors array has any members, an error was 249 | * encountered in a previous invocation of filterFunc, so all 250 | * future filtering will be skipped. 251 | */ 252 | if (errors.length > 0) { 253 | cb(); 254 | return; 255 | } 256 | 257 | filterFunc(input.elem, function inputFiltered(err, ans) { 258 | /* 259 | * We ensure here that a filterFunc callback is only 260 | * ever invoked once. 261 | */ 262 | if (results.hasOwnProperty(input.idx)) { 263 | throw (new mod_verror.VError( 264 | 'vasync.filter*: filterFunc idx %d ' + 265 | 'invoked its callback twice', input.idx)); 266 | } 267 | 268 | /* 269 | * The original element, as well as the answer "ans" 270 | * (truth value) is stored to later be filtered when 271 | * all outstanding jobs are finished. 272 | */ 273 | results[input.idx] = { 274 | elem: input.elem, 275 | ans: !!ans 276 | }; 277 | 278 | /* 279 | * Any error encountered while filtering will result in 280 | * all future operations being skipped, and the error 281 | * object being returned in the users callback. 282 | */ 283 | if (err) { 284 | errors.push(err); 285 | cb(); 286 | return; 287 | } 288 | 289 | cb(); 290 | }); 291 | } 292 | 293 | q.once('end', function queueDrained() { 294 | if (errors.length > 0) { 295 | callback(mod_verror.errorFromList(errors)); 296 | return; 297 | } 298 | 299 | /* 300 | * results is now an array of objects in the same order of the 301 | * inputs array, where each object looks like: 302 | * 303 | * { 304 | * "ans": , 305 | * "elem": 306 | * } 307 | * 308 | * we filter out elements that have a false "ans" value, and 309 | * then map the array to contain only the input elements. 310 | */ 311 | results = results.filter(function filterFalseInputs(input) { 312 | return (input.ans); 313 | }).map(function mapInputElements(input) { 314 | return (input.elem); 315 | }); 316 | callback(null, results); 317 | }); 318 | 319 | inputs.forEach(function iterateInput(elem, idx) { 320 | /* 321 | * We retain the array index to ensure that order is 322 | * maintained. 323 | */ 324 | q.push({ 325 | elem: elem, 326 | idx: idx 327 | }); 328 | }); 329 | 330 | q.close(); 331 | 332 | return (q); 333 | } 334 | 335 | /* 336 | * async-compatible "whilst" function, with a few notable exceptions/addons. 337 | * 338 | * 1. More strict typing of arguments (functions *must* be supplied). 339 | * 2. A callback function is required, not optional. 340 | * 3. An object is returned, not undefined. 341 | */ 342 | function whilst(testFunc, iterateFunc, callback) { 343 | mod_assert.equal(typeof (testFunc), 'function', 344 | '"testFunc" must be specified and must be a function'); 345 | mod_assert.equal(typeof (iterateFunc), 'function', 346 | '"iterateFunc" must be specified and must be a function'); 347 | mod_assert.equal(typeof (callback), 'function', 348 | '"callback" argument must be specified as a function'); 349 | 350 | /* 351 | * The object returned to the caller that provides a read-only 352 | * interface to introspect this specific invocation of "whilst". 353 | */ 354 | var o = { 355 | 'finished': false, 356 | 'iterations': 0 357 | }; 358 | 359 | /* 360 | * Store the last set of arguments from the final call to "iterateFunc". 361 | * The arguments will be passed to the final callback when an error is 362 | * encountered or when the testFunc returns false. 363 | */ 364 | var args = []; 365 | 366 | function iterate() { 367 | var shouldContinue = testFunc(); 368 | 369 | if (!shouldContinue) { 370 | /* 371 | * The test condition is false - break out of the loop. 372 | */ 373 | done(); 374 | return; 375 | } 376 | 377 | /* Bump iterations after testFunc but before iterateFunc. */ 378 | o.iterations++; 379 | 380 | iterateFunc(function whilstIteration(err) { 381 | /* Store the latest set of arguments seen. */ 382 | args = Array.prototype.slice.call(arguments); 383 | 384 | /* Any error with iterateFunc will break the loop. */ 385 | if (err) { 386 | done(); 387 | return; 388 | } 389 | 390 | /* Try again. */ 391 | setImmediate(iterate); 392 | }); 393 | } 394 | 395 | function done() { 396 | mod_assert.ok(!o.finished, 'whilst already finished'); 397 | o.finished = true; 398 | callback.apply(this, args); 399 | } 400 | 401 | setImmediate(iterate); 402 | 403 | return (o); 404 | } 405 | 406 | /* 407 | * async-compatible "queue" function. 408 | */ 409 | function queue(worker, concurrency) 410 | { 411 | return (new WorkQueue({ 412 | 'worker': worker, 413 | 'concurrency': concurrency 414 | })); 415 | } 416 | 417 | function queuev(args) 418 | { 419 | return (new WorkQueue(args)); 420 | } 421 | 422 | function WorkQueue(args) 423 | { 424 | mod_assert.ok(args.hasOwnProperty('worker')); 425 | mod_assert.equal(typeof (args['worker']), 'function'); 426 | mod_assert.ok(args.hasOwnProperty('concurrency')); 427 | mod_assert.equal(typeof (args['concurrency']), 'number'); 428 | mod_assert.equal(Math.floor(args['concurrency']), args['concurrency']); 429 | mod_assert.ok(args['concurrency'] > 0); 430 | 431 | mod_events.EventEmitter.call(this); 432 | 433 | this.nextid = 0; 434 | this.worker = args['worker']; 435 | this.worker_name = args['worker'].name || 'anon'; 436 | this.npending = 0; 437 | this.pending = {}; 438 | this.queued = []; 439 | this.closed = false; 440 | this.ended = false; 441 | 442 | /* user-settable fields inherited from "async" interface */ 443 | this.concurrency = args['concurrency']; 444 | this.saturated = undefined; 445 | this.empty = undefined; 446 | this.drain = undefined; 447 | } 448 | 449 | mod_util.inherits(WorkQueue, mod_events.EventEmitter); 450 | 451 | WorkQueue.prototype.push = function (tasks, callback) 452 | { 453 | if (!Array.isArray(tasks)) 454 | return (this.pushOne(tasks, callback)); 455 | 456 | var wq = this; 457 | return (tasks.map(function (task) { 458 | return (wq.pushOne(task, callback)); 459 | })); 460 | }; 461 | 462 | WorkQueue.prototype.updateConcurrency = function (concurrency) 463 | { 464 | if (this.closed) 465 | throw new mod_verror.VError( 466 | 'update concurrency invoked after queue closed'); 467 | this.concurrency = concurrency; 468 | this.dispatchNext(); 469 | }; 470 | 471 | WorkQueue.prototype.close = function () 472 | { 473 | var wq = this; 474 | 475 | if (wq.closed) 476 | return; 477 | wq.closed = true; 478 | 479 | /* 480 | * If the queue is already empty, just fire the "end" event on the 481 | * next tick. 482 | */ 483 | if (wq.npending === 0 && wq.queued.length === 0) { 484 | setImmediate(function () { 485 | if (!wq.ended) { 486 | wq.ended = true; 487 | wq.emit('end'); 488 | } 489 | }); 490 | } 491 | }; 492 | 493 | /* private */ 494 | WorkQueue.prototype.pushOne = function (task, callback) 495 | { 496 | if (this.closed) 497 | throw new mod_verror.VError('push invoked after queue closed'); 498 | 499 | var id = ++this.nextid; 500 | var entry = { 'id': id, 'task': task, 'callback': callback }; 501 | 502 | this.queued.push(entry); 503 | this.dispatchNext(); 504 | 505 | return (id); 506 | }; 507 | 508 | /* private */ 509 | WorkQueue.prototype.dispatchNext = function () 510 | { 511 | var wq = this; 512 | if (wq.npending === 0 && wq.queued.length === 0) { 513 | if (wq.drain) 514 | wq.drain(); 515 | wq.emit('drain'); 516 | /* 517 | * The queue is closed; emit the final "end" 518 | * event before we come to rest: 519 | */ 520 | if (wq.closed) { 521 | wq.ended = true; 522 | wq.emit('end'); 523 | } 524 | } else if (wq.queued.length > 0) { 525 | while (wq.queued.length > 0 && wq.npending < wq.concurrency) { 526 | var next = wq.queued.shift(); 527 | wq.dispatch(next); 528 | 529 | if (wq.queued.length === 0) { 530 | if (wq.empty) 531 | wq.empty(); 532 | wq.emit('empty'); 533 | } 534 | } 535 | } 536 | }; 537 | 538 | WorkQueue.prototype.dispatch = function (entry) 539 | { 540 | var wq = this; 541 | 542 | mod_assert.ok(!this.pending.hasOwnProperty(entry['id'])); 543 | mod_assert.ok(this.npending < this.concurrency); 544 | mod_assert.ok(!this.ended); 545 | 546 | this.npending++; 547 | this.pending[entry['id']] = entry; 548 | 549 | if (this.npending === this.concurrency) { 550 | if (this.saturated) 551 | this.saturated(); 552 | this.emit('saturated'); 553 | } 554 | 555 | /* 556 | * We invoke the worker function on the next tick so that callers can 557 | * always assume that the callback is NOT invoked during the call to 558 | * push() even if the queue is not at capacity. It also avoids O(n) 559 | * stack usage when used with synchronous worker functions. 560 | */ 561 | setImmediate(function () { 562 | wq.worker(entry['task'], function (err) { 563 | --wq.npending; 564 | delete (wq.pending[entry['id']]); 565 | 566 | if (entry['callback']) 567 | entry['callback'].apply(null, arguments); 568 | 569 | wq.dispatchNext(); 570 | }); 571 | }); 572 | }; 573 | 574 | WorkQueue.prototype.length = function () 575 | { 576 | return (this.queued.length); 577 | }; 578 | 579 | WorkQueue.prototype.kill = function () 580 | { 581 | this.killed = true; 582 | this.queued = []; 583 | this.drain = undefined; 584 | this.close(); 585 | }; 586 | 587 | /* 588 | * Barriers coordinate multiple concurrent operations. 589 | */ 590 | function barrier(args) 591 | { 592 | return (new Barrier(args)); 593 | } 594 | 595 | function Barrier(args) 596 | { 597 | mod_assert.ok(!args || !args['nrecent'] || 598 | typeof (args['nrecent']) == 'number', 599 | '"nrecent" must have type "number"'); 600 | 601 | mod_events.EventEmitter.call(this); 602 | 603 | var nrecent = args && args['nrecent'] ? args['nrecent'] : 10; 604 | 605 | if (nrecent > 0) { 606 | this.nrecent = nrecent; 607 | this.recent = []; 608 | } 609 | 610 | this.pending = {}; 611 | this.scheduled = false; 612 | } 613 | 614 | mod_util.inherits(Barrier, mod_events.EventEmitter); 615 | 616 | Barrier.prototype.start = function (name) 617 | { 618 | mod_assert.ok(!this.pending.hasOwnProperty(name), 619 | 'operation "' + name + '" is already pending'); 620 | this.pending[name] = Date.now(); 621 | }; 622 | 623 | Barrier.prototype.done = function (name) 624 | { 625 | mod_assert.ok(this.pending.hasOwnProperty(name), 626 | 'operation "' + name + '" is not pending'); 627 | 628 | if (this.recent) { 629 | this.recent.push({ 630 | 'name': name, 631 | 'start': this.pending[name], 632 | 'done': Date.now() 633 | }); 634 | 635 | if (this.recent.length > this.nrecent) 636 | this.recent.shift(); 637 | } 638 | 639 | delete (this.pending[name]); 640 | 641 | /* 642 | * If we executed at least one operation and we're now empty, we should 643 | * emit "drain". But most code doesn't deal well with events being 644 | * processed while they're executing, so we actually schedule this event 645 | * for the next tick. 646 | * 647 | * We use the "scheduled" flag to avoid emitting multiple "drain" events 648 | * on consecutive ticks if the user starts and ends another task during 649 | * this tick. 650 | */ 651 | if (!isEmpty(this.pending) || this.scheduled) 652 | return; 653 | 654 | this.scheduled = true; 655 | 656 | var self = this; 657 | 658 | setImmediate(function () { 659 | self.scheduled = false; 660 | 661 | /* 662 | * It's also possible that the user has started another task on 663 | * the previous tick, in which case we really shouldn't emit 664 | * "drain". 665 | */ 666 | if (isEmpty(self.pending)) 667 | self.emit('drain'); 668 | }); 669 | }; 670 | 671 | /* 672 | * waterfall([ funcs ], callback): invoke each of the asynchronous functions 673 | * "funcs" in series. Each function is passed any values emitted by the 674 | * previous function (none for the first function), followed by the callback to 675 | * invoke upon completion. This callback must be invoked exactly once, 676 | * regardless of success or failure. As conventional in Node, the first 677 | * argument to the callback indicates an error (if non-null). Subsequent 678 | * arguments are passed to the next function in the "funcs" chain. 679 | * 680 | * If any function fails (i.e., calls its callback with an Error), then the 681 | * remaining functions are not invoked and "callback" is invoked with the error. 682 | * 683 | * The only difference between waterfall() and pipeline() are the arguments 684 | * passed to each function in the chain. pipeline() always passes the same 685 | * argument followed by the callback, while waterfall() passes whatever values 686 | * were emitted by the previous function followed by the callback. 687 | */ 688 | function waterfall(funcs, callback) 689 | { 690 | mod_assert.ok(Array.isArray(funcs), 691 | '"funcs" must be specified and must be an array'); 692 | mod_assert.ok(arguments.length == 1 || typeof (callback) == 'function', 693 | '"callback" must be a function'); 694 | var opts = { 695 | 'funcs': funcs.slice(0), 696 | 'callback': callback, 697 | 'args': { impl: 'waterfall' }, 698 | 'stop_when': 'error', 699 | 'res_type': 'values' 700 | }; 701 | return (waterfall_impl(opts)); 702 | } 703 | 704 | /* 705 | * This function is used to implement vasync-functions that need to execute a 706 | * list of functions in a sequence, but differ in how they make use of the 707 | * intermediate callbacks and finall callback, as well as under what conditions 708 | * they stop executing the functions in the list. Examples of such functions 709 | * are `pipeline`, `waterfall`, and `tryEach`. See the documentation for those 710 | * functions to see how they operate. 711 | * 712 | * This function's behavior is influenced via the `opts` object that we pass 713 | * in. This object has the following layout: 714 | * 715 | * { 716 | * 'funcs': array of functions 717 | * 'callback': the final callback 718 | * 'args': { 719 | * 'impl': 'pipeline' or 'tryEach' or 'waterfall' 720 | * 'uarg': the arg passed to each func for 'pipeline' 721 | * } 722 | * 'stop_when': 'error' or 'success' 723 | * 'res_type': 'values' or 'arrays' or 'rv' 724 | * } 725 | * 726 | * In the object, 'res_type' is used to indicate what the type of the result 727 | * values(s) is that we pass to the final callback. We secondarily use 728 | * 'args.impl' to adjust this behavior in an implementation-specific way. For 729 | * example, 'tryEach' only returns an array if it has more than 1 result passed 730 | * to the final callback. Otherwise, it passes a solitary value to the final 731 | * callback. 732 | * 733 | * In case it's not clear, 'rv' in the `res_type` member, is just the 734 | * result-value that we also return. This is the convention in functions that 735 | * originated in `vasync` (pipeline), but not in functions that originated in 736 | * `async` (waterfall, tryEach). 737 | */ 738 | function waterfall_impl(opts) 739 | { 740 | mod_assert.ok(typeof (opts) === 'object'); 741 | var rv, current, next; 742 | var funcs = opts.funcs; 743 | var callback = opts.callback; 744 | 745 | mod_assert.ok(Array.isArray(funcs), 746 | '"opts.funcs" must be specified and must be an array'); 747 | mod_assert.ok(arguments.length == 1, 748 | 'Function "waterfall_impl" must take only 1 arg'); 749 | mod_assert.ok(opts.res_type === 'values' || 750 | opts.res_type === 'array' || opts.res_type == 'rv', 751 | '"opts.res_type" must either be "values", "array", or "rv"'); 752 | mod_assert.ok(opts.stop_when === 'error' || 753 | opts.stop_when === 'success', 754 | '"opts.stop_when" must either be "error" or "success"'); 755 | mod_assert.ok(opts.args.impl === 'pipeline' || 756 | opts.args.impl === 'waterfall' || opts.args.impl === 'tryEach', 757 | '"opts.args.impl" must be "pipeline", "waterfall", or "tryEach"'); 758 | if (opts.args.impl === 'pipeline') { 759 | mod_assert.ok(typeof (opts.args.uarg) !== undefined, 760 | '"opts.args.uarg" should be defined when pipeline is used'); 761 | } 762 | 763 | rv = { 764 | 'operations': funcs.map(function (func) { 765 | return ({ 766 | 'func': func, 767 | 'funcname': func.name || '(anon)', 768 | 'status': 'waiting' 769 | }); 770 | }), 771 | 'successes': [], 772 | 'ndone': 0, 773 | 'nerrors': 0 774 | }; 775 | 776 | if (funcs.length === 0) { 777 | if (callback) 778 | setImmediate(function () { 779 | var res = (opts.args.impl === 'pipeline') ? rv 780 | : undefined; 781 | callback(null, res); 782 | }); 783 | return (rv); 784 | } 785 | 786 | next = function (idx, err) { 787 | /* 788 | * Note that nfunc_args contains the args we will pass to the 789 | * next func in the func-list the user gave us. Except for 790 | * 'tryEach', which passes cb's. However, it will pass 791 | * 'nfunc_args' to its final callback -- see below. 792 | */ 793 | var res_key, nfunc_args, entry, nextentry; 794 | 795 | if (err === undefined) 796 | err = null; 797 | 798 | if (idx != current) { 799 | throw (new mod_verror.VError( 800 | 'vasync.waterfall: function %d ("%s") invoked ' + 801 | 'its callback twice', idx, 802 | rv['operations'][idx].funcname)); 803 | } 804 | 805 | mod_assert.equal(idx, rv['ndone'], 806 | 'idx should be equal to ndone'); 807 | entry = rv['operations'][rv['ndone']++]; 808 | if (opts.args.impl === 'tryEach' || 809 | opts.args.impl === 'waterfall') { 810 | nfunc_args = Array.prototype.slice.call(arguments, 2); 811 | res_key = 'results'; 812 | entry['results'] = nfunc_args; 813 | } else if (opts.args.impl === 'pipeline') { 814 | nfunc_args = [ opts.args.uarg ]; 815 | res_key = 'result'; 816 | entry['result'] = arguments[2]; 817 | } 818 | 819 | mod_assert.equal(entry['status'], 'pending', 820 | 'status should be pending'); 821 | entry['status'] = err ? 'fail' : 'ok'; 822 | entry['err'] = err; 823 | 824 | if (err) { 825 | rv['nerrors']++; 826 | } else { 827 | rv['successes'].push(entry[res_key]); 828 | } 829 | 830 | if ((opts.stop_when === 'error' && err) || 831 | (opts.stop_when === 'success' && 832 | rv['successes'].length > 0) || 833 | rv['ndone'] == funcs.length) { 834 | if (callback) { 835 | if (opts.res_type === 'values' || 836 | (opts.res_type === 'array' && 837 | nfunc_args.length <= 1)) { 838 | nfunc_args.unshift(err); 839 | callback.apply(null, nfunc_args); 840 | } else if (opts.res_type === 'array') { 841 | callback(err, nfunc_args); 842 | } else if (opts.res_type === 'rv') { 843 | callback(err, rv); 844 | } 845 | } 846 | } else { 847 | nextentry = rv['operations'][rv['ndone']]; 848 | nextentry['status'] = 'pending'; 849 | current++; 850 | nfunc_args.push(next.bind(null, current)); 851 | setImmediate(function () { 852 | var nfunc = nextentry['func']; 853 | /* 854 | * At first glance it may seem like this branch 855 | * is superflous with the code above that 856 | * branches on `opts.args.impl`. It may also 857 | * seem like calling `nfunc.apply` is 858 | * sufficient for both cases (after all we 859 | * pushed `next.bind(null, current)` to the 860 | * `nfunc_args` array), before we call 861 | * `setImmediate()`. However, this is not the 862 | * case, because the interface exposed by 863 | * tryEach is different from the others. The 864 | * others pass argument(s) from task to task. 865 | * tryEach passes nothing but a callback 866 | * (`next.bind` below). However, the callback 867 | * itself _can_ be called with one or more 868 | * results, which we collect into `nfunc_args` 869 | * using the aformentioned `opts.args.impl` 870 | * branch above, and which we pass to the 871 | * callback via the `opts.res_type` branch 872 | * above (where res_type is set to 'array'). 873 | */ 874 | if (opts.args.impl !== 'tryEach') { 875 | nfunc.apply(null, nfunc_args); 876 | } else { 877 | nfunc(next.bind(null, current)); 878 | } 879 | }); 880 | } 881 | }; 882 | 883 | rv['operations'][0]['status'] = 'pending'; 884 | current = 0; 885 | if (opts.args.impl !== 'pipeline') { 886 | funcs[0](next.bind(null, current)); 887 | } else { 888 | funcs[0](opts.args.uarg, next.bind(null, current)); 889 | } 890 | return (rv); 891 | } 892 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vasync", 3 | "version": "2.2.1", 4 | "description": "utilities for observable asynchronous control flow", 5 | "main": "./lib/vasync.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "https://github.com/joyent/node-vasync.git" 9 | }, 10 | "scripts": { 11 | "test": "./node_modules/.bin/tap --stdout tests/ && ./node_modules/.bin/nodeunit tests/compat.js && ./node_modules/.bin/nodeunit tests/compat_tryEach.js" 12 | }, 13 | "devDependencies": { 14 | "tap": "~0.4.8", 15 | "nodeunit": "0.8.7" 16 | }, 17 | "dependencies": { 18 | "verror": "1.10.0" 19 | }, 20 | "engines": [ 21 | "node >=0.6.0" 22 | ], 23 | "license": "MIT" 24 | } 25 | -------------------------------------------------------------------------------- /tests/compat.js: -------------------------------------------------------------------------------- 1 | /* 2 | * tests/compat.js: Some of the APIs provided by vasync are intended to be 3 | * API-compatible with node-async, so we incorporate the tests from node-async 4 | * directly here. These are copied from https://github.com/caolan/async, 5 | * available under the MIT license. To make it easy to update this from the 6 | * source, this file should remain unchanged from the source except for this 7 | * header comment, the change to the "require" line, and deleted lines for 8 | * unimplemented functions. 9 | * 10 | * The following tests are deliberately omitted: 11 | * 12 | * o "waterfall non-array": Per Joyent's Best Practices for Node.js Error 13 | * Handling, we're strict about argument types and throw on these programmer 14 | * errors rather than emitting them asynchronously. 15 | * 16 | * o "waterfall multiple callback calls": We deliberately disallow a waterfall 17 | * function to invoke its callback more than once, so we don't test for that 18 | * here. The behavior that node-async allows can potentially be used to fork 19 | * the waterfall, which may be useful, but it's often used instead as an 20 | * excuse to write code sloppily. And the downside is that it makes it really 21 | * hard to understand bugs where the waterfall was resumed too early. For 22 | * now, we're disallowing it, but if the forking behavior becomes useful, we 23 | * can always make our version less strict. 24 | */ 25 | var async = require('../lib/vasync'); 26 | 27 | exports['waterfall'] = function(test){ 28 | test.expect(6); 29 | var call_order = []; 30 | async.waterfall([ 31 | function(callback){ 32 | call_order.push('fn1'); 33 | setTimeout(function(){callback(null, 'one', 'two');}, 0); 34 | }, 35 | function(arg1, arg2, callback){ 36 | call_order.push('fn2'); 37 | test.equals(arg1, 'one'); 38 | test.equals(arg2, 'two'); 39 | setTimeout(function(){callback(null, arg1, arg2, 'three');}, 25); 40 | }, 41 | function(arg1, arg2, arg3, callback){ 42 | call_order.push('fn3'); 43 | test.equals(arg1, 'one'); 44 | test.equals(arg2, 'two'); 45 | test.equals(arg3, 'three'); 46 | callback(null, 'four'); 47 | }, 48 | function(arg4, callback){ 49 | call_order.push('fn4'); 50 | test.same(call_order, ['fn1','fn2','fn3','fn4']); 51 | callback(null, 'test'); 52 | } 53 | ], function(err){ 54 | test.done(); 55 | }); 56 | }; 57 | 58 | exports['waterfall empty array'] = function(test){ 59 | async.waterfall([], function(err){ 60 | test.done(); 61 | }); 62 | }; 63 | 64 | exports['waterfall no callback'] = function(test){ 65 | async.waterfall([ 66 | function(callback){callback();}, 67 | function(callback){callback(); test.done();} 68 | ]); 69 | }; 70 | 71 | exports['waterfall async'] = function(test){ 72 | var call_order = []; 73 | async.waterfall([ 74 | function(callback){ 75 | call_order.push(1); 76 | callback(); 77 | call_order.push(2); 78 | }, 79 | function(callback){ 80 | call_order.push(3); 81 | callback(); 82 | }, 83 | function(){ 84 | test.same(call_order, [1,2,3]); 85 | test.done(); 86 | } 87 | ]); 88 | }; 89 | 90 | exports['waterfall error'] = function(test){ 91 | test.expect(1); 92 | async.waterfall([ 93 | function(callback){ 94 | callback('error'); 95 | }, 96 | function(callback){ 97 | test.ok(false, 'next function should not be called'); 98 | callback(); 99 | } 100 | ], function(err){ 101 | test.equals(err, 'error'); 102 | }); 103 | setTimeout(test.done, 50); 104 | }; 105 | -------------------------------------------------------------------------------- /tests/compat_tryEach.js: -------------------------------------------------------------------------------- 1 | var async = require('../lib/vasync'); 2 | 3 | /* 4 | * tryEach tests, transliterated from mocha to tap. 5 | * 6 | * They are nearly identical except for some details related to vasync. For 7 | * example, we don't support calling the callback more than once from any of 8 | * the given functions. 9 | */ 10 | 11 | 12 | exports['tryEach no callback'] = function (test) { 13 | async.tryEach([]); 14 | test.done(); 15 | }; 16 | exports['tryEach empty'] = function (test) { 17 | async.tryEach([], function (err, results) { 18 | test.equals(err, null); 19 | test.same(results, undefined); 20 | test.done(); 21 | }); 22 | }; 23 | exports['tryEach one task, multiple results'] = function (test) { 24 | var RESULTS = ['something', 'something2']; 25 | async.tryEach([ 26 | function (callback) { 27 | callback(null, RESULTS[0], RESULTS[1]); 28 | } 29 | ], function (err, results) { 30 | test.equals(err, null); 31 | test.same(results, RESULTS); 32 | test.done(); 33 | }); 34 | }; 35 | exports['tryEach one task'] = function (test) { 36 | var RESULT = 'something'; 37 | async.tryEach([ 38 | function (callback) { 39 | callback(null, RESULT); 40 | } 41 | ], function (err, results) { 42 | test.equals(err, null); 43 | test.same(results, RESULT); 44 | test.done(); 45 | }); 46 | }; 47 | exports['tryEach two tasks, one failing'] = function (test) { 48 | var RESULT = 'something'; 49 | async.tryEach([ 50 | function (callback) { 51 | callback(new Error('Failure'), {}); 52 | }, 53 | function (callback) { 54 | callback(null, RESULT); 55 | } 56 | ], function (err, results) { 57 | test.equals(err, null); 58 | test.same(results, RESULT); 59 | test.done(); 60 | }); 61 | }; 62 | exports['tryEach two tasks, both failing'] = function (test) { 63 | var ERROR_RESULT = new Error('Failure2'); 64 | async.tryEach([ 65 | function (callback) { 66 | callback(new Error('Should not stop here')); 67 | }, 68 | function (callback) { 69 | callback(ERROR_RESULT); 70 | } 71 | ], function (err, results) { 72 | test.equals(err, ERROR_RESULT); 73 | test.same(results, undefined); 74 | test.done(); 75 | }); 76 | }; 77 | exports['tryEach two tasks, non failing'] = function (test) { 78 | var RESULT = 'something'; 79 | async.tryEach([ 80 | function (callback) { 81 | callback(null, RESULT); 82 | }, 83 | function () { 84 | test.fail('Should not been called'); 85 | } 86 | ], function (err, results) { 87 | test.equals(err, null); 88 | test.same(results, RESULT); 89 | test.done(); 90 | }); 91 | }; 92 | -------------------------------------------------------------------------------- /tests/filter.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Tests the "filter", "filterSeries", and "filterLimit" functions 3 | */ 4 | 5 | var mod_util = require('util'); 6 | 7 | var mod_tap = require('tap'); 8 | var mod_vasync = require('..'); 9 | 10 | mod_tap.test('filterSeries', function (test) { 11 | var inputs = [0, 1, 2, 3, 4, 5]; 12 | var curTasks = 0; 13 | var maxTasks = 0; 14 | // filterSeries has an implicit limit of 1 concurrent operation 15 | var limit = 1; 16 | 17 | function filterFunc(input, cb) { 18 | curTasks++; 19 | if (curTasks > maxTasks) { 20 | maxTasks = curTasks; 21 | } 22 | test.ok(curTasks <= limit, mod_util.format( 23 | 'input %d: current tasks %d <= %d', 24 | input, curTasks, limit)); 25 | 26 | setTimeout(function () { 27 | curTasks--; 28 | cb(null, input < 2 || input === 4); 29 | }, 50); 30 | } 31 | 32 | mod_vasync.filterSeries(inputs, filterFunc, 33 | function filterDone(err, results) { 34 | 35 | test.ok(!err, 'error unset'); 36 | test.equal(maxTasks, limit, 'max tasks reached limit'); 37 | test.deepEqual(results, [0, 1, 4], 'results array correct'); 38 | test.end(); 39 | }); 40 | }); 41 | 42 | mod_tap.test('filterLimit', function (test) { 43 | var inputs = [0, 1, 2, 3, 4, 5]; 44 | var curTasks = 0; 45 | var maxTasks = 0; 46 | var limit = 2; 47 | 48 | function filterFunc(input, cb) { 49 | curTasks++; 50 | if (curTasks > maxTasks) { 51 | maxTasks = curTasks; 52 | } 53 | test.ok(curTasks <= limit, mod_util.format( 54 | 'input %d: current tasks %d <= %d', 55 | input, curTasks, limit)); 56 | 57 | setTimeout(function () { 58 | curTasks--; 59 | cb(null, input < 2 || input === 4); 60 | }, 50); 61 | } 62 | 63 | mod_vasync.filterLimit(inputs, limit, filterFunc, 64 | function filterDone(err, results) { 65 | 66 | test.ok(!err, 'error unset'); 67 | test.equal(maxTasks, limit, 'max tasks reached limit'); 68 | test.deepEqual(results, [0, 1, 4], 'results array correct'); 69 | test.end(); 70 | }); 71 | }); 72 | 73 | mod_tap.test('filter (maintain order)', function (test) { 74 | var inputs = [0, 1, 2, 3, 4, 5]; 75 | var limit = inputs.length; 76 | var storedValues = []; 77 | 78 | function filterFunc(input, cb) { 79 | /* 80 | * Hold every callback in an array to be called when all 81 | * filterFunc's have run. This way, we can ensure that all 82 | * tasks have started without waiting for any others to finish. 83 | */ 84 | storedValues.push({ 85 | input: input, 86 | cb: cb 87 | }); 88 | 89 | test.ok(storedValues.length <= limit, mod_util.format( 90 | 'input %d: current tasks %d <= %d', 91 | input, storedValues.length, limit)); 92 | 93 | /* 94 | * When this constraint is true, all filterFunc's have run for 95 | * each input. We now call all callbacks in a pre-determined 96 | * order (out of order of the original) to ensure the final 97 | * array is in the correct order. 98 | */ 99 | if (storedValues.length === inputs.length) { 100 | [5, 2, 0, 1, 4, 3].forEach(function (i) { 101 | var o = storedValues[i]; 102 | o.cb(null, o.input < 2 || o.input === 4); 103 | }); 104 | } 105 | } 106 | 107 | mod_vasync.filter(inputs, filterFunc, 108 | function filterDone(err, results) { 109 | 110 | test.ok(!err, 'error unset'); 111 | test.equal(storedValues.length, inputs.length, 112 | 'max tasks reached limit'); 113 | test.deepEqual(results, [0, 1, 4], 'results array correct'); 114 | test.end(); 115 | }); 116 | }); 117 | 118 | mod_tap.test('filterSeries error handling', function (test) { 119 | /* 120 | * We will error half way through the list of inputs to ensure that 121 | * first half are processed while the second half are ignored. 122 | */ 123 | var inputs = [0, 1, 2, 3, 4, 5]; 124 | 125 | function filterFunc(input, cb) { 126 | switch (input) { 127 | case 0: 128 | case 1: 129 | case 2: 130 | cb(null, true); 131 | break; 132 | case 3: 133 | cb(new Error('error on ' + input)); 134 | break; 135 | case 4: 136 | case 5: 137 | test.ok(false, 'processed too many inputs'); 138 | cb(new Error('processed too many inputs')); 139 | break; 140 | default: 141 | test.ok(false, 'unexpected input: ' + input); 142 | cb(new Error('unexpected input')); 143 | break; 144 | } 145 | } 146 | 147 | mod_vasync.filterSeries(inputs, filterFunc, 148 | function filterDone(err, results) { 149 | 150 | test.ok(err, 'error set'); 151 | test.ok(err.message === 'error on 3', 'error on input 3'); 152 | test.ok(results === undefined, 'results is unset'); 153 | test.end(); 154 | }); 155 | }); 156 | 157 | mod_tap.test('filterSeries double callback', function (test) { 158 | var inputs = [0, 1, 2, 3, 4, 5]; 159 | 160 | function filterFunc(input, cb) { 161 | switch (input) { 162 | case 0: 163 | case 1: 164 | case 2: 165 | cb(null, true); 166 | break; 167 | case 3: 168 | /* 169 | * The first call to cb() should "win" - meaning this 170 | * value will be filtered out of the final array of 171 | * results. 172 | */ 173 | cb(null, false); 174 | test.throws(function () { 175 | cb(null, true); 176 | }); 177 | break; 178 | case 4: 179 | /* 180 | * Like input 3, all subsequent calls to cb() will 181 | * throw an error and not affect the original call to 182 | * cb(). 183 | */ 184 | cb(null, true); 185 | test.throws(function () { 186 | cb(new Error('uh oh')); 187 | }); 188 | break; 189 | case 5: 190 | cb(null, true); 191 | break; 192 | default: 193 | test.ok(false, 'unexpected input: ' + input); 194 | cb(new Error('unexpected input')); 195 | break; 196 | } 197 | } 198 | 199 | mod_vasync.filterSeries(inputs, filterFunc, 200 | function filterDone(err, results) { 201 | 202 | test.ok(!err, 'error not set'); 203 | test.deepEqual(results, [0, 1, 2, 4, 5], 204 | 'results array correct'); 205 | test.end(); 206 | }); 207 | }); 208 | 209 | mod_tap.test('filter push to queue object error', function (test) { 210 | var inputs = [0, 1, 2, 3, 4, 5]; 211 | 212 | function filterFunc(input, cb) { 213 | cb(null, true); 214 | } 215 | 216 | var q = mod_vasync.filterSeries(inputs, filterFunc, 217 | function filterDone(err, results) { 218 | 219 | test.end(); 220 | }); 221 | 222 | test.equal(q.closed, true, 'queue is closed'); 223 | test.throws(function () { 224 | q.push(6); 225 | }); 226 | }); 227 | -------------------------------------------------------------------------------- /tests/issue-21.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Tests that if the user modifies the list of functions passed to 3 | * vasync.pipeline, vasync ignores the changes and does not crash. 4 | */ 5 | var assert = require('assert'); 6 | var vasync = require('../lib/vasync'); 7 | var count = 0; 8 | var funcs; 9 | 10 | function doStuff(_, callback) 11 | { 12 | count++; 13 | setImmediate(callback); 14 | } 15 | 16 | funcs = [ doStuff, doStuff, doStuff ]; 17 | 18 | vasync.pipeline({ 19 | 'funcs': funcs 20 | }, function (err) { 21 | assert.ok(!err); 22 | assert.ok(count === 3); 23 | }); 24 | 25 | funcs.push(doStuff); 26 | -------------------------------------------------------------------------------- /tests/pipeline.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Tests the "pipeline" primitive. 3 | */ 4 | 5 | var mod_tap = require('tap'); 6 | var mod_vasync = require('..'); 7 | var st; 8 | 9 | 10 | mod_tap.test('empty pipeline', function (test) { 11 | var count = 0; 12 | st = mod_vasync.pipeline({'funcs': [], 'arg': null}, 13 | function (err, result) { 14 | 15 | test.ok(err === null); 16 | test.ok(result.ndone === 0); 17 | test.ok(result.nerrors === 0); 18 | test.ok(result.operations.length === 0); 19 | test.ok(result.successes.length === 0); 20 | test.equal(count, 1); 21 | test.end(); 22 | }); 23 | count++; 24 | test.ok(st.ndone === 0); 25 | test.ok(st.nerrors === 0); 26 | test.ok(st.operations.length === 0); 27 | test.ok(st.successes.length === 0); 28 | }); 29 | 30 | mod_tap.test('normal 4-stage pipeline', function (test) { 31 | var count = 0; 32 | st = mod_vasync.pipeline({'funcs': [ 33 | function func1(_, cb) { 34 | test.equal(st.successes[0], undefined, 35 | 'func1: successes'); 36 | test.ok(count === 0, 'func1: count === 0'); 37 | test.ok(st.ndone === 0); 38 | count++; 39 | setImmediate(cb, null, count); 40 | }, 41 | function func2(_, cb) { 42 | test.equal(st.successes[0], 1, 'func2: successes'); 43 | test.ok(count == 1, 'func2: count == 1'); 44 | test.ok(st.ndone === 1); 45 | test.ok(st.operations[0].status == 'ok'); 46 | test.ok(st.operations[1].status == 'pending'); 47 | test.ok(st.operations[2].status == 'waiting'); 48 | count++; 49 | setImmediate(cb, null, count); 50 | }, 51 | function (_, cb) { 52 | test.equal(st.successes[0], 1, 'func3: successes'); 53 | test.equal(st.successes[1], 2, 'func3: successes'); 54 | test.ok(count == 2, 'func3: count == 2'); 55 | test.ok(st.ndone === 2); 56 | count++; 57 | setImmediate(cb, null, count); 58 | }, 59 | function func4(_, cb) { 60 | test.equal(st.successes[0], 1, 'func4: successes'); 61 | test.equal(st.successes[1], 2, 'func4: successes'); 62 | test.equal(st.successes[2], 3, 'func4: successes'); 63 | test.ok(count == 3, 'func4: count == 3'); 64 | test.ok(st.ndone === 3); 65 | count++; 66 | setImmediate(cb, null, count); 67 | } 68 | ]}, function (err, result) { 69 | test.ok(count == 4, 'final: count == 4'); 70 | test.ok(err === null, 'no error'); 71 | test.ok(result === st); 72 | test.equal(result, st, 'final-cb: st == result'); 73 | test.equal(st.successes[0], 1, 'final-cb: successes'); 74 | test.equal(st.successes[1], 2, 'final-cb: successes'); 75 | test.equal(st.successes[2], 3, 'final-cb: successes'); 76 | test.equal(st.successes[3], 4, 'final-cb: successes'); 77 | test.ok(st.ndone === 4); 78 | test.ok(st.nerrors === 0); 79 | test.ok(st.operations.length === 4); 80 | test.ok(st.successes.length === 4); 81 | test.ok(st.operations[0].status == 'ok'); 82 | test.ok(st.operations[1].status == 'ok'); 83 | test.ok(st.operations[2].status == 'ok'); 84 | test.ok(st.operations[3].status == 'ok'); 85 | test.end(); 86 | }); 87 | test.ok(st.ndone === 0); 88 | test.ok(st.nerrors === 0); 89 | test.ok(st.operations.length === 4); 90 | test.ok(st.operations[0].funcname == 'func1', 'func1 name'); 91 | test.ok(st.operations[0].status == 'pending'); 92 | test.ok(st.operations[1].funcname == 'func2', 'func2 name'); 93 | test.ok(st.operations[1].status == 'waiting'); 94 | test.ok(st.operations[2].funcname == '(anon)', 'anon name'); 95 | test.ok(st.operations[2].status == 'waiting'); 96 | test.ok(st.operations[3].funcname == 'func4', 'func4 name'); 97 | test.ok(st.operations[3].status == 'waiting'); 98 | test.ok(st.successes.length === 0); 99 | }); 100 | 101 | mod_tap.test('bailing out early', function (test) { 102 | var count = 0; 103 | st = mod_vasync.pipeline({'funcs': [ 104 | function func1(_, cb) { 105 | test.ok(count === 0, 'func1: count === 0'); 106 | count++; 107 | setImmediate(cb, null, count); 108 | }, 109 | function func2(_, cb) { 110 | test.ok(count == 1, 'func2: count == 1'); 111 | count++; 112 | setImmediate(cb, new Error('boom!')); 113 | }, 114 | function func3(_, cb) { 115 | test.ok(count == 2, 'func3: count == 2'); 116 | count++; 117 | setImmediate(cb, null, count); 118 | } 119 | ]}, function (err, result) { 120 | test.ok(count == 2, 'final: count == 3'); 121 | test.equal(err.message, 'boom!'); 122 | test.ok(result === st); 123 | test.equal(result, st, 'final-cb: st == result'); 124 | test.ok(st.ndone == 2); 125 | test.ok(st.nerrors == 1); 126 | test.ok(st.operations[0].status == 'ok'); 127 | test.ok(st.operations[1].status == 'fail'); 128 | test.ok(st.operations[2].status == 'waiting'); 129 | test.ok(st.successes.length == 1); 130 | test.end(); 131 | }); 132 | }); 133 | -------------------------------------------------------------------------------- /tests/queue.js: -------------------------------------------------------------------------------- 1 | /* vim: set ts=8 sts=8 sw=8 noet: */ 2 | 3 | var mod_tap = require('tap'); 4 | var mod_vasync = require('..'); 5 | 6 | function 7 | immediate_worker(task, next) 8 | { 9 | setImmediate(function () { 10 | next(); 11 | }); 12 | } 13 | 14 | function 15 | sametick_worker(task, next) 16 | { 17 | next(); 18 | } 19 | 20 | function 21 | random_delay_worker(task, next) 22 | { 23 | setTimeout(function () { 24 | next(); 25 | }, Math.floor(Math.random() * 250)); 26 | } 27 | 28 | mod_tap.test('must not push after close', function (test) { 29 | test.plan(3); 30 | 31 | var q = mod_vasync.queuev({ 32 | worker: immediate_worker, 33 | concurrency: 10 34 | }); 35 | test.ok(q); 36 | 37 | test.doesNotThrow(function () { 38 | q.push({}); 39 | }, 'push should not throw _before_ close()'); 40 | 41 | q.close(); 42 | 43 | /* 44 | * If we attempt to add tasks to the queue _after_ calling close(), 45 | * we should get an exception: 46 | */ 47 | test.throws(function () { 48 | q.push({}); 49 | }, 'push should throw _after_ close()'); 50 | 51 | test.end(); 52 | }); 53 | 54 | mod_tap.test('get \'end\' event with close()', function (test) { 55 | var task_count = 45; 56 | var tasks_finished = 0; 57 | var seen_end = false; 58 | var seen_drain = false; 59 | 60 | test.plan(14 + task_count); 61 | 62 | var q = mod_vasync.queuev({ 63 | worker: random_delay_worker, 64 | concurrency: 5 65 | }); 66 | test.ok(q); 67 | 68 | /* 69 | * Enqueue a bunch of tasks; more than our concurrency: 70 | */ 71 | for (var i = 0; i < 45; i++) { 72 | q.push({}, function () { 73 | tasks_finished++; 74 | test.ok(true); 75 | }); 76 | } 77 | 78 | /* 79 | * Close the queue to signify that we're done now. 80 | */ 81 | test.equal(q.ended, false); 82 | test.equal(q.closed, false); 83 | q.close(); 84 | test.equal(q.closed, true); 85 | test.equal(q.ended, false); 86 | 87 | q.on('drain', function () { 88 | /* 89 | * 'drain' should fire before 'end': 90 | */ 91 | test.notOk(seen_drain); 92 | test.notOk(seen_end); 93 | seen_drain = true; 94 | }); 95 | q.on('end', function () { 96 | /* 97 | * 'end' should fire after 'drain': 98 | */ 99 | test.ok(seen_drain); 100 | test.notOk(seen_end); 101 | seen_end = true; 102 | 103 | /* 104 | * Check the public state: 105 | */ 106 | test.equal(q.closed, true); 107 | test.equal(q.ended, true); 108 | 109 | /* 110 | * We should have fired the callbacks for _all_ enqueued 111 | * tasks by now: 112 | */ 113 | test.equal(task_count, tasks_finished); 114 | test.end(); 115 | }); 116 | 117 | /* 118 | * Check that we see neither the 'drain', nor the 'end' event before 119 | * the end of this tick: 120 | */ 121 | test.notOk(seen_drain); 122 | test.notOk(seen_end); 123 | }); 124 | 125 | mod_tap.test('get \'end\' event with close() and no tasks', function (test) { 126 | var seen_drain = false; 127 | var seen_end = false; 128 | 129 | test.plan(10); 130 | 131 | var q = mod_vasync.queuev({ 132 | worker: immediate_worker, 133 | concurrency: 10 134 | }); 135 | 136 | setImmediate(function () { 137 | test.notOk(seen_end); 138 | }); 139 | 140 | test.equal(q.ended, false); 141 | test.equal(q.closed, false); 142 | q.close(); 143 | test.equal(q.closed, true); 144 | test.equal(q.ended, false); 145 | test.notOk(seen_end); 146 | 147 | q.on('drain', function () { 148 | seen_drain = true; 149 | }); 150 | q.on('end', function () { 151 | /* 152 | * We do not expect to see a 'drain' event, as there were no 153 | * tasks pushed onto the queue before we closed it. 154 | */ 155 | test.notOk(seen_drain); 156 | test.notOk(seen_end); 157 | test.equal(q.closed, true); 158 | test.equal(q.ended, true); 159 | seen_end = true; 160 | test.end(); 161 | }); 162 | }); 163 | 164 | /* 165 | * We want to ensure that both the 'drain' event and the q.drain() hook are 166 | * called the same number of times: 167 | */ 168 | mod_tap.test('equivalence of on(\'drain\') and q.drain()', function (test) { 169 | var enqcount = 4; 170 | var drains = 4; 171 | var ee_count = 0; 172 | var fn_count = 0; 173 | 174 | test.plan(enqcount + drains + 3); 175 | 176 | var q = mod_vasync.queuev({ 177 | worker: immediate_worker, 178 | concurrency: 10 179 | }); 180 | 181 | var enq = function () { 182 | if (--enqcount < 0) 183 | return; 184 | 185 | q.push({}, function () { 186 | test.ok(true, 'task completion'); 187 | }); 188 | }; 189 | 190 | var draino = function () { 191 | test.ok(true, 'drain called'); 192 | if (--drains === 0) { 193 | test.equal(q.closed, false, 'not closed'); 194 | test.equal(q.ended, false, 'not ended'); 195 | test.equal(fn_count, ee_count, 'same number of calls'); 196 | test.end(); 197 | } 198 | }; 199 | 200 | enq(); 201 | enq(); 202 | 203 | q.on('drain', function () { 204 | ee_count++; 205 | enq(); 206 | draino(); 207 | }); 208 | q.drain = function () { 209 | fn_count++; 210 | enq(); 211 | draino(); 212 | }; 213 | }); 214 | 215 | /* 216 | * In the past, we've only handed on the _first_ argument to the task completion 217 | * callback. Make sure we hand on _all_ of the arguments now: 218 | */ 219 | mod_tap.test('ensure all arguments passed to push() callback', function (test) { 220 | test.plan(13); 221 | 222 | var q = mod_vasync.queuev({ 223 | worker: function (task, callback) { 224 | if (task.fail) { 225 | callback(new Error('guru meditation')); 226 | return; 227 | } 228 | callback(null, 1, 2, 3, 5, 8); 229 | }, 230 | concurrency: 1 231 | }); 232 | 233 | q.push({ fail: true }, function (err, a, b, c, d, e) { 234 | test.ok(err, 'got the error'); 235 | test.equal(err.message, 'guru meditation'); 236 | test.type(a, 'undefined'); 237 | test.type(b, 'undefined'); 238 | test.type(c, 'undefined'); 239 | test.type(d, 'undefined'); 240 | test.type(e, 'undefined'); 241 | }); 242 | 243 | q.push({ fail: false }, function (err, a, b, c, d, e) { 244 | test.notOk(err, 'got no error'); 245 | test.equal(a, 1); 246 | test.equal(b, 2); 247 | test.equal(c, 3); 248 | test.equal(d, 5); 249 | test.equal(e, 8); 250 | }); 251 | 252 | q.drain = function () { 253 | test.end(); 254 | }; 255 | }); 256 | 257 | mod_tap.test('queue kill', function (test) { 258 | // Derived from async queue.kill test 259 | var count = 0; 260 | var q = mod_vasync.queuev({ 261 | worker: function (task, callback) { 262 | setImmediate(function () { 263 | test.ok(++count < 2, 264 | 'Function should be called once'); 265 | callback(); 266 | }); 267 | }, 268 | concurrency: 1 269 | }); 270 | q.drain = function () { 271 | test.ok(false, 'Function should never be called'); 272 | }; 273 | 274 | // Queue twice, the first will exec immediately 275 | q.push(0); 276 | q.push(0); 277 | 278 | q.kill(); 279 | 280 | q.on('end', function () { 281 | test.ok(q.killed); 282 | test.end(); 283 | }); 284 | }); 285 | -------------------------------------------------------------------------------- /tests/queue_concurrency.js: -------------------------------------------------------------------------------- 1 | /* vim: set ts=8 sts=8 sw=8 noet: */ 2 | 3 | var mod_tap = require('tap'); 4 | var mod_vasync = require('..'); 5 | 6 | function 7 | latched_worker(task, cb) 8 | { 9 | if (task.immediate) { 10 | cb(); 11 | } else { 12 | task.latched = true; 13 | task.unlatch = function () { 14 | task.latched = false; 15 | cb(); 16 | }; 17 | } 18 | } 19 | 20 | function 21 | unlatchAll(tasks) 22 | { 23 | tasks.forEach(function (t) { 24 | if (t.latched) { 25 | t.unlatch(); 26 | } 27 | }); 28 | } 29 | 30 | function 31 | setAllImmediate(tasks) 32 | { 33 | tasks.forEach(function (t) { 34 | t.immediate = true; 35 | }); 36 | } 37 | 38 | mod_tap.test('test serial tasks', function (test) { 39 | test.plan(2); 40 | 41 | var q = mod_vasync.queuev({ 42 | worker: latched_worker, 43 | concurrency: 1 44 | }); 45 | test.ok(q); 46 | 47 | var tasks = []; 48 | for (var i = 0; i < 2; ++i) { 49 | tasks.push({ 50 | 'id': i, 51 | 'latched': false, 52 | 'immediate': false 53 | }); 54 | } 55 | 56 | setTimeout(function () { 57 | var latched = 0; 58 | tasks.forEach(function (t) { 59 | if (t.latched) { 60 | ++latched; 61 | } 62 | }); 63 | test.ok(latched === 1); 64 | unlatchAll(tasks); 65 | setAllImmediate(tasks); 66 | }, 10); 67 | 68 | q.on('drain', function () { 69 | q.close(); 70 | }); 71 | 72 | q.on('end', function () { 73 | test.end(); 74 | }); 75 | 76 | q.push(tasks); 77 | }); 78 | 79 | mod_tap.test('test parallel tasks', function (test) { 80 | test.plan(2); 81 | 82 | var q = mod_vasync.queuev({ 83 | worker: latched_worker, 84 | concurrency: 2 85 | }); 86 | test.ok(q); 87 | 88 | var tasks = []; 89 | for (var i = 0; i < 3; ++i) { 90 | tasks.push({ 91 | 'id': i, 92 | 'latched': false, 93 | 'immediate': false 94 | }); 95 | } 96 | 97 | setTimeout(function () { 98 | var latched = 0; 99 | tasks.forEach(function (t) { 100 | if (t.latched) { 101 | ++latched; 102 | } 103 | }); 104 | test.ok(latched === 2); 105 | unlatchAll(tasks); 106 | setAllImmediate(tasks); 107 | }, 10); 108 | 109 | q.on('drain', function () { 110 | q.close(); 111 | }); 112 | 113 | q.on('end', function () { 114 | test.end(); 115 | }); 116 | 117 | q.push(tasks); 118 | }); 119 | 120 | mod_tap.test('test ratchet up and down', function (test) { 121 | test.plan(8); 122 | 123 | var q = mod_vasync.queuev({ 124 | worker: latched_worker, 125 | concurrency: 2 126 | }); 127 | test.ok(q); 128 | 129 | var bounced = 0; 130 | var tasks = []; 131 | for (var i = 0; i < 21; ++i) { 132 | tasks.push({ 133 | 'id': i, 134 | 'latched': false, 135 | 'immediate': false 136 | }); 137 | } 138 | 139 | function count() { 140 | var latched = 0; 141 | tasks.forEach(function (t) { 142 | if (t.latched) { 143 | ++latched; 144 | } 145 | }); 146 | return (latched); 147 | } 148 | 149 | function fiveLatch() { 150 | if (!q.closed) { 151 | ++bounced; 152 | test.ok(count() === 5); 153 | q.updateConcurrency(2); 154 | unlatchAll(tasks); 155 | setTimeout(twoLatch, 10); 156 | } 157 | } 158 | 159 | function twoLatch() { 160 | if (!q.closed) { 161 | ++bounced; 162 | test.ok(count() === 2); 163 | q.updateConcurrency(5); 164 | unlatchAll(tasks); 165 | setTimeout(fiveLatch, 10); 166 | } 167 | } 168 | setTimeout(twoLatch, 10); 169 | 170 | q.on('drain', function () { 171 | q.close(); 172 | }); 173 | 174 | q.on('end', function () { 175 | // 21 tasks === 5 * 3 + 2 * 3 === 6 bounces 176 | test.ok(bounced === 6); 177 | test.end(); 178 | }); 179 | 180 | q.push(tasks); 181 | }); 182 | -------------------------------------------------------------------------------- /tests/waterfall.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Tests the "waterfall" primitive. 3 | */ 4 | 5 | var mod_tap = require('tap'); 6 | var mod_vasync = require('..'); 7 | 8 | var count = 0; 9 | var st; 10 | 11 | mod_tap.test('empty waterfall', function (test) { 12 | st = mod_vasync.waterfall([], function (err) { 13 | test.ok(err === null); 14 | test.ok(st.ndone === 0); 15 | test.ok(st.nerrors === 0); 16 | test.ok(st.operations.length === 0); 17 | test.ok(st.successes.length === 0); 18 | test.equal(count, 1); 19 | test.end(); 20 | }); 21 | count++; 22 | test.ok(st.ndone === 0); 23 | test.ok(st.nerrors === 0); 24 | test.ok(st.operations.length === 0); 25 | test.ok(st.successes.length === 0); 26 | }); 27 | 28 | mod_tap.test('normal 4-stage waterfall', function (test) { 29 | count = 0; 30 | st = mod_vasync.waterfall([ 31 | function func1(cb) { 32 | test.ok(count === 0, 'func1: count === 0'); 33 | test.ok(st.ndone === 0); 34 | count++; 35 | setTimeout(cb, 20, null, { 'hello': 'world' }); 36 | }, 37 | function func2(extra, cb) { 38 | test.equal(extra.hello, 'world', 'func2: extra arg'); 39 | test.ok(count == 1, 'func2: count == 1'); 40 | test.ok(st.ndone === 1); 41 | test.ok(st.operations[0].status == 'ok'); 42 | test.ok(st.operations[1].status == 'pending'); 43 | test.ok(st.operations[2].status == 'waiting'); 44 | count++; 45 | setTimeout(cb, 20, null, 5, 6, 7); 46 | }, 47 | function (five, six, seven, cb) { 48 | test.equal(five, 5, 'func3: extra arg'); 49 | test.equal(six, 6, 'func3: extra arg'); 50 | test.equal(seven, 7, 'func3: extra arg'); 51 | test.ok(count == 2, 'func3: count == 2'); 52 | test.ok(st.ndone === 2); 53 | count++; 54 | setTimeout(cb, 20); 55 | }, 56 | function func4(cb) { 57 | test.ok(count == 3, 'func4: count == 2'); 58 | test.ok(st.ndone === 3); 59 | count++; 60 | setTimeout(cb, 20, null, 8, 9); 61 | } 62 | ], function (err, eight, nine) { 63 | test.ok(count == 4, 'final: count == 4'); 64 | test.ok(err === null, 'no error'); 65 | test.ok(eight == 8); 66 | test.ok(nine == 9); 67 | test.ok(st.ndone === 4); 68 | test.ok(st.nerrors === 0); 69 | test.ok(st.operations.length === 4); 70 | test.ok(st.successes.length === 4); 71 | test.ok(st.operations[0].status == 'ok'); 72 | test.ok(st.operations[1].status == 'ok'); 73 | test.ok(st.operations[2].status == 'ok'); 74 | test.ok(st.operations[3].status == 'ok'); 75 | test.end(); 76 | }); 77 | test.ok(st.ndone === 0); 78 | test.ok(st.nerrors === 0); 79 | test.ok(st.operations.length === 4); 80 | test.ok(st.operations[0].funcname == 'func1', 'func1 name'); 81 | test.ok(st.operations[0].status == 'pending'); 82 | test.ok(st.operations[1].funcname == 'func2', 'func2 name'); 83 | test.ok(st.operations[1].status == 'waiting'); 84 | test.ok(st.operations[2].funcname == '(anon)', 'anon name'); 85 | test.ok(st.operations[2].status == 'waiting'); 86 | test.ok(st.operations[3].funcname == 'func4', 'func4 name'); 87 | test.ok(st.operations[3].status == 'waiting'); 88 | test.ok(st.successes.length === 0); 89 | }); 90 | 91 | mod_tap.test('bailing out early', function (test) { 92 | count = 0; 93 | st = mod_vasync.waterfall([ 94 | function func1(cb) { 95 | test.ok(count === 0, 'func1: count === 0'); 96 | count++; 97 | setTimeout(cb, 20); 98 | }, 99 | function func2(cb) { 100 | test.ok(count == 1, 'func2: count == 1'); 101 | count++; 102 | setTimeout(cb, 20, new Error('boom!')); 103 | }, 104 | function func3(cb) { 105 | test.ok(count == 2, 'func3: count == 2'); 106 | count++; 107 | setTimeout(cb, 20); 108 | } 109 | ], function (err) { 110 | test.ok(count == 2, 'final: count == 3'); 111 | test.equal(err.message, 'boom!'); 112 | test.ok(st.ndone == 2); 113 | test.ok(st.nerrors == 1); 114 | test.ok(st.operations[0].status == 'ok'); 115 | test.ok(st.operations[1].status == 'fail'); 116 | test.ok(st.operations[2].status == 'waiting'); 117 | test.ok(st.successes.length == 1); 118 | test.end(); 119 | }); 120 | }); 121 | 122 | mod_tap.test('bad function', function (test) { 123 | count = 0; 124 | st = mod_vasync.waterfall([ 125 | function func1(cb) { 126 | count++; 127 | cb(); 128 | setTimeout(function () { 129 | test.throws( 130 | function () { cb(); process.abort(); }, 131 | 'vasync.waterfall: ' + 132 | 'function 0 ("func1") invoked its ' + 133 | 'callback twice'); 134 | test.equal(count, 2); 135 | test.end(); 136 | }, 100); 137 | }, 138 | function func2(cb) { 139 | count++; 140 | /* do nothing -- we'll throw an exception first */ 141 | } 142 | ], function (err) { 143 | /* not reached */ 144 | console.error('didn\'t expect to finish'); 145 | process.abort(); 146 | }); 147 | }); 148 | 149 | mod_tap.test('badargs', function (test) { 150 | test.throws(function () { mod_vasync.waterfall(); }); 151 | test.throws(function () { mod_vasync.waterfall([], 'foo'); }); 152 | test.throws(function () { mod_vasync.waterfall('foo', 'bar'); }); 153 | test.end(); 154 | }); 155 | 156 | mod_tap.test('normal waterfall, no callback', function (test) { 157 | count = 0; 158 | st = mod_vasync.waterfall([ 159 | function func1(cb) { 160 | test.ok(count === 0); 161 | count++; 162 | setImmediate(cb); 163 | }, 164 | function func2(cb) { 165 | test.ok(count == 1); 166 | count++; 167 | setImmediate(cb); 168 | setTimeout(function () { 169 | test.ok(count == 2); 170 | test.end(); 171 | }, 100); 172 | } 173 | ]); 174 | }); 175 | 176 | mod_tap.test('empty waterfall, no callback', function (test) { 177 | st = mod_vasync.waterfall([]); 178 | setTimeout(function () { test.end(); }, 100); 179 | }); 180 | -------------------------------------------------------------------------------- /tests/whilst.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Tests the "whilst" function 3 | */ 4 | 5 | var mod_util = require('util'); 6 | 7 | var mod_tap = require('tap'); 8 | var mod_vasync = require('..'); 9 | 10 | mod_tap.test('basic whilst', function (test) { 11 | var n = 0; 12 | 13 | mod_vasync.whilst( 14 | function condition() { 15 | return (n < 5); 16 | }, 17 | function body(cb) { 18 | n++; 19 | cb(null, n); 20 | }, 21 | function done(err, arg) { 22 | test.ok(!err, 'error unset'); 23 | test.equal(n, 5, 'n == 5'); 24 | test.equal(n, arg, 'n == arg'); 25 | test.end(); 26 | }); 27 | }); 28 | 29 | mod_tap.test('whilst return object', function (test) { 30 | var n = 0; 31 | 32 | var w = mod_vasync.whilst( 33 | function condition() { 34 | return (n < 5); 35 | }, 36 | function body(cb) { 37 | n++; 38 | 39 | test.equal(n, w.iterations, 'n == w.iterations: ' + n); 40 | 41 | cb(null, n, 'foo'); 42 | }, 43 | function done(err, arg1, arg2, arg3) { 44 | test.ok(!err, 'error unset'); 45 | test.equal(w.iterations, 5, 'whilst had 5 iterations'); 46 | test.equal(w.finished, true, 'whilst has finished'); 47 | test.equal(arg1, n, 'whilst arg1 == n'); 48 | test.equal(arg2, 'foo', 'whilst arg2 == "foo"'); 49 | test.equal(arg3, undefined, 'whilst arg3 == undefined'); 50 | test.end(); 51 | }); 52 | 53 | test.equal(typeof (w), 'object', 'whilst returns an object'); 54 | test.equal(w.finished, false, 'whilst is not finished'); 55 | test.equal(w.iterations, 0, 'whilst has not started yet'); 56 | }); 57 | 58 | mod_tap.test('whilst false condition', function (test) { 59 | mod_vasync.whilst( 60 | function condition() { 61 | return (false); 62 | }, 63 | function body(cb) { 64 | cb(); 65 | }, 66 | function done(err, arg) { 67 | test.ok(!err, 'error is unset'); 68 | test.ok(!arg, 'arg is unset'); 69 | test.end(); 70 | }); 71 | }); 72 | 73 | mod_tap.test('whilst error', function (test) { 74 | var n = 0; 75 | 76 | var w = mod_vasync.whilst( 77 | function condition() { 78 | return (true); 79 | }, 80 | function body(cb) { 81 | n++; 82 | 83 | if (n > 5) { 84 | cb(new Error('n > 5'), 'bar'); 85 | } else { 86 | cb(null, 'foo'); 87 | } 88 | }, 89 | function done(err, arg) { 90 | test.ok(err, 'error is set'); 91 | test.equal(err.message, 'n > 5'); 92 | test.equal(arg, 'bar'); 93 | 94 | test.equal(w.finished, true, 'whilst is finished'); 95 | 96 | /* 97 | * Iterations is bumped after the test condition is run and 98 | * before the iteration function is run. Because the condition 99 | * in this example is inside the iteration function (the test 100 | * condition always returns true), the iteration count will be 101 | * 1 higher than expected, since it will fail when (n > 5), or 102 | * when iterations is 6. 103 | */ 104 | test.equal(w.iterations, 6, 'whilst had 6 iterations'); 105 | 106 | test.end(); 107 | }); 108 | }); 109 | --------------------------------------------------------------------------------