├── async.plugin.zsh ├── .editorconfig ├── LICENSE ├── tools └── publish.sh ├── .github └── workflows │ └── test.yml ├── test.zsh ├── README.md ├── async_test.zsh └── async.zsh /async.plugin.zsh: -------------------------------------------------------------------------------- 1 | 0=${(%):-%N} 2 | source ${0:A:h}/async.zsh 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_style = tab 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | 10 | [*.{yml,json}] 11 | indent_style = space 12 | indent_size = 2 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Mathias Fredriksson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /tools/publish.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | emulate -R zsh 3 | setopt errexit 4 | 5 | 0=${(%):-%N} 6 | 7 | REPO=mafredri/zsh-async 8 | 9 | run() { 10 | dirty="$(git status --porcelain | grep -v '^??')" || true 11 | if [[ -n $dirty ]]; then 12 | print error: uncommitted changes, aborting... 13 | print $dirty 14 | exit 1 15 | fi 16 | 17 | v=$(git tag --list 'v*' --sort=-v:refname | head -n1 | tr -d v) 18 | vv=(${(@s,.,)v}) 19 | case $1 in 20 | major) ((vv[1]++)); vv[2]=0; vv[3]=0;; 21 | minor) ((vv[2]++)); vv[3]=0;; 22 | patch) ((vv[3]++));; 23 | *) print 'error: unknown semver method $1, use patch, minor or major'; exit 1;; 24 | esac 25 | nv=${(j,.,)vv} 26 | 27 | print "Current version is v${v}" 28 | print -n "Publish v${nv} (Y/n): " 29 | read -r -k1 30 | case $REPLY in 31 | [Yy$'\n']) print;; 32 | [Nn]) print; exit 0;; 33 | *) print $'\n'error: bad response $REPLY; exit 1;; 34 | esac 35 | 36 | sed -i '' \ 37 | -e "s/# version: .*/# version: v${nv}/" \ 38 | -e "s/ASYNC_VERSION=.*/ASYNC_VERSION=${nv}/" \ 39 | async.zsh 40 | 41 | git add async.zsh 42 | git commit -m "v${nv}" 43 | git tag "v${nv}" -m "Release v${nv}" 44 | git push --follow-tags 45 | 46 | changes=(${(f)"$(git log --format='* %s %h' v${v}...v${nv}~1)"}) 47 | body=($changes '' https://github.com/${REPO}/compare/v${v}...v${nv}) 48 | 49 | typeset -a params=( 50 | tag=v${nv} 51 | title=v${nv} 52 | body="$(urlencode ${(F)body})" 53 | ) 54 | 55 | open https://github.com/${REPO}/releases/new'?'${(j.&.)params} 56 | } 57 | 58 | # https://stackoverflow.com/questions/28971539/zsh-script-to-encode-full-file-path 59 | urlencode() { 60 | setopt localoptions extendedglob 61 | local input=(${(s::)*}) 62 | print ${(j::)input/(#b)([^A-Za-z0-9_.\!~*\'\(\)-\/])/%${(l:2::0:)$(([##16]#match))}} 63 | } 64 | 65 | if [[ -z $1 ]]; then 66 | 1=patch 67 | fi 68 | 69 | (cd ${0:h}/..; run $1) 70 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | push: 4 | branches: [main] 5 | pull_request: 6 | workflow_dispatch: 7 | 8 | jobs: 9 | macos: 10 | name: macOS 11 | strategy: 12 | matrix: 13 | os: [macos-12, macos-11] 14 | runs-on: ${{matrix.os}} 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v2 18 | 19 | - name: Run tests 20 | timeout-minutes: 1 21 | run: | 22 | zsh --version 23 | script -q <<<'zsh -if ./test.zsh -v; exit $?' 24 | 25 | ubuntu: 26 | name: Ubuntu 27 | strategy: 28 | matrix: 29 | os: [ubuntu-latest] 30 | zsh-version: 31 | - "5.9" 32 | - "5.8.1" 33 | - "5.8" 34 | - "5.7.1" 35 | - "5.7" 36 | - "5.6.2" 37 | - "5.5.1" 38 | - "5.4.2" 39 | # Too old to build/use on GitHub Actions (missing signals). 40 | # - "5.3.1" 41 | # - "5.3" 42 | # - "5.2" 43 | # - "5.1.1" 44 | # - "5.0.8" 45 | # - "5.0.2" 46 | runs-on: ${{matrix.os}} 47 | steps: 48 | - name: Install deps 49 | run: | 50 | sudo apt-get install zsh 51 | 52 | - name: Cache zsh 53 | id: cache-zsh 54 | uses: actions/cache@v2 55 | with: 56 | path: /opt/zsh 57 | key: ${{runner.os}}-${{matrix.zsh-version}} 58 | 59 | - name: Checkout zsh 60 | if: steps.cache-zsh.outputs.cache-hit != 'true' 61 | uses: actions/checkout@v2 62 | with: 63 | repository: zsh-users/zsh 64 | ref: zsh-${{matrix.zsh-version}} 65 | path: zsh-build 66 | 67 | - name: Build zsh 68 | if: steps.cache-zsh.outputs.cache-hit != 'true' 69 | run: | 70 | sudo apt-get install build-essential autoconf yodl libncurses-dev 71 | cd "$GITHUB_WORKSPACE/zsh-build" 72 | aclocal 73 | autoconf 74 | autoheader 75 | script -qec './configure --prefix=/opt/zsh' 76 | make 77 | sudo make install.bin install.modules install.fns 78 | 79 | - name: Checkout code 80 | uses: actions/checkout@v2 81 | 82 | - name: Run tests 83 | timeout-minutes: 1 84 | run: | 85 | export PATH="/opt/zsh/bin:$PATH" 86 | zsh --version 87 | script -qec 'zsh -if ./test.zsh -v' /dev/null 88 | -------------------------------------------------------------------------------- /test.zsh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | # 3 | # zsh-async test runner. 4 | # Checks for test files named *_test.zsh or *_test.sh and runs all functions 5 | # named test_*. 6 | # 7 | emulate -R zsh 8 | 9 | zmodload zsh/datetime 10 | zmodload zsh/parameter 11 | zmodload zsh/zutil 12 | zmodload zsh/system 13 | zmodload zsh/zselect 14 | 15 | TEST_GLOB=. 16 | TEST_RUN= 17 | TEST_VERBOSE=0 18 | TEST_TRACE=1 19 | TEST_CODE_SKIP=100 20 | TEST_CODE_ERROR=101 21 | TEST_CODE_TIMEOUT=102 22 | 23 | show_help() { 24 | print "usage: ./test.zsh [-v] [-x] [-run pattern] [search pattern]" 25 | } 26 | 27 | parse_opts() { 28 | local -a verbose debug trace help run 29 | 30 | local out 31 | zparseopts -E -D \ 32 | v=verbose verbose=verbose -verbose=verbose \ 33 | d=debug debug=debug -debug=debug \ 34 | x=trace trace=trace -trace=trace \ 35 | h=help -help=help \ 36 | \?=help \ 37 | run:=run -run:=run 38 | 39 | (( $? )) || (( $+help[1] )) && show_help && exit 0 40 | 41 | if (( $#@ > 1 )); then 42 | print -- "unknown arguments: $@" 43 | show_help 44 | exit 1 45 | fi 46 | 47 | [[ -n $1 ]] && TEST_GLOB=$1 48 | TEST_VERBOSE=$+verbose[1] 49 | TEST_TRACE=$+trace[1] 50 | ZTEST_DEBUG=$+debug[1] 51 | (( $+run[2] )) && TEST_RUN=$run[2] 52 | } 53 | 54 | t_runner_init() { 55 | emulate -L zsh 56 | 57 | zmodload zsh/parameter 58 | 59 | # _t_runner is the main loop that waits for tests, 60 | # used to abort test execution by exec. 61 | _t_runner() { 62 | local -a _test_defer_funcs 63 | integer _test_errors=0 64 | while read -r; do 65 | eval "$REPLY" 66 | done 67 | } 68 | 69 | _t_log() { 70 | local trace=$1; shift 71 | local -a lines indent 72 | lines=("${(@f)@}") 73 | indent=($'\t\t'${^lines[2,$#lines]}) 74 | print -u7 -lr - $'\t'"$trace: $lines[1]" ${(F)indent} 75 | } 76 | 77 | # t_log is for printing log output, visible in verbose (-v) mode. 78 | t_log() { 79 | local line=$funcfiletrace[1] 80 | [[ ${line%:[0-9]*} = "" ]] && line=ztest:$functrace[1] # Not from a file. 81 | _t_log $line "$*" 82 | } 83 | 84 | # t_skip is for skipping a test. 85 | t_skip() { 86 | _t_log $funcfiletrace[1] "$*" 87 | () { return 100 } 88 | t_done 89 | } 90 | 91 | # t_error logs the error and fails the test without aborting. 92 | t_error() { 93 | (( _test_errors++ )) 94 | _t_log $funcfiletrace[1] "$*" 95 | } 96 | 97 | # t_fatal fails the test and halts execution immediately. 98 | t_fatal() { 99 | _t_log $funcfiletrace[1] "$*" 100 | () { return 101 } 101 | t_done 102 | } 103 | 104 | # t_defer takes a function (and optionally, arguments) 105 | # to be executed after the test has completed. 106 | t_defer() { 107 | _test_defer_funcs+=("$*") 108 | } 109 | 110 | # t_done completes the test execution, called automatically after a test. 111 | # Can also be called manually when the test is done. 112 | t_done() { 113 | local ret=$? w=${1:-1} 114 | (( _test_errors )) && ret=101 115 | 116 | (( w )) && wait # Wait for test children to exit. 117 | for d in $_test_defer_funcs; do 118 | eval "$d" 119 | done 120 | print -n -u8 $ret # Send exit code to ztest. 121 | exec _t_runner # Replace shell, wait for new test. 122 | } 123 | 124 | source $1 # Load the test module. 125 | 126 | # Send available test functions to main process. 127 | print -u7 ${(R)${(okM)functions:#test_*}:#test_main} 128 | 129 | # Run test_main. 130 | if [[ -n $functions[test_main] ]]; then 131 | test_main 132 | fi 133 | 134 | exec _t_runner # Wait for commands. 135 | } 136 | 137 | # run_test_module runs all the tests from a test module (asynchronously). 138 | run_test_module() { 139 | local module=$1 140 | local -a tests 141 | float start module_time 142 | 143 | # Create fd's for communication with test runner. 144 | integer run_pid cmdoutfd cmdinfd outfd infd doneoutfd doneinfd 145 | 146 | coproc cat; exec {cmdoutfd}>&p; exec {cmdinfd}<&p 147 | coproc cat; exec {outfd}>&p; exec {infd}<&p 148 | coproc cat; exec {doneoutfd}>&p; exec {doneinfd}<&p 149 | 150 | # No need to keep coproc (&p) open since we 151 | # have redirected the outputs and inputs. 152 | coproc exit 153 | 154 | # Launch a new interactive zsh test runner. We don't capture stdout 155 | typeset -a run_args 156 | (( TEST_TRACE )) && run_args+=('-x') 157 | zsh -s $run_args <&$cmdinfd 7>&$outfd 8>&$doneoutfd & 158 | run_pid=$! 159 | 160 | # Initialize by sending function body from t_runner_init 161 | # and immediately execute it as an anonymous function. 162 | syswrite -o $cmdoutfd "() { ${functions[t_runner_init]} } $module"$'\n' 163 | sysread -i $infd 164 | tests=(${(@)=REPLY}) 165 | [[ -n $TEST_RUN ]] && tests=(${(M)tests:#*$TEST_RUN*}) 166 | 167 | integer mod_exit=0 168 | float mod_start mod_time 169 | 170 | mod_start=$EPOCHREALTIME # Store the module start time. 171 | 172 | # Run all tests. 173 | local test_out 174 | float test_start test_time 175 | integer text_exit 176 | 177 | for test in $tests; do 178 | (( TEST_VERBOSE )) && print "=== RUN $test" 179 | 180 | test_start=$EPOCHREALTIME # Store the test start time. 181 | 182 | # Start the test. 183 | syswrite -o $cmdoutfd "$test; t_done"$'\n' 184 | 185 | test_out= 186 | test_exit=-1 187 | while (( test_exit == -1 )); do 188 | # Block until there is data to be read. 189 | zselect -r $doneinfd -r $infd 190 | 191 | if [[ $reply[2] = $doneinfd ]]; then 192 | sysread -i $doneinfd 193 | test_exit=$REPLY # Store reply from sysread 194 | # Store the test execution time. 195 | test_time=$(( EPOCHREALTIME - test_start )) 196 | fi 197 | 198 | # Read all output from the test output channel. 199 | while sysread -i $infd -t 0; do 200 | test_out+=$REPLY 201 | unset REPLY 202 | done 203 | done 204 | 205 | case $test_exit in 206 | (0|1) state=PASS;; 207 | (100) state=SKIP;; 208 | (101|102) state=FAIL; mod_exit=1;; 209 | *) state="????";; 210 | esac 211 | 212 | if [[ $state = FAIL ]] || (( TEST_VERBOSE )); then 213 | printf -- "--- $state: $test (%.2fs)\n" $test_time 214 | print -n $test_out 215 | fi 216 | done 217 | 218 | # Store module execution time. 219 | mod_time=$(( EPOCHREALTIME - mod_start )) 220 | 221 | # Perform cleanup. 222 | kill -HUP $run_pid 223 | exec {outfd}>&- 224 | exec {infd}<&- 225 | exec {cmdinfd}>&- 226 | exec {cmdoutfd}<&- 227 | exec {doneinfd}<&- 228 | exec {doneoutfd}>&- 229 | 230 | if (( mod_exit )); then 231 | print "FAIL" 232 | (( TEST_VERBOSE )) && print "exit code $mod_exit" 233 | printf "FAIL\t$module\t%.3fs\n" $mod_time 234 | else 235 | (( TEST_VERBOSE )) && print "PASS" 236 | printf "ok\t$module\t%.3fs\n" $mod_time 237 | fi 238 | 239 | return $mod_exit 240 | } 241 | 242 | cleanup() { 243 | trap '' HUP 244 | kill -HUP -$$ 2>/dev/null 245 | trap - HUP 246 | kill -HUP $$ 2>/dev/null 247 | } 248 | 249 | trap cleanup EXIT INT HUP QUIT TERM USR1 250 | 251 | # Parse command arguments. 252 | parse_opts $@ 253 | 254 | (( ZTEST_DEBUG )) && setopt xtrace 255 | 256 | # Execute tests modules. 257 | failed=0 258 | for tf in ${~TEST_GLOB}/*_test.(zsh|sh); do 259 | run_test_module $tf & 260 | wait $! 261 | (( $? )) && failed=1 262 | done 263 | 264 | trap - EXIT 265 | trap '' HUP 266 | kill -HUP -$$ 2>/dev/null 267 | exit $failed 268 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zsh-async 2 | 3 | ![Test](https://github.com/mafredri/zsh-async/workflows/Test/badge.svg) 4 | 5 | ``` 6 | Because your terminal should be able to perform tasks asynchronously without external tools! 7 | ``` 8 | 9 | ## Intro (TL;DR) 10 | 11 | With `zsh-async` you can run multiple asynchronous jobs, enforce unique jobs (multiple instances of the same job will not run), flush all currently running jobs and create multiple workers (each with their own jobs). For each worker you can register a callback-function through which you will be notified about the job results (job name, return code, output and execution time). 12 | 13 | ## Overview 14 | 15 | `zsh-async` is a small library for running asynchronous tasks in zsh without requiring any external tools. It utilizes `zsh/zpty` to launch a pseudo-terminal in which all commands get executed without blocking any other processes. Checking for completed tasks can be done manually, by polling, or better yet, automatically whenever a process has finished executing by notifying through a `SIGWINCH` kill-signal. 16 | 17 | This library bridges the gap between spawning child processes and disowning them. Child processes launched by normal means clutter the terminal with output about their state, and disowned processes become separate entities, no longer under control of the parent. Now you can have both! 18 | 19 | ## Usage 20 | 21 | The async worker is a separate environment (think web worker). You send it a job (command + parameters) to execute and it returns the result of that execution through a callback function. If you find that you need to stop/start a worker to update global state (variables) you should consider refactoring so that state is passed during the `async_job` call (e.g. `async_job my_worker my_function $state1 $state2`). 22 | 23 | Note that because the worker is a separate forked environment, any functions you want to use as jobs in the worker need to be defined before the worker is started, otherwise you will get a `command not found` error when you try to launch the job. 24 | 25 | ### Installation 26 | 27 | #### Manual 28 | 29 | You can either source the `async.zsh` script directly or insert under your `$fpath` as async and autoload it through `autoload -Uz async && async`. 30 | 31 | #### Integration 32 | 33 | ##### zplug 34 | 35 | ``` 36 | zplug "mafredri/zsh-async", from:"github", use:"async.zsh" 37 | ``` 38 | 39 | ### Functions 40 | 41 | The `zsh-async` library has a bunch of functions that need to be used to perform async actions: 42 | 43 | #### `async_init` 44 | 45 | Initializes the async library (not required if using async from `$fpath` with autoload.) 46 | 47 | #### `async_start_worker [-u] [-n] [-p ]` 48 | 49 | Start a new async worker with optional parameters, a worker can be told to only run unique tasks and to notify a process when tasks are complete. 50 | 51 | * `-u` unique. Only unique job names can run, e.g. the command `git status` will have `git` as the unique job name identifier 52 | 53 | * `-n` notify through `SIGWINCH` signal. Needs to be caught with a `trap '' WINCH` in the process defined by `-p` 54 | 55 | **NOTE:** When `zsh-async` is used in an interactive shell with ZLE enabled this option is not needed. Signaling through `SIGWINCH` has been replaced by a ZLE watcher that is triggered on output from the `zpty` instance (still requires a callback function through `async_register_callback` though). Technically zsh versions prior to `5.2` do not return the file descriptor for zpty instances, however, `zsh-async` attempts to deduce it anyway. 56 | 57 | * `-p` pid to notify (defaults to current pid) 58 | 59 | #### `async_stop_worker []` 60 | 61 | Simply stops a worker and all active jobs will be terminated immediately. 62 | 63 | #### `async_job []` 64 | 65 | Start a new asynchronous job on specified worker, assumes the worker is running. Note if you are using a function for the job, it must have been defined before the worker was started or you will get a `command not found` error. 66 | 67 | #### `async_worker_eval []` 68 | 69 | Evaluate a command (like async_job) inside the async worker, then worker environment can be manipulated. For example, issuing a cd command will change the PWD of the worker which will then be inherited by all future async jobs. 70 | 71 | Output will be returned via callback, job name will be [async/eval]. 72 | 73 | #### `async_process_results ` 74 | 75 | Get results from finished jobs and pass it to the to callback function. This is the only way to reliably return the job name, return code, output and execution time and with minimal effort. 76 | 77 | If the async process buffer becomes corrupt, the callback will be invoked with the first argument being `[async]` (job name), non-zero return code and fifth argument describing the error (stderr). 78 | 79 | The `callback_function` is called with the following parameters: 80 | 81 | * `$1` job name, e.g. the function passed to async_job 82 | * `$2` return code 83 | * Returns `-1` if return code is missing, this should never happen, if it does, you have likely run into a bug. Please open a new [issue](https://github.com/mafredri/zsh-async/issues/new) with a detailed description of what you were doing. 84 | * `$3` resulting (stdout) output from job execution 85 | * `$4` execution time, floating point e.g. 0.0076138973 seconds 86 | * `$5` resulting (stderr) error output from job execution 87 | * `$6` has next result in buffer (0 = buffer empty, 1 = yes) 88 | * This means another async job has completed and is pending in the buffer, it's very likely that your callback function will be called a second time (or more) in this execution. It's generally a good idea to e.g. delay prompt updates (`zle reset-prompt`) until the buffer is empty to prevent strange states in ZLE. 89 | 90 | Possible error return codes for the job name `[async]`: 91 | 92 | * `1` Corrupt worker output. 93 | * `2` ZLE watcher detected an error on the worker fd. 94 | * `3` Response from async_job when worker is missing. 95 | * `130` Async worker crashed, this should not happen but it can mean the file descriptor has become corrupt. This must be followed by a `async_stop_worker [name]` and then the worker and tasks should be restarted. It is unknown why this happens. 96 | 97 | #### `async_register_callback ` 98 | 99 | Register a callback for completed jobs. As soon as a job is finished, `async_process_results` will be called with the specified callback function. This requires that a worker is initialized with the -n (notify) option. 100 | 101 | #### `async_unregister_callback ` 102 | 103 | Unregister the callback for a specific worker. 104 | 105 | #### `async_flush_jobs ` 106 | 107 | Flush all current jobs running on a worker. This will terminate any and all running processes under the worker by sending a `SIGTERM` to the entire process group, use with caution. 108 | 109 | ## Example code 110 | 111 | ```zsh 112 | #!/usr/bin/env zsh 113 | source ./async.zsh 114 | async_init 115 | 116 | # Initialize a new worker (with notify option) 117 | async_start_worker my_worker -n 118 | 119 | # Create a callback function to process results 120 | COMPLETED=0 121 | completed_callback() { 122 | COMPLETED=$(( COMPLETED + 1 )) 123 | print $@ 124 | } 125 | 126 | # Register callback function for the workers completed jobs 127 | async_register_callback my_worker completed_callback 128 | 129 | # Give the worker some tasks to perform 130 | async_job my_worker print hello 131 | async_job my_worker sleep 0.3 132 | 133 | # Wait for the two tasks to be completed 134 | while (( COMPLETED < 2 )); do 135 | print "Waiting..." 136 | sleep 0.1 137 | done 138 | 139 | print "Completed $COMPLETED tasks!" 140 | 141 | # Output: 142 | # Waiting... 143 | # print 0 hello 0.001583099365234375 144 | # Waiting... 145 | # Waiting... 146 | # sleep 0 0.30631208419799805 147 | # Completed 2 tasks! 148 | ``` 149 | 150 | ## Testing 151 | 152 | Tests are located in `*_test.zsh` and can be run by executing the test runner: `./test.zsh`. 153 | 154 | Example: 155 | 156 | ```console 157 | $ ./test.zsh 158 | ok ./async_test.zsh 2.334s 159 | ``` 160 | 161 | The test suite can also run specific tasks that match a pattern, for example: 162 | 163 | ```console 164 | $ ./test.zsh -v -run zle 165 | === RUN test_zle_watcher 166 | --- PASS: test_zle_watcher (0.07s) 167 | PASS 168 | ok ./async_test.zsh 0.070s 169 | ``` 170 | 171 | ## Limitations 172 | 173 | * A NULL-character (`$'\0'`) is used by `async_job` to signify the end of the command, it is recommended not to pass them as arguments, although they should work when passing multiple arguments to `async_job` (because of quoting). 174 | * Tell me? :) 175 | 176 | ## Tips 177 | 178 | If you do not wish to use the `notify` feature, you can couple `zsh-async` with `zsh/sched` or the zsh `periodic` function for scheduling the worker results to be processed. 179 | 180 | ## Why did I make this? 181 | 182 | I found a great theme for zsh, [Pure](https://github.com/sindresorhus/pure) by Sindre Sorhus. After using it for a while I noticed some graphical glitches due to the terminal being updated by a disowned process. Thus, I became inspired to get my hands dirty and find a solution. I tried many things, coprocesses (seemed too limited by themselves), different combinations of trapping kill-signals, etc. I also had problems with the zsh process ending up in a deadlock due to some zsh bug. After working out the kinks, I ended up with this and thought, hey, why not make it a library. 183 | -------------------------------------------------------------------------------- /async_test.zsh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | 3 | test__async_job_print_hi() { 4 | coproc cat 5 | print -n -p t # Insert token into coproc. 6 | 7 | local line 8 | local -a out 9 | line=$(_async_job print hi) 10 | # Remove leading/trailing null, parse, unquote and interpret as array. 11 | line=$line[2,$#line-1] 12 | out=("${(@Q)${(z)line}}") 13 | 14 | coproc exit 15 | 16 | [[ $out[1] = print ]] || t_error "command name should be print, got" $out[1] 17 | [[ $out[2] = 0 ]] || t_error "want exit code 0, got" $out[2] 18 | [[ $out[3] = hi ]] || t_error "want output: hi, got" $out[3] 19 | } 20 | 21 | test__async_job_stderr() { 22 | coproc cat 23 | print -n -p t # Insert token into coproc. 24 | 25 | local line 26 | local -a out 27 | line=$(_async_job print 'hi 1>&2') 28 | # Remove trailing null, parse, unquote and interpret as array. 29 | line=$line[1,$#line-1] 30 | out=("${(@Q)${(z)line}}") 31 | 32 | coproc exit 33 | 34 | [[ $out[2] = 0 ]] || t_error "want status 0, got" $out[2] 35 | [[ -z $out[3] ]] || t_error "want empty output, got" $out[3] 36 | [[ $out[5] = hi ]] || t_error "want stderr: hi, got" $out[5] 37 | } 38 | 39 | test__async_job_wait_for_token() { 40 | float start duration 41 | coproc cat 42 | 43 | _async_job print hi >/dev/null & 44 | job=$! 45 | start=$EPOCHREALTIME 46 | { 47 | sleep 0.1 48 | print -n -p t 49 | } & 50 | 51 | wait $job 52 | 53 | coproc exit 54 | 55 | duration=$(( EPOCHREALTIME - start )) 56 | # Fail if the execution time was faster than 0.1 seconds. 57 | (( duration >= 0.1 )) || t_error "execution was too fast, want >= 0.1, got" $duration 58 | } 59 | 60 | test__async_job_multiple_commands() { 61 | coproc cat 62 | print -n -p t 63 | 64 | local line 65 | local -a out 66 | line="$(_async_job print '-n hi; for i in "1 2" 3 4; do print -n $i; done')" 67 | # Remove trailing null, parse, unquote and interpret as array. 68 | line=$line[1,$#line-1] 69 | out=("${(@Q)${(z)line}}") 70 | 71 | coproc exit 72 | 73 | # $out[1] here will be the entire string passed to _async_job() 74 | # ('print -n hi...') since proper command parsing is done by 75 | # the async worker. 76 | [[ $out[3] = "hi1 234" ]] || t_error "want output hi1 234, got " $out[3] 77 | } 78 | 79 | test_async_start_stop_worker() { 80 | local out 81 | 82 | async_start_worker test 83 | out=$(zpty -L) 84 | [[ $out =~ "test _async_worker" ]] || t_error "want zpty worker running, got ${(Vq-)out}" 85 | 86 | async_stop_worker test || t_error "stop worker: want exit code 0, got $?" 87 | out=$(zpty -L) 88 | [[ -z $out ]] || t_error "want no zpty worker running, got ${(Vq-)out}" 89 | 90 | async_stop_worker nonexistent && t_error "stop non-existent worker: want exit code 1, got $?" 91 | } 92 | 93 | test_async_job_print_matches_input_exactly() { 94 | local -a result 95 | cb() { result=("$@") } 96 | 97 | async_start_worker test 98 | t_defer async_stop_worker test 99 | 100 | want=' 101 | Hello world! 102 | Much *formatting*, 103 | many space\t...\n\n 104 | Such "quote", v '$'\'quote\''' 105 | ' 106 | 107 | async_job test print -r - "$want" 108 | while ! async_process_results test cb; do :; done 109 | 110 | [[ $result[3] = $want ]] || t_error "output, want ${(Vqqqq)want}, got ${(Vqqqq)result[3]}" 111 | } 112 | 113 | test_async_process_results() { 114 | local -a r 115 | cb() { r+=("$@") } 116 | 117 | async_start_worker test 118 | t_defer async_stop_worker test 119 | 120 | async_process_results test cb # No results. 121 | ret=$? 122 | (( ret == 1 )) || t_error "want exit code 1, got $ret" 123 | 124 | async_job test print -n hi 125 | while ! async_process_results test cb; do :; done 126 | (( $#r == 6 )) || t_error "want one result, got $(( $#r % 6 ))" 127 | } 128 | 129 | test_async_process_results_stress() { 130 | # NOTE: This stress test does not always pass properly on older versions of 131 | # zsh, sometimes writing to zpty can hang and other times reading can hang, 132 | # etc. 133 | local -a r 134 | cb() { r+=("$@") } 135 | 136 | async_start_worker test 137 | t_defer async_stop_worker test 138 | 139 | integer iter=20 timeout=5 140 | for i in {1..$iter}; do 141 | async_job test "print -n $i" 142 | done 143 | 144 | float start=$EPOCHSECONDS 145 | 146 | while (( $#r / 6 < iter )); do 147 | async_process_results test cb 148 | (( EPOCHSECONDS - start > timeout )) && { 149 | t_log "timed out after ${timeout}s" 150 | t_fatal "wanted $iter results, got $(( $#r / 6 ))" 151 | } 152 | done 153 | 154 | local -a stdouts 155 | while (( $#r > 0 )); do 156 | [[ $r[1] = print ]] || t_error "want 'print', got ${(Vq-)r[1]}" 157 | [[ $r[2] = 0 ]] || t_error "want exit 0, got $r[2]" 158 | stdouts+=($r[3]) 159 | [[ -z $r[5] ]] || t_error "want no stderr, got ${(Vq-)r[5]}" 160 | shift 6 r 161 | done 162 | 163 | local got want 164 | # Check that we received all numbers. 165 | got=(${(on)stdouts}) 166 | want=({1..$iter}) 167 | [[ $want = $got ]] || t_error "want stdout: ${(Vq-)want}, got ${(Vq-)got}" 168 | 169 | # Test with longer running commands (sleep, then print). 170 | iter=20 171 | for i in {1..$iter}; do 172 | async_job test "sleep 1 && print -n $i" 173 | sleep 0.00001 174 | (( iter % 6 == 0 )) && async_process_results test cb 175 | done 176 | 177 | start=$EPOCHSECONDS 178 | 179 | while (( $#r / 6 < iter )); do 180 | async_process_results test cb 181 | (( EPOCHSECONDS - start > timeout )) && { 182 | t_log "timed out after ${timeout}s" 183 | t_fatal "wanted $iter results, got $(( $#r / 6 ))" 184 | } 185 | done 186 | 187 | stdouts=() 188 | while (( $#r > 0 )); do 189 | [[ $r[1] = sleep ]] || t_error "want 'sleep', got ${(Vq-)r[1]}" 190 | [[ $r[2] = 0 ]] || t_error "want exit 0, got $r[2]" 191 | stdouts+=($r[3]) 192 | [[ -z $r[5] ]] || t_error "want no stderr, got ${(Vq-)r[5]}" 193 | shift 6 r 194 | done 195 | 196 | # Check that we received all numbers. 197 | got=(${(on)stdouts}) 198 | want=({1..$iter}) 199 | [[ $want = $got ]] || t_error "want stdout: ${(Vq-)want}, got ${(Vq-)got}" 200 | } 201 | 202 | test_async_job_multiple_commands_in_multiline_string() { 203 | local -a result 204 | cb() { result=("$@") } 205 | 206 | async_start_worker test 207 | # Test multi-line (single string) command. 208 | async_job test 'print "hi\n 123 "'$'\nprint -n bye' 209 | while ! async_process_results test cb; do :; done 210 | async_stop_worker test 211 | 212 | [[ $result[1] = print ]] || t_error "want command name: print, got" $result[1] 213 | local want=$'hi\n 123 \nbye' 214 | [[ $result[3] = $want ]] || t_error "want output: ${(Vq-)want}, got ${(Vq-)result[3]}" 215 | } 216 | 217 | test_async_job_git_status() { 218 | local -a result 219 | cb() { result=("$@") } 220 | 221 | async_start_worker test 222 | async_job test git status --porcelain 223 | while ! async_process_results test cb; do :; done 224 | async_stop_worker test 225 | 226 | [[ $result[1] = git ]] || t_error "want command name: git, got" $result[1] 227 | [[ $result[2] = 0 ]] || t_error "want exit code: 0, got" $result[2] 228 | 229 | want=$(git status --porcelain) 230 | got=$result[3] 231 | [[ $got = $want ]] || t_error "want ${(Vq-)want}, got ${(Vq-)got}" 232 | } 233 | 234 | test_async_job_multiple_arguments_and_spaces() { 235 | local -a result 236 | cb() { result=("$@") } 237 | 238 | async_start_worker test 239 | async_job test print "hello world" 240 | while ! async_process_results test cb; do :; done 241 | async_stop_worker test 242 | 243 | [[ $result[1] = print ]] || t_error "want command name: print, got" $result[1] 244 | [[ $result[2] = 0 ]] || t_error "want exit code: 0, got" $result[2] 245 | 246 | [[ $result[3] = "hello world" ]] || { 247 | t_error "want output: \"hello world\", got" ${(Vq-)result[3]} 248 | } 249 | } 250 | 251 | test_async_job_unique_worker() { 252 | local -a result 253 | cb() { 254 | # Add to result so we can detect if it was called multiple times. 255 | result+=("$@") 256 | } 257 | helper() { 258 | sleep 0.1; print $1 259 | } 260 | 261 | # Start a unique (job) worker. 262 | async_start_worker test -u 263 | 264 | # Launch two jobs with the same name, the first one should be 265 | # allowed to complete whereas the second one is never run. 266 | async_job test helper one 267 | async_job test helper two 268 | 269 | while ! async_process_results test cb; do :; done 270 | 271 | # If both jobs were running but only one was complete, 272 | # async_process_results() could've returned true for 273 | # the first job, wait a little extra to make sure the 274 | # other didn't run. 275 | sleep 0.1 276 | async_process_results test cb 277 | 278 | async_stop_worker test 279 | 280 | # Ensure that cb was only called once with correc output. 281 | [[ ${#result} = 6 ]] || t_error "result: want 6 elements, got" ${#result} 282 | [[ $result[3] = one ]] || t_error "output: want 'one', got" ${(Vq-)result[3]} 283 | } 284 | 285 | test_async_job_error_and_nonzero_exit() { 286 | local -a r 287 | cb() { r+=("$@") } 288 | error() { 289 | print "Errors!" 290 | 12345 291 | 54321 292 | print "Done!" 293 | exit 99 294 | } 295 | 296 | async_start_worker test 297 | async_job test error 298 | 299 | while ! async_process_results test cb; do :; done 300 | 301 | [[ $r[1] = error ]] || t_error "want 'error', got ${(Vq-)r[1]}" 302 | [[ $r[2] = 99 ]] || t_error "want exit code 99, got $r[2]" 303 | 304 | want=$'Errors!\nDone!' 305 | [[ $r[3] = $want ]] || t_error "want ${(Vq-)want}, got ${(Vq-)r[3]}" 306 | 307 | want=$'.*command not found: 12345\n.*command not found: 54321' 308 | [[ $r[5] =~ $want ]] || t_error "want ${(Vq-)want}, got ${(Vq-)r[5]}" 309 | } 310 | 311 | test_async_worker_notify_sigwinch() { 312 | local -a result 313 | cb() { result=("$@") } 314 | 315 | if ! is-at-least 5.0.3 && [[ -n $CI ]]; then 316 | t_skip "Skip winch test on GitHub Actions for zsh 5.0.2: undefined signal: WINCH" 317 | fi 318 | 319 | ASYNC_USE_ZLE_HANDLER=0 320 | 321 | async_start_worker test -n 322 | async_register_callback test cb 323 | 324 | async_job test 'sleep 0.1; print hi' 325 | 326 | while (( ! $#result )); do sleep 0.01; done 327 | 328 | async_stop_worker test 329 | 330 | [[ $result[3] = hi ]] || t_error "expected output: hi, got" $result[3] 331 | } 332 | 333 | test_async_job_keeps_nulls() { 334 | local -a r 335 | cb() { r=("$@") } 336 | null_echo() { 337 | print Hello$'\0' with$'\0' nulls! 338 | print "Did we catch them all?"$'\0' 339 | print $'\0'"What about the errors?"$'\0' 1>&2 340 | } 341 | 342 | async_start_worker test 343 | async_job test null_echo 344 | 345 | while ! async_process_results test cb; do :; done 346 | 347 | async_stop_worker test 348 | 349 | local want 350 | want=$'Hello\0 with\0 nulls!\nDid we catch them all?\0' 351 | [[ $r[3] = $want ]] || t_error stdout: want ${(Vq-)want}, got ${(Vq-)r[3]} 352 | want=$'\0What about the errors?\0' 353 | [[ $r[5] = $want ]] || t_error stderr: want ${(Vq-)want}, got ${(Vq-)r[5]} 354 | } 355 | 356 | test_async_flush_jobs() { 357 | local -a r 358 | cb() { r=+("$@") } 359 | 360 | print_four() { print -n 4 } 361 | print_123_delayed_exit() { 362 | print -n 1 363 | { sleep 0.25 && print -n 2 } &! 364 | { sleep 0.3 && print -n 3 } &! 365 | } 366 | 367 | async_start_worker test 368 | 369 | # Start a job that prints 1 and starts two disowned child processes that 370 | # print 2 and 3, respectively, after a timeout. The job will not exit 371 | # immediately (and thus print 1) because the child processes are still 372 | # running. 373 | async_job test print_123_delayed_exit 374 | 375 | # Check that the job is waiting for the child processes. 376 | sleep 0.05 377 | async_process_results test cb 378 | (( $#r == 0 )) || t_error "want no output, got ${(Vq-)r}" 379 | 380 | # Start a job that prints four, it will produce 381 | # output but we will not process it. 382 | async_job test print_four 383 | sleep 0.2 384 | 385 | # Flush jobs, this kills running jobs and discards unprocessed results. 386 | # TODO: Confirm that they no longer exist in the process tree. 387 | local output 388 | output="${(Q)$(ASYNC_DEBUG=1 async_flush_jobs test)}" 389 | # NOTE(mafredri): First 'p' in print_four is lost when null-prefixing 390 | # _async_job output. 391 | [[ $output = *'rint_four 0 4'* ]] || { 392 | t_error "want discarded output 'rint_four 0 4' when ASYNC_DEBUG=1, got ${(Vq-)output}" 393 | } 394 | 395 | # Check that the killed job did not produce output. 396 | sleep 0.1 397 | async_process_results test cb 398 | (( $#r == 0 )) || t_error "want no output, got ${(Vq-)r}" 399 | 400 | async_stop_worker test 401 | } 402 | 403 | test_async_worker_survives_termination_of_other_worker() { 404 | local -a result 405 | cb() { result+=("$@") } 406 | 407 | async_start_worker test1 408 | t_defer async_stop_worker test1 409 | 410 | # Start and stop a worker, will send SIGHUP to previous worker 411 | # (probably has to do with some shell inheritance). 412 | async_start_worker test2 413 | async_stop_worker test2 414 | 415 | async_job test1 print hi 416 | 417 | integer start=$EPOCHREALTIME 418 | while (( EPOCHREALTIME - start < 2.0 )); do 419 | async_process_results test1 cb && break 420 | done 421 | 422 | (( $#result == 6 )) || t_error "wanted a result, got (${(@Vq)result})" 423 | } 424 | 425 | test_async_worker_update_pwd() { 426 | local -a result 427 | local eval_out 428 | cb() { 429 | if [[ $1 == '[async/eval]' ]]; then 430 | eval_out="$3" 431 | else 432 | result+=("$3") 433 | fi 434 | } 435 | 436 | async_start_worker test1 437 | t_defer async_stop_worker test1 438 | 439 | async_job test1 'print $PWD' 440 | async_worker_eval test1 'print -n foo; cd ..; print -n bar; print -n -u2 baz' 441 | async_job test1 'print $PWD' 442 | 443 | start=$EPOCHREALTIME 444 | while (( EPOCHREALTIME - start < 2.0 && $#result < 2 )); do 445 | async_process_results test1 cb 446 | done 447 | 448 | (( $#result == 2 )) || t_error "wanted 2 results, got ${#result}" 449 | [[ $eval_out = foobarbaz ]] || t_error "wanted async_worker_eval to output foobarbaz, got ${(q)eval_out}" 450 | [[ -n $result[2] ]] || t_error "wanted second pwd to be non-empty" 451 | [[ $result[1] != $result[2] ]] || t_error "wanted worker to change pwd, was ${(q)result[1]}, got ${(q)result[2]}" 452 | } 453 | 454 | test_async_worker_update_pwd_and_env() { 455 | local -a result 456 | local eval_out 457 | cb() { 458 | if [[ $1 == '[async/eval]' ]]; then 459 | eval_out="$3" 460 | else 461 | result+=("$3") 462 | fi 463 | } 464 | 465 | input=$'my\ninput' 466 | 467 | async_start_worker test1 468 | t_defer async_stop_worker test1 469 | 470 | async_job test1 "print -n $myenv" 471 | async_worker_eval test1 "cd ..; export myenv=${(q)input}" 472 | async_job test1 'print -n $myenv' 473 | 474 | start=$EPOCHREALTIME 475 | while (( EPOCHREALTIME - start < 2.0 && $#result < 2 )); do 476 | async_process_results test1 cb 477 | done 478 | 479 | (( $#result == 2 )) || t_error "wanted 2 results, got ${#result}" 480 | [[ $result[2] = $input ]] || t_error "wanted second print to output ${(q-)input}, got ${(q-)result[2]}" 481 | [[ $result[1] != $result[2] ]] || t_error "wanted worker to change env, was ${(q-)result[1]}, got ${(q-)result[2]}" 482 | } 483 | 484 | setopt_helper() { 485 | setopt localoptions $1 486 | 487 | # Make sure to test with multiple options 488 | local -a result 489 | cb() { result=("$@") } 490 | 491 | async_start_worker test 492 | async_job test print "hello world" 493 | while ! async_process_results test cb; do :; done 494 | async_stop_worker test 495 | 496 | # At this point, ksh arrays will only mess with the test. 497 | setopt noksharrays 498 | 499 | [[ $result[1] = print ]] || t_fatal "$1 want command name: print, got" $result[1] 500 | [[ $result[2] = 0 ]] || t_fatal "$1 want exit code: 0, got" $result[2] 501 | 502 | [[ $result[3] = "hello world" ]] || { 503 | t_fatal "$1 want output: \"hello world\", got" ${(Vq-)result[3]} 504 | } 505 | } 506 | 507 | test_all_options() { 508 | local -a opts exclude 509 | 510 | if [[ $ZSH_VERSION == 5.0.? ]]; then 511 | t_skip "Test is not reliable on zsh 5.0.X" 512 | fi 513 | 514 | # Make sure worker is stopped, even if tests fail. 515 | t_defer async_stop_worker test 516 | 517 | { sleep 15 && t_fatal "timed out" } & 518 | local tpid=$! 519 | 520 | opts=(${(k)options}) 521 | 522 | # These options can't be tested. 523 | exclude=( 524 | zle interactive restricted shinstdin stdin onecmd singlecommand 525 | warnnestedvar errreturn 526 | ) 527 | 528 | for opt in ${opts:|exclude}; do 529 | if [[ $options[$opt] = on ]]; then 530 | setopt_helper no$opt 531 | else 532 | setopt_helper $opt 533 | fi 534 | done 2>/dev/null # Remove redirect to see output. 535 | 536 | kill $tpid # Stop timeout. 537 | } 538 | 539 | test_async_job_with_rc_expand_param() { 540 | setopt localoptions rcexpandparam 541 | 542 | # Make sure to test with multiple options 543 | local -a result 544 | cb() { result=("$@") } 545 | 546 | async_start_worker test 547 | async_job test print "hello world" 548 | while ! async_process_results test cb; do :; done 549 | async_stop_worker test 550 | 551 | [[ $result[1] = print ]] || t_error "want command name: print, got" $result[1] 552 | [[ $result[2] = 0 ]] || t_error "want exit code: 0, got" $result[2] 553 | 554 | [[ $result[3] = "hello world" ]] || { 555 | t_error "want output: \"hello world\", got" ${(Vq-)result[3]} 556 | } 557 | } 558 | 559 | zpty_init() { 560 | zmodload zsh/zpty 561 | 562 | export PS1="" 563 | zpty zsh 'zsh -f +Z' 564 | zpty -r zsh zpty_init1 "**" || { 565 | t_log "initial prompt missing" 566 | return 1 567 | } 568 | 569 | zpty -w zsh "{ $@ }" 570 | zpty -r -m zsh zpty_init2 "**" || { 571 | t_log "prompt missing" 572 | return 1 573 | } 574 | 575 | local junk 576 | if zpty -r -t zsh junk '*'; then 577 | while zpty -r -t zsh junk '*'; do 578 | # Noop. 579 | done 580 | fi 581 | } 582 | 583 | zpty_run() { 584 | zpty -w zsh "$*" 585 | zpty -r -m zsh zpty_run "**" || { 586 | t_log "prompt missing after ${(Vq-)*}" 587 | return 1 588 | } 589 | } 590 | 591 | zpty_deinit() { 592 | zpty -d zsh 593 | } 594 | 595 | test_zle_watcher() { 596 | t_skip "Test is not reliable on zsh 5.0.X" 597 | 598 | setopt localoptions 599 | zpty_init ' 600 | emulate -R zsh 601 | setopt zle 602 | stty 38400 columns 80 rows 24 tabs -icanon -iexten 603 | TERM=vt100 604 | 605 | . "'$PWD'/async.zsh" 606 | async_init 607 | 608 | print_result_cb() { print ${(Vq-)@} } 609 | async_start_worker test 610 | async_register_callback test print_result_cb 611 | ' || { 612 | zpty_deinit 613 | t_fatal "failed to init zpty" 614 | } 615 | 616 | t_defer zpty_deinit # Deinit after test completion. 617 | 618 | zpty -w zsh "zle -F" 619 | zpty -r -m zsh result "*_async_zle_watcher*" || { 620 | t_fatal "want _async_zle_watcher to be registered as zle watcher, got output ${(Vq-)result}" 621 | } 622 | 623 | zpty_run async_job test 'print hello world' || t_fatal "could not send async_job command" 624 | 625 | zpty -r -m zsh result "*print 0 'hello world'*" || { 626 | t_fatal "want \"print 0 'hello world'\", got output ${(Vq-)result}" 627 | } 628 | } 629 | 630 | test_main() { 631 | # Load zsh-async before running each test. 632 | zmodload zsh/datetime 633 | . ./async.zsh 634 | async_init 635 | } 636 | -------------------------------------------------------------------------------- /async.zsh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env zsh 2 | 3 | # 4 | # zsh-async 5 | # 6 | # version: v1.8.6 7 | # author: Mathias Fredriksson 8 | # url: https://github.com/mafredri/zsh-async 9 | # 10 | 11 | typeset -g ASYNC_VERSION=1.8.6 12 | # Produce debug output from zsh-async when set to 1. 13 | typeset -g ASYNC_DEBUG=${ASYNC_DEBUG:-0} 14 | 15 | # Execute commands that can manipulate the environment inside the async worker. Return output via callback. 16 | _async_eval() { 17 | local ASYNC_JOB_NAME 18 | # Rename job to _async_eval and redirect all eval output to cat running 19 | # in _async_job. Here, stdout and stderr are not separated for 20 | # simplicity, this could be improved in the future. 21 | { 22 | eval "$@" 23 | } &> >(ASYNC_JOB_NAME=[async/eval] _async_job 'command -p cat') 24 | } 25 | 26 | # Wrapper for jobs executed by the async worker, gives output in parseable format with execution time 27 | _async_job() { 28 | # Disable xtrace as it would mangle the output. 29 | setopt localoptions noxtrace 30 | 31 | # Store start time for job. 32 | float -F duration=$EPOCHREALTIME 33 | 34 | # Run the command and capture both stdout (`eval`) and stderr (`cat`) in 35 | # separate subshells. When the command is complete, we grab write lock 36 | # (mutex token) and output everything except stderr inside the command 37 | # block, after the command block has completed, the stdin for `cat` is 38 | # closed, causing stderr to be appended with a $'\0' at the end to mark the 39 | # end of output from this job. 40 | local jobname=${ASYNC_JOB_NAME:-$1} out 41 | out="$( 42 | local stdout stderr ret tok 43 | { 44 | stdout=$(eval "$@") 45 | ret=$? 46 | duration=$(( EPOCHREALTIME - duration )) # Calculate duration. 47 | 48 | print -r -n - $'\0'${(q)jobname} $ret ${(q)stdout} $duration 49 | } 2> >(stderr=$(command -p cat) && print -r -n - " "${(q)stderr}$'\0') 50 | )" 51 | if [[ $out != $'\0'*$'\0' ]]; then 52 | # Corrupted output (aborted job?), skipping. 53 | return 54 | fi 55 | 56 | # Grab mutex lock, stalls until token is available. 57 | read -r -k 1 -p tok || return 1 58 | 59 | # Return output ( ). 60 | print -r -n - "$out" 61 | 62 | # Unlock mutex by inserting a token. 63 | print -n -p $tok 64 | } 65 | 66 | # The background worker manages all tasks and runs them without interfering with other processes 67 | _async_worker() { 68 | # Reset all options to defaults inside async worker. 69 | emulate -R zsh 70 | 71 | # Make sure monitor is unset to avoid printing the 72 | # pids of child processes. 73 | unsetopt monitor 74 | 75 | # Redirect stderr to `/dev/null` in case unforseen errors produced by the 76 | # worker. For example: `fork failed: resource temporarily unavailable`. 77 | # Some older versions of zsh might also print malloc errors (know to happen 78 | # on at least zsh 5.0.2 and 5.0.8) likely due to kill signals. 79 | exec 2>/dev/null 80 | 81 | # When a zpty is deleted (using -d) all the zpty instances created before 82 | # the one being deleted receive a SIGHUP, unless we catch it, the async 83 | # worker would simply exit (stop working) even though visible in the list 84 | # of zpty's (zpty -L). This has been fixed around the time of Zsh 5.4 85 | # (not released). 86 | if ! is-at-least 5.4.1; then 87 | TRAPHUP() { 88 | return 0 # Return 0, indicating signal was handled. 89 | } 90 | fi 91 | 92 | local -A storage 93 | local unique=0 94 | local notify_parent=0 95 | local parent_pid=0 96 | local coproc_pid=0 97 | local processing=0 98 | 99 | local -a zsh_hooks zsh_hook_functions 100 | zsh_hooks=(chpwd periodic precmd preexec zshexit zshaddhistory) 101 | zsh_hook_functions=(${^zsh_hooks}_functions) 102 | unfunction $zsh_hooks &>/dev/null # Deactivate all zsh hooks inside the worker. 103 | unset $zsh_hook_functions # And hooks with registered functions. 104 | unset zsh_hooks zsh_hook_functions # Cleanup. 105 | 106 | close_idle_coproc() { 107 | local -a pids 108 | pids=(${${(v)jobstates##*:*:}%\=*}) 109 | 110 | # If coproc (cat) is the only child running, we close it to avoid 111 | # leaving it running indefinitely and cluttering the process tree. 112 | if (( ! processing )) && [[ $#pids = 1 ]] && [[ $coproc_pid = $pids[1] ]]; then 113 | coproc : 114 | coproc_pid=0 115 | fi 116 | } 117 | 118 | child_exit() { 119 | close_idle_coproc 120 | 121 | # On older version of zsh (pre 5.2) we notify the parent through a 122 | # SIGWINCH signal because `zpty` did not return a file descriptor (fd) 123 | # prior to that. 124 | if (( notify_parent )); then 125 | # We use SIGWINCH for compatibility with older versions of zsh 126 | # (pre 5.1.1) where other signals (INFO, ALRM, USR1, etc.) could 127 | # cause a deadlock in the shell under certain circumstances. 128 | kill -WINCH $parent_pid 129 | fi 130 | } 131 | 132 | # Register a SIGCHLD trap to handle the completion of child processes. 133 | trap child_exit CHLD 134 | 135 | # Process option parameters passed to worker. 136 | while getopts "np:uz" opt; do 137 | case $opt in 138 | n) notify_parent=1;; 139 | p) parent_pid=$OPTARG;; 140 | u) unique=1;; 141 | z) notify_parent=0;; # Uses ZLE watcher instead. 142 | esac 143 | done 144 | 145 | # Terminate all running jobs, note that this function does not 146 | # reinstall the child trap. 147 | terminate_jobs() { 148 | trap - CHLD # Ignore child exits during kill. 149 | coproc : # Quit coproc. 150 | coproc_pid=0 # Reset pid. 151 | 152 | if is-at-least 5.4.1; then 153 | trap '' HUP # Catch the HUP sent to this process. 154 | kill -HUP -$$ # Send to entire process group. 155 | trap - HUP # Disable HUP trap. 156 | else 157 | # We already handle HUP for Zsh < 5.4.1. 158 | kill -HUP -$$ # Send to entire process group. 159 | fi 160 | } 161 | 162 | killjobs() { 163 | local tok 164 | local -a pids 165 | pids=(${${(v)jobstates##*:*:}%\=*}) 166 | 167 | # No need to send SIGHUP if no jobs are running. 168 | (( $#pids == 0 )) && continue 169 | (( $#pids == 1 )) && [[ $coproc_pid = $pids[1] ]] && continue 170 | 171 | # Grab lock to prevent half-written output in case a child 172 | # process is in the middle of writing to stdin during kill. 173 | (( coproc_pid )) && read -r -k 1 -p tok 174 | 175 | terminate_jobs 176 | trap child_exit CHLD # Reinstall child trap. 177 | } 178 | 179 | local request do_eval=0 180 | local -a cmd 181 | while :; do 182 | # Wait for jobs sent by async_job. 183 | read -r -d $'\0' request || { 184 | # Unknown error occurred while reading from stdin, the zpty 185 | # worker is likely in a broken state, so we shut down. 186 | terminate_jobs 187 | 188 | # Stdin is broken and in case this was an unintended 189 | # crash, we try to report it as a last hurrah. 190 | print -r -n $'\0'"'[async]'" $(( 127 + 3 )) "''" 0 "'$0:$LINENO: zpty fd died, exiting'"$'\0' 191 | 192 | # We use `return` to abort here because using `exit` may 193 | # result in an infinite loop that never exits and, as a 194 | # result, high CPU utilization. 195 | return $(( 127 + 1 )) 196 | } 197 | 198 | # We need to clean the input here because sometimes when a zpty 199 | # has died and been respawned, messages will be prefixed with a 200 | # carraige return (\r, or \C-M). 201 | request=${request#$'\C-M'} 202 | 203 | # Check for non-job commands sent to worker 204 | case $request in 205 | _killjobs) killjobs; continue;; 206 | _async_eval*) do_eval=1;; 207 | esac 208 | 209 | # Parse the request using shell parsing (z) to allow commands 210 | # to be parsed from single strings and multi-args alike. 211 | cmd=("${(z)request}") 212 | 213 | # Name of the job (first argument). 214 | local job=$cmd[1] 215 | 216 | # Check if a worker should perform unique jobs, unless 217 | # this is an eval since they run synchronously. 218 | if (( !do_eval )) && (( unique )); then 219 | # Check if a previous job is still running, if yes, 220 | # skip this job and let the previous one finish. 221 | for pid in ${${(v)jobstates##*:*:}%\=*}; do 222 | if [[ ${storage[$job]} == $pid ]]; then 223 | continue 2 224 | fi 225 | done 226 | fi 227 | 228 | # Guard against closing coproc from trap before command has started. 229 | processing=1 230 | 231 | # Because we close the coproc after the last job has completed, we must 232 | # recreate it when there are no other jobs running. 233 | if (( ! coproc_pid )); then 234 | # Use coproc as a mutex for synchronized output between children. 235 | coproc command -p cat 236 | coproc_pid="$!" 237 | # Insert token into coproc 238 | print -n -p "t" 239 | fi 240 | 241 | if (( do_eval )); then 242 | shift cmd # Strip _async_eval from cmd. 243 | _async_eval $cmd 244 | else 245 | # Run job in background, completed jobs are printed to stdout. 246 | _async_job $cmd & 247 | # Store pid because zsh job manager is extremely unflexible (show jobname as non-unique '$job')... 248 | storage[$job]="$!" 249 | fi 250 | 251 | processing=0 # Disable guard. 252 | 253 | if (( do_eval )); then 254 | do_eval=0 255 | 256 | # When there are no active jobs we can't rely on the CHLD trap to 257 | # manage the coproc lifetime. 258 | close_idle_coproc 259 | fi 260 | done 261 | } 262 | 263 | # 264 | # Get results from finished jobs and pass it to the to callback function. This is the only way to reliably return the 265 | # job name, return code, output and execution time and with minimal effort. 266 | # 267 | # If the async process buffer becomes corrupt, the callback will be invoked with the first argument being `[async]` (job 268 | # name), non-zero return code and fifth argument describing the error (stderr). 269 | # 270 | # usage: 271 | # async_process_results 272 | # 273 | # callback_function is called with the following parameters: 274 | # $1 = job name, e.g. the function passed to async_job 275 | # $2 = return code 276 | # $3 = resulting stdout from execution 277 | # $4 = execution time, floating point e.g. 2.05 seconds 278 | # $5 = resulting stderr from execution 279 | # $6 = has next result in buffer (0 = buffer empty, 1 = yes) 280 | # 281 | async_process_results() { 282 | setopt localoptions unset noshwordsplit noksharrays noposixidentifiers noposixstrings 283 | 284 | local worker=$1 285 | local callback=$2 286 | local caller=$3 287 | local -a items 288 | local null=$'\0' data 289 | integer -l len pos num_processed has_next 290 | 291 | typeset -gA ASYNC_PROCESS_BUFFER 292 | 293 | # Read output from zpty and parse it if available. 294 | while zpty -r -t $worker data 2>/dev/null; do 295 | ASYNC_PROCESS_BUFFER[$worker]+=$data 296 | len=${#ASYNC_PROCESS_BUFFER[$worker]} 297 | pos=${ASYNC_PROCESS_BUFFER[$worker][(i)$null]} # Get index of NULL-character (delimiter). 298 | 299 | # Keep going until we find a NULL-character. 300 | if (( ! len )) || (( pos > len )); then 301 | continue 302 | fi 303 | 304 | while (( pos <= len )); do 305 | # Take the content from the beginning, until the NULL-character and 306 | # perform shell parsing (z) and unquoting (Q) as an array (@). 307 | items=("${(@Q)${(z)ASYNC_PROCESS_BUFFER[$worker][1,$pos-1]}}") 308 | 309 | # Remove the extracted items from the buffer. 310 | ASYNC_PROCESS_BUFFER[$worker]=${ASYNC_PROCESS_BUFFER[$worker][$pos+1,$len]} 311 | 312 | len=${#ASYNC_PROCESS_BUFFER[$worker]} 313 | if (( len > 1 )); then 314 | pos=${ASYNC_PROCESS_BUFFER[$worker][(i)$null]} # Get index of NULL-character (delimiter). 315 | fi 316 | 317 | has_next=$(( len != 0 )) 318 | if (( $#items == 5 )); then 319 | items+=($has_next) 320 | $callback "${(@)items}" # Send all parsed items to the callback. 321 | (( num_processed++ )) 322 | elif [[ -z $items ]]; then 323 | # Empty items occur between results due to double-null ($'\0\0') 324 | # caused by commands being both pre and suffixed with null. 325 | else 326 | # In case of corrupt data, invoke callback with *async* as job 327 | # name, non-zero exit status and an error message on stderr. 328 | $callback "[async]" 1 "" 0 "$0:$LINENO: error: bad format, got ${#items} items (${(q)items})" $has_next 329 | fi 330 | done 331 | done 332 | 333 | (( num_processed )) && return 0 334 | 335 | # Avoid printing exit value when `setopt printexitvalue` is active.` 336 | [[ $caller = trap || $caller = watcher ]] && return 0 337 | 338 | # No results were processed 339 | return 1 340 | } 341 | 342 | # Watch worker for output 343 | _async_zle_watcher() { 344 | setopt localoptions noshwordsplit 345 | typeset -gA ASYNC_PTYS ASYNC_CALLBACKS 346 | local worker=$ASYNC_PTYS[$1] 347 | local callback=$ASYNC_CALLBACKS[$worker] 348 | 349 | if [[ -n $2 ]]; then 350 | # from man zshzle(1): 351 | # `hup' for a disconnect, `nval' for a closed or otherwise 352 | # invalid descriptor, or `err' for any other condition. 353 | # Systems that support only the `select' system call always use 354 | # `err'. 355 | 356 | # this has the side effect to unregister the broken file descriptor 357 | async_stop_worker $worker 358 | 359 | if [[ -n $callback ]]; then 360 | $callback '[async]' 2 "" 0 "$0:$LINENO: error: fd for $worker failed: zle -F $1 returned error $2" 0 361 | fi 362 | return 363 | fi; 364 | 365 | if [[ -n $callback ]]; then 366 | async_process_results $worker $callback watcher 367 | fi 368 | } 369 | 370 | _async_send_job() { 371 | setopt localoptions noshwordsplit noksharrays noposixidentifiers noposixstrings 372 | 373 | local caller=$1 374 | local worker=$2 375 | shift 2 376 | 377 | zpty -t $worker &>/dev/null || { 378 | typeset -gA ASYNC_CALLBACKS 379 | local callback=$ASYNC_CALLBACKS[$worker] 380 | 381 | if [[ -n $callback ]]; then 382 | $callback '[async]' 3 "" 0 "$0:$LINENO: error: no such worker: $worker" 0 383 | else 384 | print -u2 "$caller: no such async worker: $worker" 385 | fi 386 | return 1 387 | } 388 | 389 | zpty -w $worker "$@"$'\0' 390 | } 391 | 392 | # 393 | # Start a new asynchronous job on specified worker, assumes the worker is running. 394 | # 395 | # Note if you are using a function for the job, it must have been defined before the worker was 396 | # started or you will get a `command not found` error. 397 | # 398 | # usage: 399 | # async_job [] 400 | # 401 | async_job() { 402 | setopt localoptions noshwordsplit noksharrays noposixidentifiers noposixstrings 403 | 404 | local worker=$1; shift 405 | 406 | local -a cmd 407 | cmd=("$@") 408 | if (( $#cmd > 1 )); then 409 | cmd=(${(q)cmd}) # Quote special characters in multi argument commands. 410 | fi 411 | 412 | _async_send_job $0 $worker "$cmd" 413 | } 414 | 415 | # 416 | # Evaluate a command (like async_job) inside the async worker, then worker environment can be manipulated. For example, 417 | # issuing a cd command will change the PWD of the worker which will then be inherited by all future async jobs. 418 | # 419 | # Output will be returned via callback, job name will be [async/eval]. 420 | # 421 | # usage: 422 | # async_worker_eval [] 423 | # 424 | async_worker_eval() { 425 | setopt localoptions noshwordsplit noksharrays noposixidentifiers noposixstrings 426 | 427 | local worker=$1; shift 428 | 429 | local -a cmd 430 | cmd=("$@") 431 | if (( $#cmd > 1 )); then 432 | cmd=(${(q)cmd}) # Quote special characters in multi argument commands. 433 | fi 434 | 435 | # Quote the cmd in case RC_EXPAND_PARAM is set. 436 | _async_send_job $0 $worker "_async_eval $cmd" 437 | } 438 | 439 | # This function traps notification signals and calls all registered callbacks 440 | _async_notify_trap() { 441 | setopt localoptions noshwordsplit 442 | 443 | local k 444 | for k in ${(k)ASYNC_CALLBACKS}; do 445 | async_process_results $k ${ASYNC_CALLBACKS[$k]} trap 446 | done 447 | } 448 | 449 | # 450 | # Register a callback for completed jobs. As soon as a job is finnished, async_process_results will be called with the 451 | # specified callback function. This requires that a worker is initialized with the -n (notify) option. 452 | # 453 | # usage: 454 | # async_register_callback 455 | # 456 | async_register_callback() { 457 | setopt localoptions noshwordsplit nolocaltraps 458 | 459 | typeset -gA ASYNC_PTYS ASYNC_CALLBACKS 460 | local worker=$1; shift 461 | 462 | ASYNC_CALLBACKS[$worker]="$*" 463 | 464 | # Enable trap when the ZLE watcher is unavailable, allows 465 | # workers to notify (via -n) when a job is done. 466 | if [[ ! -o interactive ]] || [[ ! -o zle ]]; then 467 | trap '_async_notify_trap' WINCH 468 | elif [[ -o interactive ]] && [[ -o zle ]]; then 469 | local fd w 470 | for fd w in ${(@kv)ASYNC_PTYS}; do 471 | if [[ $w == $worker ]]; then 472 | zle -F $fd _async_zle_watcher # Register the ZLE handler. 473 | break 474 | fi 475 | done 476 | fi 477 | } 478 | 479 | # 480 | # Unregister the callback for a specific worker. 481 | # 482 | # usage: 483 | # async_unregister_callback 484 | # 485 | async_unregister_callback() { 486 | typeset -gA ASYNC_CALLBACKS 487 | 488 | unset "ASYNC_CALLBACKS[$1]" 489 | } 490 | 491 | # 492 | # Flush all current jobs running on a worker. This will terminate any and all running processes under the worker, use 493 | # with caution. 494 | # 495 | # usage: 496 | # async_flush_jobs 497 | # 498 | async_flush_jobs() { 499 | setopt localoptions noshwordsplit 500 | 501 | local worker=$1; shift 502 | 503 | # Check if the worker exists 504 | zpty -t $worker &>/dev/null || return 1 505 | 506 | # Send kill command to worker 507 | async_job $worker "_killjobs" 508 | 509 | # Clear the zpty buffer. 510 | local junk 511 | if zpty -r -t $worker junk '*'; then 512 | (( ASYNC_DEBUG )) && print -n "async_flush_jobs $worker: ${(V)junk}" 513 | while zpty -r -t $worker junk '*'; do 514 | (( ASYNC_DEBUG )) && print -n "${(V)junk}" 515 | done 516 | (( ASYNC_DEBUG )) && print 517 | fi 518 | 519 | # Finally, clear the process buffer in case of partially parsed responses. 520 | typeset -gA ASYNC_PROCESS_BUFFER 521 | unset "ASYNC_PROCESS_BUFFER[$worker]" 522 | } 523 | 524 | # 525 | # Start a new async worker with optional parameters, a worker can be told to only run unique tasks and to notify a 526 | # process when tasks are complete. 527 | # 528 | # usage: 529 | # async_start_worker [-u] [-n] [-p ] 530 | # 531 | # opts: 532 | # -u unique (only unique job names can run) 533 | # -n notify through SIGWINCH signal 534 | # -p pid to notify (defaults to current pid) 535 | # 536 | async_start_worker() { 537 | setopt localoptions noshwordsplit noclobber 538 | 539 | local worker=$1; shift 540 | local -a args 541 | args=("$@") 542 | zpty -t $worker &>/dev/null && return 543 | 544 | typeset -gA ASYNC_PTYS 545 | typeset -h REPLY 546 | typeset has_xtrace=0 547 | 548 | if [[ -o interactive ]] && [[ -o zle ]]; then 549 | # Inform the worker to ignore the notify flag and that we're 550 | # using a ZLE watcher instead. 551 | args+=(-z) 552 | 553 | if (( ! ASYNC_ZPTY_RETURNS_FD )); then 554 | # When zpty doesn't return a file descriptor (on older versions of zsh) 555 | # we try to guess it anyway. 556 | integer -l zptyfd 557 | exec {zptyfd}>&1 # Open a new file descriptor (above 10). 558 | exec {zptyfd}>&- # Close it so it's free to be used by zpty. 559 | fi 560 | fi 561 | 562 | # Workaround for stderr in the main shell sometimes (incorrectly) being 563 | # reassigned to /dev/null by the reassignment done inside the async 564 | # worker. 565 | # See https://github.com/mafredri/zsh-async/issues/35. 566 | integer errfd=-1 567 | 568 | # Redirect of errfd is broken on zsh 5.0.2. 569 | if is-at-least 5.0.8; then 570 | exec {errfd}>&2 571 | fi 572 | 573 | # Make sure async worker is started without xtrace 574 | # (the trace output interferes with the worker). 575 | [[ -o xtrace ]] && { 576 | has_xtrace=1 577 | unsetopt xtrace 578 | } 579 | 580 | if (( errfd != -1 )); then 581 | zpty -b $worker _async_worker -p $$ $args 2>&$errfd 582 | else 583 | zpty -b $worker _async_worker -p $$ $args 584 | fi 585 | local ret=$? 586 | 587 | # Re-enable it if it was enabled, for debugging. 588 | (( has_xtrace )) && setopt xtrace 589 | (( errfd != -1 )) && exec {errfd}>& - 590 | 591 | if (( ret )); then 592 | async_stop_worker $worker 593 | return 1 594 | fi 595 | 596 | if ! is-at-least 5.0.8; then 597 | # For ZSH versions older than 5.0.8 we delay a bit to give 598 | # time for the worker to start before issuing commands, 599 | # otherwise it will not be ready to receive them. 600 | sleep 0.001 601 | fi 602 | 603 | if [[ -o interactive ]] && [[ -o zle ]]; then 604 | if (( ! ASYNC_ZPTY_RETURNS_FD )); then 605 | REPLY=$zptyfd # Use the guessed value for the file desciptor. 606 | fi 607 | 608 | ASYNC_PTYS[$REPLY]=$worker # Map the file desciptor to the worker. 609 | fi 610 | } 611 | 612 | # 613 | # Stop one or multiple workers that are running, all unfetched and incomplete work will be lost. 614 | # 615 | # usage: 616 | # async_stop_worker [] 617 | # 618 | async_stop_worker() { 619 | setopt localoptions noshwordsplit 620 | 621 | local ret=0 worker k v 622 | for worker in $@; do 623 | # Find and unregister the zle handler for the worker 624 | for k v in ${(@kv)ASYNC_PTYS}; do 625 | if [[ $v == $worker ]]; then 626 | zle -F $k 627 | unset "ASYNC_PTYS[$k]" 628 | fi 629 | done 630 | async_unregister_callback $worker 631 | zpty -d $worker 2>/dev/null || ret=$? 632 | 633 | # Clear any partial buffers. 634 | typeset -gA ASYNC_PROCESS_BUFFER 635 | unset "ASYNC_PROCESS_BUFFER[$worker]" 636 | done 637 | 638 | return $ret 639 | } 640 | 641 | # 642 | # Initialize the required modules for zsh-async. To be called before using the zsh-async library. 643 | # 644 | # usage: 645 | # async_init 646 | # 647 | async_init() { 648 | (( ASYNC_INIT_DONE )) && return 649 | typeset -g ASYNC_INIT_DONE=1 650 | 651 | zmodload zsh/zpty 652 | zmodload zsh/datetime 653 | 654 | # Load is-at-least for reliable version check. 655 | autoload -Uz is-at-least 656 | 657 | # Check if zsh/zpty returns a file descriptor or not, 658 | # shell must also be interactive with zle enabled. 659 | typeset -g ASYNC_ZPTY_RETURNS_FD=0 660 | [[ -o interactive ]] && [[ -o zle ]] && { 661 | typeset -h REPLY 662 | zpty _async_test : 663 | (( REPLY )) && ASYNC_ZPTY_RETURNS_FD=1 664 | zpty -d _async_test 665 | } 666 | } 667 | 668 | async() { 669 | async_init 670 | } 671 | 672 | async "$@" 673 | --------------------------------------------------------------------------------