├── .github
    ├── action
    │   ├── Dockerfile
    │   └── action.yml
    └── workflows
    │   └── main.yml
├── .gitignore
├── README.md
├── expect-run
├── expect-warmup
├── frameworks
    ├── antibody-static.zsh
    ├── antigen.zsh
    ├── ohmyzsh.zsh
    ├── prezto.zsh
    ├── vanilla.zsh
    ├── zcomet.zsh
    ├── zgen.zsh
    ├── zim.zsh
    ├── zinit-light.zsh
    ├── zinit-turbo.zsh
    ├── zplug.zsh
    └── zsh4humans.zsh
├── plot-graph
└── run.zsh


/.github/action/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM alpine:latest
2 | RUN apk add -U --no-cache zsh coreutils curl expect git ncurses perl gnuplot
3 | # Change default shell from ash to zsh
4 | RUN sed -i -e "s/bin\/ash/bin\/zsh/" /etc/passwd
5 | ENV TERM=xterm-256color
6 | ENTRYPOINT ["zsh", "-c"]
7 | 


--------------------------------------------------------------------------------
/.github/action/action.yml:
--------------------------------------------------------------------------------
 1 | name: 'Zsh Command'
 2 | inputs:
 3 |   cmd:
 4 |     required: true
 5 |     default: 'exit'
 6 | runs:
 7 |   using: 'docker'
 8 |   image: 'Dockerfile'
 9 |   args:
10 |     - ${{ inputs.cmd }}
11 | 


--------------------------------------------------------------------------------
/.github/workflows/main.yml:
--------------------------------------------------------------------------------
 1 | name: CI
 2 | on:
 3 |   push:
 4 |     branches:
 5 |     - master
 6 |   pull_request:
 7 |     branches:
 8 |     - master
 9 |   schedule:
10 |   - cron: 0 0 * * 0
11 | jobs:
12 |   run:
13 |     runs-on: ubuntu-latest
14 |     timeout-minutes: 30
15 |     steps:
16 |     - uses: actions/checkout@v4
17 |       with:
18 |         path: main
19 |     - name: Run benchmarks
20 |       uses: ./main/.github/action
21 |       with:
22 |         cmd: |
23 |           cd main && \
24 |           source run.zsh && \
25 |           gnuplot -e "zsh_version = 'zsh $ZSH_VERSION'" -c plot-graph
26 |     - uses: actions/checkout@v4
27 |       if: ${{ github.ref == 'refs/heads/master' }}
28 |       with:
29 |         repository: zimfw/zimfw.github.io
30 |         token: ${{ secrets.PAT }}
31 |         path: artifacts
32 |     - name: Push images/results.svg
33 |       if: ${{ github.ref == 'refs/heads/master' }}
34 |       working-directory: artifacts
35 |       run: |
36 |         cp ../main/results.svg images/results.svg && \
37 |         git config user.name github-actions && \
38 |         git config user.email github-actions@github.com && \
39 |         git add images/results.svg && \
40 |         git commit -m "Update images/results.svg" && \
41 |         git push
42 |     - uses: actions/upload-artifact@v4
43 |       if: success() || failure()
44 |       with:
45 |         path: |
46 |           main/results/*.log
47 |           main/results/results.csv
48 |           main/results.svg
49 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /results
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Zsh Framework Benchmark
 2 | =======================
 3 | 
 4 | This is a small utility to benchmark various Zsh frameworks. All of the frameworks are built with the instructions provided by the project README.md's.
 5 | 
 6 | To run, simply clone the repo and run `source run.zsh`.
 7 | 
 8 | The options are:
 9 | ```
10 | source run.zsh <options>
11 | Options:
12 |   -h              Show this help
13 |   -k              Keep the frameworks after the tests are complete (default: delete)
14 |   -p <path>       Set the path to where the frameworks should be installed (default: results)
15 |   -w <path>       Set the working directory (default: temp directory)
16 |   -n <number>     Set the number of iterations to run for each framework (default: 100)
17 |   -f <framework>  Specify framework to benchmark (default: all; can specify more than once)
18 | ```
19 | 
20 | Requirements
21 | ------------
22 | 
23 | The [expect](https://core.tcl-lang.org/expect/index) command.
24 | 
25 | Benchmarks
26 | ----------
27 | 
28 | See the [Zim wiki 'Speed' page](https://github.com/zimfw/zimfw/wiki/Speed) for the latest benchmarks.
29 | 


--------------------------------------------------------------------------------
/expect-run:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/expect -f
 2 | 
 3 | set wd [file tail [pwd]]
 4 | set c0 [clock microseconds]
 5 | spawn -noecho zsh -li
 6 | expect_before {
 7 |   -re {\e](?:[0127]|1337);.*?(?:\a|\e\\)} { exp_continue } ; # match escape sequences to be ignored
 8 | }
 9 | expect {
10 |   $wd { send_error "[expr [clock microseconds] - $c0]\n" } ; # match working dir at prompt
11 |   timeout { exit 1 }
12 | }
13 | sleep 1
14 | send "exit\r"
15 | 


--------------------------------------------------------------------------------
/expect-warmup:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/expect -f
 2 | 
 3 | set timeout 120 ; # warmup can take longer
 4 | set wd [file tail [pwd]]
 5 | spawn -noecho zsh -li
 6 | expect_before {
 7 |   -re {\e](?:[0127]|1337);.*?(?:\a|\e\\)} { exp_continue } ; # match escape sequences to be ignored
 8 | }
 9 | expect {
10 |   $wd { send ":\r" } ; # match working dir at prompt
11 |   timeout { exit 1 }
12 | }
13 | expect {
14 |   $wd { } ; # match working dir at prompt
15 |   timeout { exit 1 }
16 | }
17 | sleep 1
18 | send "exit\r"
19 | 


--------------------------------------------------------------------------------
/frameworks/antibody-static.zsh:
--------------------------------------------------------------------------------
 1 | # install antibody
 2 | [[ ! -e bin/antibody ]] && command curl -sL git.io/antibody | sh -s - -b bin
 3 | 
 4 | >>! .zsh_plugins.txt <<\END
 5 | zimfw/environment
 6 | zimfw/git
 7 | zimfw/input
 8 | zimfw/termtitle
 9 | zimfw/utility
10 | zimfw/duration-info
11 | zimfw/git-info
12 | zimfw/asciiship
13 | zsh-users/zsh-completions
14 | zimfw/completion
15 | zsh-users/zsh-autosuggestions
16 | zsh-users/zsh-syntax-highlighting
17 | zsh-users/zsh-history-substring-search
18 | END
19 | 
20 | HOME=${PWD} ANTIBODY_HOME=${PWD}/.antibody bin/antibody bundle < .zsh_plugins.txt > .zsh_plugins.sh
21 | 
22 | >>! .zshrc <<\END
23 | export ANTIBODY_HOME=~/.antibody
24 | 
25 | # antibody does not add functions subdirs to fpath, nor autoloads them!
26 | setopt EXTENDED_GLOB
27 | fpath+=(${ANTIBODY_HOME}/*zimfw*/functions(NF))
28 | autoload -Uz -- ${ANTIBODY_HOME}/*zimfw*/functions/^(*~|*.zwc(|.old)|_*|prompt_*_setup)(N-.:t)
29 | 
30 | source ~/.zsh_plugins.sh
31 | bindkey "^[[A" history-substring-search-up
32 | bindkey "^[[B" history-substring-search-down
33 | END
34 | 


--------------------------------------------------------------------------------
/frameworks/antigen.zsh:
--------------------------------------------------------------------------------
 1 | # download the script
 2 | command curl -Ss -L git.io/antigen > antigen.zsh
 3 | 
 4 | # add modules to .zshrc
 5 | >>! .zshrc <<\END
 6 | source ~/antigen.zsh
 7 | antigen bundle zimfw/environment
 8 | antigen bundle zimfw/git
 9 | antigen bundle zimfw/input
10 | antigen bundle zimfw/termtitle
11 | antigen bundle zimfw/utility
12 | antigen bundle zimfw/duration-info
13 | antigen bundle zimfw/git-info
14 | antigen theme zimfw/asciiship
15 | antigen bundle zsh-users/zsh-completions
16 | antigen bundle zsh-users/zsh-autosuggestions
17 | antigen bundle zsh-users/zsh-syntax-highlighting
18 | antigen bundle zsh-users/zsh-history-substring-search
19 | 
20 | # antigen adds functions to fpath but does not autoload them!
21 | setopt EXTENDED_GLOB
22 | autoload -Uz -- ~/.antigen/bundles/zimfw/*/functions/^(*~|*.zwc(|.old)|_*|prompt_*_setup)(N-.:t)
23 | 
24 | antigen apply
25 | bindkey "^[[A" history-substring-search-up
26 | bindkey "^[[B" history-substring-search-down
27 | END
28 | 
29 | # Force reinstall, as it was failing in alpine linux
30 | HOME=${PWD} zsh -ic 'antigen reset'
31 | 


--------------------------------------------------------------------------------
/frameworks/ohmyzsh.zsh:
--------------------------------------------------------------------------------
 1 | # download the install script
 2 | command curl -sS -fsSL https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh > install.sh
 3 | # also remove the automatic-start of the new terminal
 4 | # silence the git clone output
 5 | command sed -i.bak -e '/env zsh -l/d' -e 's/git clone/git clone --quiet/g' install.sh
 6 | 
 7 | # run though sh as per the instructions
 8 | HOME=${PWD} ZSH=${PWD}/.oh-my-zsh sh install.sh --unattended
 9 | 
10 | # grab zsh-syntax-highlighting
11 | command git clone --quiet https://github.com/zsh-users/zsh-autosuggestions .oh-my-zsh/custom/plugins/zsh-autosuggestions
12 | 
13 | # grab zsh-syntax-highlighting
14 | command git clone --quiet https://github.com/zsh-users/zsh-syntax-highlighting.git .oh-my-zsh/custom/plugins/zsh-syntax-highlighting
15 | 
16 | # we don't need auto-update stuff
17 | # replace the plugin string with the selected plugins
18 | command sed -i.bak -E -e 's/^# (DISABLE_AUTO_UPDATE="true")/\1/' -e 's/^(plugins=\([^\)]*)/\1 git-prompt colored-man-pages common-aliases timer zsh-autosuggestions zsh-syntax-highlighting history-substring-search/' .zshrc
19 | 


--------------------------------------------------------------------------------
/frameworks/prezto.zsh:
--------------------------------------------------------------------------------
 1 | # clone the repository
 2 | command git clone --quiet --recursive https://github.com/sorin-ionescu/prezto.git .zprezto
 3 | 
 4 | # follow the install instructions
 5 | setopt LOCAL_OPTIONS EXTENDED_GLOB
 6 | local rcfile
 7 | for rcfile in .zprezto/runcoms/^README.md(.N); do
 8 |   command ln -s "${rcfile}" ".${rcfile:t}"
 9 | done
10 | 
11 | # add the modules to the .zpreztorc file
12 | command rm -f .zpreztorc
13 | command cp .zprezto/runcoms/zpreztorc .zpreztorc
14 | command sed -i.bak -E -e "/^ *'spectrum' \\\\/d" -e "s/^( *'prompt')/\\1 'git' 'autosuggestions' 'syntax-highlighting' 'history-substring-search'/" .zpreztorc
15 | 
16 | # start login shell
17 | HOME=${PWD} zsh -lc 'exit'
18 | 


--------------------------------------------------------------------------------
/frameworks/vanilla.zsh:
--------------------------------------------------------------------------------
1 | # just set PS1
2 | >>! .zshrc <<\END
3 | PS1="%~%# "
4 | END
5 | 


--------------------------------------------------------------------------------
/frameworks/zcomet.zsh:
--------------------------------------------------------------------------------
 1 | # clone the repository
 2 | git clone https://github.com/agkozak/zcomet.git .zcomet/bin
 3 | 
 4 | # add modules to .zshrc
 5 | >! .zshrc <<\END
 6 | source ~/.zcomet/bin/zcomet.zsh
 7 | zcomet load zimfw/environment
 8 | zcomet load zimfw/git
 9 | zcomet load zimfw/input
10 | zcomet load zimfw/termtitle
11 | zcomet load zimfw/utility
12 | zcomet load zimfw/duration-info
13 | zcomet load zimfw/git-info
14 | zcomet load zimfw/asciiship
15 | zcomet fpath zsh-users/zsh-completions src
16 | zcomet load zsh-users/zsh-autosuggestions
17 | zcomet load zsh-users/zsh-syntax-highlighting
18 | zcomet load zsh-users/zsh-history-substring-search
19 | 
20 | [[ ${TERM} != dumb ]] && zcomet compinit
21 | 
22 | bindkey "^[[A" history-substring-search-up
23 | bindkey "^[[B" history-substring-search-down
24 | END
25 | 


--------------------------------------------------------------------------------
/frameworks/zgen.zsh:
--------------------------------------------------------------------------------
 1 | # clone the repository
 2 | command git clone --quiet https://github.com/tarjoilija/zgen.git .zgen
 3 | 
 4 | # add modules to .zshrc
 5 | >>! .zshrc <<\END
 6 | ZGEN_AUTOLOAD_COMPINIT=0
 7 | 
 8 | # zgen does not add functions subdirs to fpath, nor autoloads them!
 9 | setopt EXTENDED_GLOB
10 | fpath+=(~/.zgen/zimfw/*/functions(NF))
11 | autoload -Uz -- ~/.zgen/zimfw/*/functions/^(*~|*.zwc(|.old)|_*|prompt_*_setup)(N-.:t)
12 | 
13 | source ~/.zgen/zgen.zsh
14 | if ! zgen saved; then
15 |   zgen load zimfw/environment
16 |   zgen load zimfw/git
17 |   zgen load zimfw/input
18 |   zgen load zimfw/termtitle
19 |   zgen load zimfw/utility
20 |   zgen load zimfw/duration-info
21 |   zgen load zimfw/git-info
22 |   zgen load zimfw/asciiship asciiship
23 |   zgen load zsh-users/zsh-completions
24 |   zgen load zimfw/completion
25 |   zgen load zsh-users/zsh-autosuggestions
26 |   zgen load zsh-users/zsh-syntax-highlighting
27 |   zgen load zsh-users/zsh-history-substring-search
28 |   zgen save
29 | fi
30 | bindkey "^[[A" history-substring-search-up
31 | bindkey "^[[B" history-substring-search-down
32 | END
33 | 


--------------------------------------------------------------------------------
/frameworks/zim.zsh:
--------------------------------------------------------------------------------
1 | # download the install script
2 | command curl -fsSL https://raw.githubusercontent.com/zimfw/install/master/install.zsh | HOME=${PWD} zsh
3 | 


--------------------------------------------------------------------------------
/frameworks/zinit-light.zsh:
--------------------------------------------------------------------------------
 1 | # clone the repository
 2 | command git clone --quiet https://github.com/zdharma-continuum/zinit.git .zinit/bin
 3 | 
 4 | # add modules to .zshrc
 5 | >>! .zshrc <<\END
 6 | PS1="%~%# "
 7 | source ~/.zinit/bin/zinit.zsh
 8 | zinit light "zimfw/environment"
 9 | zinit ice autoload"git-alias-lookup;git-branch-current;git-branch-delete-interactive;git-branch-remote-tracking;git-dir;git-ignore-add;git-root;git-stash-clear-interactive;git-stash-recover;git-submodule-move;git-submodule-remove"
10 | zinit light "zimfw/git"
11 | zinit light "zimfw/input"
12 | zinit light "zimfw/termtitle"
13 | zinit ice autoload"mkcd;mkpw"
14 | zinit light "zimfw/utility"
15 | zinit ice autoload"duration-info-precmd;duration-info-preexec"
16 | zinit light "zimfw/duration-info"
17 | zinit ice autoload"coalesce;git-action;git-info" nocompile
18 | zinit light "zimfw/git-info"
19 | zinit light "zimfw/asciiship"
20 | zinit ice as"completion"
21 | zinit light "zsh-users/zsh-completions"
22 | zinit light "zsh-users/zsh-autosuggestions"
23 | zinit light "zsh-users/zsh-syntax-highlighting"
24 | zinit light "zsh-users/zsh-history-substring-search"
25 | 
26 | autoload -Uz compinit
27 | compinit
28 | 
29 | zinit cdreplay -q
30 | 
31 | bindkey "^[[A" history-substring-search-up
32 | bindkey "^[[B" history-substring-search-down
33 | END
34 | 
35 | # compile the plugins
36 | HOME=${PWD} zsh -ic 'zinit compile --all; exit'
37 | 


--------------------------------------------------------------------------------
/frameworks/zinit-turbo.zsh:
--------------------------------------------------------------------------------
 1 | # clone the repository
 2 | command git clone --quiet https://github.com/zdharma-continuum/zinit.git .zinit/bin
 3 | 
 4 | # add modules to .zshrc
 5 | >>! .zshrc <<\END
 6 | source ~/.zinit/bin/zinit.zsh
 7 | zinit wait lucid for "zimfw/environment"
 8 | zinit wait lucid autoload"git-alias-lookup;git-branch-current;git-branch-delete-interactive;git-branch-remote-tracking;git-dir;git-ignore-add;git-root;git-stash-clear-interactive;git-stash-recover;git-submodule-move;git-submodule-remove" for "zimfw/git"
 9 | zinit wait lucid for "zimfw/input"
10 | zinit wait lucid for "zimfw/termtitle"
11 | zinit wait lucid autoload"mkcd;mkpw" for "zimfw/utility"
12 | zinit wait lucid autoload"duration-info-precmd;duration-info-preexec" for "zimfw/duration-info"
13 | zinit wait lucid autoload"coalesce;git-action;git-info" for "zimfw/git-info"
14 | zinit wait lucid for "zimfw/asciiship"
15 | zinit wait lucid as"completion" for "zsh-users/zsh-completions"
16 | zinit wait"0a" lucid for "zsh-users/zsh-autosuggestions"
17 | zinit wait"0b" lucid for "zsh-users/zsh-syntax-highlighting"
18 | zinit wait"0c" lucid atload"zicompinit;zicdreplay;bindkey \"^[[A\" history-substring-search-up;bindkey \"^[[B\" history-substring-search-down" for "zsh-users/zsh-history-substring-search"
19 | END
20 | 
21 | # compile the plugins
22 | HOME=${PWD} zsh -ic 'zinit compile --all; exit'
23 | 


--------------------------------------------------------------------------------
/frameworks/zplug.zsh:
--------------------------------------------------------------------------------
 1 | # clone the repository
 2 | command git clone --quiet https://github.com/zplug/zplug .zplug
 3 | 
 4 | # add modules to .zshrc
 5 | >>! .zshrc <<\END
 6 | source ~/.zplug/init.zsh
 7 | zplug "zimfw/environment"
 8 | zplug "zimfw/git"
 9 | zplug "zimfw/input"
10 | zplug "zimfw/termtitle"
11 | zplug "zimfw/utility"
12 | zplug "zimfw/duration-info", lazy:yes
13 | zplug "zimfw/git-info", lazy:yes
14 | zplug "zimfw/asciiship", as:theme
15 | zplug "zsh-users/zsh-completions"
16 | zplug "zsh-users/zsh-autosuggestions", defer:2
17 | zplug "zsh-users/zsh-syntax-highlighting", defer:3
18 | zplug "zsh-users/zsh-history-substring-search", defer:3
19 | zplug load
20 | if zplug check zsh-users/zsh-history-substring-search; then
21 |   bindkey "^[[A" history-substring-search-up
22 |   bindkey "^[[B" history-substring-search-down
23 | fi
24 | END
25 | 
26 | # install the plugins
27 | HOME=${PWD} zsh -ic 'zplug install; exit'
28 | 


--------------------------------------------------------------------------------
/frameworks/zsh4humans.zsh:
--------------------------------------------------------------------------------
 1 | emulate -L zsh -o err_return -o no_unset -o no_aliases
 2 | # download zsh4humans v5
 3 | local v=5
 4 | curl -fsSLO https://github.com/romkatv/zsh4humans/archive/refs/heads/v${v}.tar.gz
 5 | # copy over .zshenv and .zshrc
 6 | tar -xzf v${v}.tar.gz
 7 | cp zsh4humans-${v}/{.zshenv,.zshrc} ./
 8 | # create a config for powerlevel10k
 9 | >.p10k.zsh <<<$'POWERLEVEL9K_MODE=ascii\nPOWERLEVEL9K_DISABLE_CONFIGURATION_WIZARD=true'
10 | # disable automatic start of tmux
11 | sed -i.bak -E '1i\
12 | zstyle :z4h: start-tmux no
13 | ' .zshrc
14 | # install and load zimfw/git
15 | sed -i.bak -E 's|^(z4h install [^|]*)|\1 zimfw/git|' .zshrc
16 | sed -i.bak -E 's|^(z4h load [^|]*)|\1 $Z4H/zimfw/git|' .zshrc
17 | rm -r v${v}.tar.gz zsh4humans-${v} .zshrc.bak
18 | # don't ask to change shell
19 | mkdir -p .cache/zsh4humans/v5/stickycache
20 | touch .cache/zsh4humans/v5/stickycache/no-chsh
21 | # initialize zsh4humans
22 | HOME=${PWD} zsh -ic 'exit' </dev/null
23 | 


--------------------------------------------------------------------------------
/plot-graph:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/gnuplot -c
 2 | 
 3 | # Output W3C Scalable Vector Graphics
 4 | set terminal svg
 5 | set output "results.svg"
 6 | 
 7 | # Read comma-delimited data from file
 8 | set datafile separator comma
 9 | 
10 | # Set graph title
11 | set label center at graph 0.5, char 1 "last updated on ".strftime("%b %d, %Y at %H:%M:%S UTC", time(0))." using ".zsh_version
12 | 
13 | # Set label of y-axis
14 | set ylabel 'time to first prompt in seconds'
15 | 
16 | # Set y-axis range to start at 0
17 | set yrange [0:]
18 | 
19 | # Disable the tiny tick lines on both axis
20 | set xtics scale 0
21 | set ytics scale 0
22 | 
23 | # Disable legend
24 | set key off
25 | 
26 | # White background
27 | set style rectangle fillstyle noborder
28 | set object 1 rectangle behind from screen 0,0 to screen 1,1 fillcolor rgb "white"
29 | 
30 | set bmargin 4.5
31 | set boxwidth 0.5
32 | set style fill solid
33 | set xtics font ",10"
34 | set ytics font ",10"
35 | plot '<(grep -v vanilla results/results.csv | sed "s/-/\\\n/" | sort -t, -g -k2)' using 2: xtic(1) with boxes linecolor rgb "#dc1e3d"
36 | 


--------------------------------------------------------------------------------
/run.zsh:
--------------------------------------------------------------------------------
  1 | () {
  2 | builtin emulate -L zsh -o EXTENDED_GLOB
  3 | 
  4 | if (( SHLVL > 1 )); then
  5 |   print -u2 "::error::Cannot be executed on a nested shell. SHLVL is ${SHLVL}."
  6 |   return 1
  7 | fi
  8 | if (( ! ${+commands[expect]} )); then
  9 |   print -u2 "::error::expect command required."
 10 |   return 1
 11 | fi
 12 | if ! autoload -Uz is-at-least || ! is-at-least '5.2'; then
 13 |   print -u2 "::error::Zsh >= 5.2 required."
 14 |   return 1
 15 | fi
 16 | 
 17 | local -r run_dir=${PWD:A}
 18 | local test_dir=${run_dir}/results
 19 | local work_dir=${${TMPDIR:-${TMPPREFIX}}:A}/${RANDOM}
 20 | local -i keep_frameworks=0
 21 | local -i iterations=100
 22 | # adding vanilla first, because it should always be the baseline
 23 | local -r available_frameworks=(vanilla frameworks/(^vanilla.*)(N:t:r))
 24 | local frameworks=()
 25 | 
 26 | # ensure to use dot ('.') as decimal separator, because some locale (ex: it_IT) use comma (',')
 27 | unset LC_NUMERIC
 28 | 
 29 | local -r usage="source run.zsh [options]
 30 | Options:
 31 |   -h              Show this help
 32 |   -k              Keep the frameworks after the tests are complete (default: delete)
 33 |   -p <path>       Set the path to where the frameworks should be installed (default: results)
 34 |   -w <path>       Set the working directory (default: temp directory)
 35 |   -n <number>     Set the number of iterations to run for each framework (default: 100)
 36 |   -f <framework>  Specify framework to benchmark (default: all; can specify more than once)"
 37 | 
 38 | while (( # )); do
 39 |   case ${1} in
 40 |     -h) print ${usage}
 41 |         return 0
 42 |         ;;
 43 |     -k) keep_frameworks=1
 44 |         shift
 45 |         ;;
 46 |     -p) shift
 47 |         test_dir=${1:A}
 48 |         shift
 49 |         ;;
 50 |     -w) shift
 51 |         work_dir=${1:A}
 52 |         shift
 53 |         ;;
 54 |     -n) shift
 55 |         iterations=${1}
 56 |         shift
 57 |         ;;
 58 |     -f) shift
 59 |         if [[ ${available_frameworks[(r)${1}]} == ${1} ]]; then
 60 |           frameworks+=(${1})
 61 |         else
 62 |           print -u2 "::error::Framework \"${1}\" unknown. Available frameworks are: ${available_frameworks}"
 63 |           return 1
 64 |         fi
 65 |         shift
 66 |         ;;
 67 |     *) print -u2 "::error::Invalid option \"${1}\"\n"
 68 |        print -u2 ${usage}
 69 |        return 1
 70 |        ;;
 71 |   esac
 72 | done
 73 | 
 74 | if (( # )); then
 75 |   print -u2 ${usage}
 76 |   return 1
 77 | fi
 78 | 
 79 | command mkdir -p ${test_dir} || return 1
 80 | command mkdir -p ${work_dir} || return 1
 81 | if (( ! ${#frameworks} )); then
 82 |   frameworks=(${available_frameworks})
 83 | fi
 84 | 
 85 | set_up() {
 86 |   local -r home_dir=${test_dir}/${1}
 87 |   if [[ -e ${home_dir} && ${keep_frameworks} -eq 1 ]]; then
 88 |     print "${1} already installed"
 89 |     return 0
 90 |   fi
 91 | 
 92 |   # first delete any old instances of the frameworks
 93 |   command rm -rf ${home_dir} || return 1
 94 | 
 95 |   # setup the directory for the framework
 96 |   command mkdir -p ${home_dir} || return 1
 97 | 
 98 |   # source the installer
 99 |   print "::group::Setting up ${1} ..."
100 |   {
101 |     ( cd ${home_dir} && source ${run_dir}/frameworks/${1}.zsh || return 1 )
102 |   } always {
103 |     print '\n::endgroup::'
104 |   }
105 | }
106 | 
107 | benchmark() {
108 |   local -r home_dir=${test_dir}/${1} timediv=1000000
109 |   # warmup
110 |   builtin pushd -q ${work_dir}
111 |   {
112 |     repeat 3 do
113 |       HOME=${home_dir} ${run_dir}/expect-warmup || return 1
114 |     done >! ${test_dir}/${1}_out.log 2>! ${test_dir}/${1}_err.log
115 |     repeat ${iterations} do
116 |       HOME=${home_dir} ${run_dir}/expect-run || return 1
117 |     done >> ${test_dir}/${1}_out.log 2>! ${test_dir}/${1}.log
118 |   } always {
119 |     builtin popd -q
120 |   }
121 |   if command grep -v '^[0-9]\+$' ${test_dir}/${1}.log; then
122 |     print -u2 "::error::Unexpected output when benchmarking ${1}"
123 |     return 1
124 |   fi
125 |   command awk -v framework=${1} -v timediv=${timediv} '
126 | count == 0 || $1 < min { min = $1 }
127 | count == 0 || $1 > max { max = $1 }
128 | {
129 |   sum += $1
130 |   sumsq += $1^2
131 |   count++
132 | }
133 | END {
134 |   if (count > 0) {
135 |     mean = sum/count
136 |     if (min < max) { stddev = sqrt(sumsq/count - mean^2) } else { stddev = 0 }
137 |   }
138 |   print framework "," mean/timediv "," stddev/timediv "," min/timediv "," max/timediv
139 | }' ${test_dir}/${1}.log | command tee -a ${results_file}
140 | }
141 | 
142 | # Useful for debugging.
143 | local -r results_file=${test_dir}/results.csv
144 | print "Results: ${results_file}\n"
145 | 
146 | print "This may take a LONG time, as it runs each framework startup ${iterations} times.
147 | Average startup times for each framework will be printed as the tests progress.\n"
148 | 
149 | print "Using Zsh ${ZSH_VERSION}\n"
150 | 
151 | {
152 |   local framework
153 |   for framework in ${frameworks}; do
154 |     set_up ${framework} || return 1
155 |   done
156 |   print -P "\n%F{green}Benchmarking ...%f"
157 |   print 'framework,mean,stddev,min,max' | command tee ${results_file}
158 |   for framework in ${frameworks}; do
159 |     benchmark ${framework} || return 1
160 |   done
161 | } always {
162 |   # cleanup frameworks unless '-k' was provided
163 |   if (( ! keep_frameworks )); then
164 |     command rm -rf ${test_dir}/*(/) || return 1
165 |   fi
166 | }
167 | } "${@}"
168 | 


--------------------------------------------------------------------------------