├── .circleci └── config.yml ├── .github ├── ISSUE_TEMPLATE │ ├── bug.md │ └── feature_request.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── LICENSE ├── README.md ├── _config.yml ├── assets ├── build_aborted.png ├── build_noqueue.png ├── build_progressed.png ├── build_queue2.png └── build_queued.png ├── bors.toml ├── scripts ├── goodbye.bash └── loop.bash ├── src ├── @orb.yml ├── commands │ └── until_front_of_line.yml └── jobs │ └── block_workflow.yml └── test ├── api ├── jobs │ ├── nopreviousjobs.json │ ├── onepreviousjob-differentname.json │ ├── onepreviousjobsamename.json │ ├── regex-matches.json │ └── regex-no-matches.json └── workflows │ ├── first_workflow.json │ └── second_workflow.json ├── bats_helper.bash ├── inputs ├── command-anybranch.yml ├── command-defaults.yml ├── command-dont-quit.yml ├── command-job-regex.yml ├── command-non-default.yml ├── fulljob-noblock.yml └── fulljob.yml └── test_expansion.bats /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2.1 3 | 4 | parameters: 5 | orbname: 6 | type: string 7 | default: "queue" 8 | description: Change this to whatever name is correct for your orb. 9 | 10 | workflows: 11 | build-deploy: 12 | jobs: 13 | - validate: 14 | filters: 15 | branches: 16 | ignore: 17 | - master 18 | - test: 19 | context: [orbs] 20 | requires: 21 | - validate 22 | filters: 23 | branches: 24 | only: 25 | - staging 26 | - trying 27 | - publish: 28 | context: [orbs] 29 | requires: 30 | - test 31 | filters: 32 | branches: 33 | only: staging 34 | 35 | jobs: 36 | validate: 37 | docker: 38 | - image: cimg/base:stable 39 | working_directory: ~/repo 40 | steps: 41 | - checkout 42 | - install-circleci 43 | - pack-and-validate 44 | test: 45 | docker: 46 | - image: cimg/base:stable 47 | working_directory: ~/repo 48 | steps: 49 | - checkout 50 | - install-circleci 51 | - pack-and-validate 52 | - pr-info 53 | - run: 54 | name: Publish Dev 55 | command: | 56 | PUBLISH_MESSAGE=`circleci orb publish packed.yml eddiewebb/<>@dev:<> --token ${CIRCLECI_API_KEY}` 57 | ORB_VERSION=$(echo $PUBLISH_MESSAGE | sed -n 's/Orb `\(.*\)` was published.*/\1/p') 58 | echo "export ORB_VERSION=\"${ORB_VERSION}\"" >> $BASH_ENV 59 | echo $ORB_VERSION 60 | echo "export PR_MESSAGE=\"BotComment: *Development* version of orb available for manual validation - \\\`${ORB_VERSION}\\\`\"" >> $BASH_ENV 61 | - install-bats 62 | - run: 63 | name: Import Tests using BATS 64 | command: | 65 | export BATS_IMPORT_DEV_ORB="eddiewebb/<>@dev:<>" 66 | bats --jobs 8 test 67 | - pr-comment 68 | - run: 69 | name: Check Semver 70 | command: | 71 | if [ "$PR_NUMBER" == "" ];then 72 | echo "No pr found, do nothing" 73 | exit 0 74 | fi 75 | TITLE=`curl -u eddiewebb:${GHI_TOKEN} "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/issues/${PR_NUMBER}" | jq '.title' ` 76 | SEMVER_INCREMENT=`echo $TITLE | sed -En 's/.*\[semver:(major|minor|patch|skip)\].*/\1/p'` 77 | if [ -z ${SEMVER_INCREMENT} ];then 78 | echo "The **source PR** did not indicate which SemVer increment to make. Please ammend commit with [semver:FOO] where FOO is major, minor, or patch" 79 | exit 1 80 | fi 81 | 82 | publish: 83 | docker: 84 | - image: cimg/base:stable 85 | working_directory: ~/repo 86 | steps: 87 | - checkout 88 | - install-circleci 89 | - pr-info 90 | - when: 91 | condition: 92 | or: 93 | - equal: [ trying, << pipeline.git.branch >> ] 94 | - equal: [ staging, << pipeline.git.branch >> ] 95 | steps: 96 | - run: 97 | name: Promote to prod 98 | command: | 99 | SEMVER_INCREMENT=`git log -1 --pretty=%B | sed -En 's/.*\[semver:(major|minor|patch|skip)\].*/\1/p'` 100 | if [ -z ${SEMVER_INCREMENT} ];then 101 | echo "Merge commit did not indicate which SemVer increment to make. Please ammend commit with [semver:FOO] where FOO is major, minor, or patch" 102 | exit 1 103 | elif [ "$SEMVER_INCREMENT" == "skip" ];then 104 | echo "SEMVER in commit indicated to skip orb release" 105 | echo "export PR_MESSAGE=\"Orb publish was skipped due to [semver:skip] in commit message.\"" >> $BASH_ENV 106 | exit 0 107 | else 108 | PUBLISH_MESSAGE=`circleci orb publish promote eddiewebb/<>@dev:<> ${SEMVER_INCREMENT} --token ${CIRCLECI_API_KEY}` 109 | echo $PUBLISH_MESSAGE 110 | ORB_VERSION=$(echo $PUBLISH_MESSAGE | sed -n 's/Orb .* was promoted to `\(.*\)`.*/\1/p') 111 | echo "export PR_MESSAGE=\"BotComment: *Production* version of orb available for use - \\\`${ORB_VERSION}\\\`\"" >> $BASH_ENV 112 | fi 113 | - pr-comment 114 | 115 | 116 | 117 | 118 | commands: 119 | install-bats: 120 | description: installs the BATS bash testing tool 121 | steps: 122 | - run: 123 | name: Install BATS (bash testing) 124 | command: | 125 | cd /tmp && git clone https://github.com/bats-core/bats-core.git && cd bats-core 126 | sudo ./install.sh /usr/local 127 | - run: 128 | name: Install YQ 129 | command: | 130 | curl -L https://github.com/mikefarah/yq/releases/download/v4.14.1/yq_linux_amd64 -o yq 131 | chmod a+x yq 132 | mv yq /usr/local/bin/ 133 | install-circleci: 134 | description: installs the new CIrcleCI CLI with orb support 135 | steps: 136 | - run: 137 | name: Install CircleCI CLI (the new one) 138 | command: | 139 | curl https://raw.githubusercontent.com/CircleCI-Public/circleci-cli/master/install.sh --fail --show-error | sudo bash 140 | circleci version 141 | echo "Run circleci help" 142 | circleci --help 143 | echo -e "token: ${CIRCLECI_API_KEY}\nverbose: false" > .circleci/cli.yml 144 | pack-and-validate: 145 | description: pack directory to single file and validate 146 | steps: 147 | - run: 148 | name: Pack and Validate 149 | command: | 150 | circleci orb pack src > packed.yml 151 | circleci orb validate packed.yml 152 | pr-comment: 153 | description: add message to pr this build originated from 154 | steps: 155 | - run: 156 | name: Publish Version to PR 157 | command: | 158 | if [ "$PR_NUMBER" == "" ];then 159 | echo "No pr found, do nothing" 160 | exit 0 161 | fi 162 | curl -X POST -u eddiewebb:${GHI_TOKEN} "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/issues/${PR_NUMBER}/comments" -d "{\"body\":\"${PR_MESSAGE}\"}" 163 | pr-info: 164 | description: get PR number this change originated from 165 | steps: 166 | - run: 167 | name: Get PR Info 168 | command: | 169 | #During OPEN PR, get from CCI. 170 | PR_NUMBER=${CIRCLE_PULL_REQUEST##*/} 171 | if [ -z "$PR_NUMBER" ];then 172 | # get from merge commit on closed PR. 173 | PR_NUMBER=`git log -1 --pretty=%s | sed -En 's/.*\(#([0-9]*)\).*/\1/p'` 174 | fi 175 | echo "PR_NUMBER is ${PR_NUMBER}" 176 | echo "export PR_NUMBER=${PR_NUMBER}" >> $BASH_ENV 177 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug 3 | about: A technical problem 4 | title: "[semver:patch] DESCRIPTION OF BUG" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Orb version 11 | 12 | 19 | 20 | ### What happened 21 | 22 | 26 | 27 | ### Expected behavior 28 | 29 | 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[semver:minor] HOW ITS BETTER HERE" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ### Checklist 2 | 3 | 8 | 9 | - [ ] All new jobs, commands, executors, parameters have descriptions 10 | - [ ] Examples have been added for any significant new features 11 | - [ ] README has been updated, if necessary 12 | 13 | ### Motivation, issues 14 | 15 | 20 | 21 | ### Description 22 | 23 | 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | tempDockerfile 103 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Eddie Webb 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > **This Orb is deprecated by native CircleCI functionality.** 2 | > 3 | > For details, see the [Deprecation Notice issue](https://github.com/eddiewebb/circleci-queue/issues/142) and the official [CircleCI changelog entry](https://circleci.com/changelog/avoid-commit-conflicts-in-your-pipelines-with-serial-jobs/) for `serial-group`. 4 | 5 | # CircleCI Concurrency Control Orb 6 | 7 | [![CircleCI](https://img.shields.io/circleci/build/gh/eddiewebb/circleci-queue)](https://circleci.com/gh/eddiewebb/circleci-queue/tree/master) 8 | [![GitHub license](https://img.shields.io/github/license/eddiewebb/circleci-queue)](https://github.com/eddiewebb/circleci-queue/blob/master/LICENSE) 9 | [![CircleCI Orb Version](https://img.shields.io/badge/endpoint.svg?url=https://badges.circleci.io/orb/eddiewebb/queue)](https://circleci.com/orbs/registry/orb/eddiewebb/queue) 10 | [![Bors enabled](https://bors.tech/images/badge_small.svg)](https://app.bors.tech/repositories/21077) 11 | 12 | CircleCI Orb to limit workflow concurrency. 13 | 14 | Why? Some jobs (typically deployments) need to run sequentially and not parallel, but also run to completion. So CircleCI's native `auto-cancel` is not quite the right fit. 15 | See https://github.com/eddiewebb/circleci-challenge as an example using blue/green cloud foundry deployments. 16 | 17 | 18 | ## Basic Usage 19 | 20 | This adds concurrency limits by ensuring any jobs with this step will only continue once no previous builds are running. It supports a single argument of how many minutes to wait before aborting itself and it requires a single Environment Variable `CIRCLECI_API_KEY`, which must be a **Personal API Token** (rather than a project-specific API Permissions token). This can be created at [Personal API Tokens](https://app.circleci.com/settings/user/tokens) under Users Settings. Note that the account must have write access (at least the **Contributor** role) on the Project you wish to enable this orb for. However, if the `dont-quit` parameter is enabled, view-only access (the **Viewer** role) is sufficient. 21 | 22 | ## Screenshots / Examples 23 | 24 | Suppose we have a workflow take takes a little while to run. Normally the build (#18) will run immediately, with no queuing. 25 | ![no queuing if only active build](assets/build_noqueue.png) 26 | 27 | Someone else on the team makes another commit, since the first build (#18) is still running, it will queue build #19. 28 | ![no queuing if only active build](assets/build_queue2.png) 29 | 30 | It's late afternoon, everyone is pushing their commits in to ensure they are good before they leave for the day. Build #20 also queues. 31 | ![no queuing if only active build](assets/build_queued.png) 32 | 33 | Meanwhile, build #19 is now allowed to move forward since build #18 finished. 34 | 35 | ![no queuing if only active build](assets/build_progressed.png) 36 | 37 | Oh No! Since `1 minute` is abnormally long for things to be queued, build #20 aborts itself, letting build #19 finish uninterrupted. 38 | 39 | ![no queuing if only active build](assets/build_aborted.png) 40 | 41 | # Setup 42 | See https://circleci.com/orbs/registry/orb/eddiewebb/queue#usage-examples for current examples 43 | 44 | ## Note 45 | 46 | Queueing is not supported on forked repos. If a queue from a fork happens the queue will immediately exit and the next step of the job will begin. 47 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-hacker -------------------------------------------------------------------------------- /assets/build_aborted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eddiewebb/circleci-queue/907af86ab9704c679208ef7c3c65b423a31fadbd/assets/build_aborted.png -------------------------------------------------------------------------------- /assets/build_noqueue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eddiewebb/circleci-queue/907af86ab9704c679208ef7c3c65b423a31fadbd/assets/build_noqueue.png -------------------------------------------------------------------------------- /assets/build_progressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eddiewebb/circleci-queue/907af86ab9704c679208ef7c3c65b423a31fadbd/assets/build_progressed.png -------------------------------------------------------------------------------- /assets/build_queue2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eddiewebb/circleci-queue/907af86ab9704c679208ef7c3c65b423a31fadbd/assets/build_queue2.png -------------------------------------------------------------------------------- /assets/build_queued.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eddiewebb/circleci-queue/907af86ab9704c679208ef7c3c65b423a31fadbd/assets/build_queued.png -------------------------------------------------------------------------------- /bors.toml: -------------------------------------------------------------------------------- 1 | status = [ 2 | 'build-deploy' 3 | ] 4 | -------------------------------------------------------------------------------- /scripts/goodbye.bash: -------------------------------------------------------------------------------- 1 | RED='\033[0;31m' 2 | GREEN='\033[0;32m' 3 | GREY='\033[0;37m' 4 | BOLD='\033[1m' 5 | ENDBOLD='\033[0m' 6 | NC='\033[0m' # No Color 7 | echo -e ${RED} 8 | cat <<"EOF" 9 | 10 | 11 | _____ _ _ 12 | | __ \ | | (_) 13 | | | | | ___ _ __ _ __ ___ ___ __ _| |_ _ ___ _ __ 14 | | | | |/ _ \ '_ \| '__/ _ \/ __/ _` | __| |/ _ \| '_ \ 15 | | |__| | __/ |_) | | | __/ (_| (_| | |_| | (_) | | | | 16 | |_____/ \___| .__/|_| \___|\___\__,_|\__|_|\___/|_| |_| 17 | | | 18 | |_| 19 | _ _ _ _ 20 | | \ | | | | (_) 21 | | \| | ___ | |_ _ ___ ___ 22 | | . ` |/ _ \| __| |/ __/ _ \ 23 | | |\ | (_) | |_| | (_| __/ 24 | |_| \_|\___/ \__|_|\___\___| 25 | EOF 26 | echo -e "${NC}" 27 | echo -e "${GREEN}Good News!${NC}" 28 | echo -e " 29 | This orb was first authored in 2018, and has served over 10,000,000 builds. 30 | 31 | ${BOLD}But as of March, 2025, it is no longer supported${ENDBOLD} as CircleCI now supports native 32 | serialization. ${RED} 33 | - The author will no longer accept enhancement requests effectively immediately. 34 | - The CircleCI team is planning to remove v1 APIs used by this orb in the future.${NC} 35 | 36 | ${BOLD}Users of this orb are encouraged to adopt the \`serial-group\` stanza${ENDBOLD} instead. 37 | 38 | 39 | 40 | workflows: 41 | main-workflow: 42 | jobs: 43 | - test 44 | - build 45 | - deploy: 46 | ${GREEN}serial-group: << pipeline.project.slug >>/deploy-group${NC} 47 | requires: 48 | - test 49 | - build 50 | 51 | 52 | 53 | Please visit https://circleci.com/docs/configuration-reference/#serial-group to learn more. 54 | " 55 | 56 | echo -e "${grey}" 57 | 58 | cat <<"EOF" 59 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣀⣤⣤⣤⣤⣤⣤⣤⣤⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 60 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣠⣤⠖⠛⢭⣭⣥⣀⣠⡤⢤⣄⣀⣀⣈⡉⠉⠛⠳⢦⣤⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 61 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣠⡶⠟⠉⠀⠀⠤⠤⠀⣀⣨⣭⠷⣦⠲⢤⡀⠀⣉⡟⠙⠛⠲⠾⢿⣷⣄⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 62 | ⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣟⣋⣠⣤⠤⠤⣤⡴⠖⠛⠉⠀⠀⠀⠈⣳⡀⡟⠛⠛⠁⠀⠀⠀⠀⠀⠈⠙⠿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 63 | ⠀⠀⠀⠀⠀⠀⠀⣴⣿⠿⠭⠉⠀⢛⣷⠞⠉⠀⠀⠀⠀⠀⠀⠀⠀⢹⣼⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢻⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 64 | ⠀⠀⠀⠀⠀⢀⣾⣯⠄⠠⠀⢐⣶⠟⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⣴⣿⣧⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣸⠞⢿⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 65 | ⠀⠀⠀⠀⢠⣿⠁⠀⠀⢀⣤⡾⡁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣟⠁⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠰⠏⠀⠈⢻⣆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 66 | ⠀⠀⠀⢀⡿⠁⠀⠀⠀⣴⡟⢈⣿⢞⠞⠃⠀⠀⠀⠀⠀⠀⠀⢀⣿⡅⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠐⢹⣇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 67 | ⠀⠀⠀⣾⠃⠀⠀⠤⢴⡿⠀⡴⣷⣫⠆⠀⠀⠀⠀⠀⠀⠀⠀⣾⠀⠛⠀⠀⠀⠀⠠⣶⣶⣿⣿⣷⡄⠀⠀⠀⠀⠀⠀⠀⠀⢹⣆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 68 | ⠀⠀⣸⡯⠴⠒⠒⠒⣾⠃⠀⠾⡽⡍⠀⠀⠀⠀⠀⠀⠀⠀⢠⠟⠄⢠⣤⣶⠀⠀⠀⢹⣿⣍⠉⢿⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⢿⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 69 | ⠀⠀⣿⠓⠃⠀⠀⢨⡟⠀⠀⠀⠀⠀⠀⠀⠀⣶⣶⣾⣿⣷⣏⠀⠀⠈⣿⣿⡆⠀⠀⠘⣿⣿⡄⠘⣿⡆⠀⠀⠀⠀⠀⠀⠀⠀⠈⣷⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 70 | ⠀⢠⡿⠀⠀⠀⠉⢹⡇⠀⠀⠀⠀⠀⠀⠀⠀⢸⣿⣯⠙⢿⣿⡆⠀⠀⢹⣿⣿⠀⠀⠀⢹⣿⣇⠀⣿⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⣧⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 71 | ⠀⠘⣷⡄⠀⠚⠛⢻⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⠀⠀⣿⣇⠀⠀⠀⣿⣿⡆⠀⠀⠘⣿⣿⣿⣿⡿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 72 | ⠀⠀⠘⣿⠒⠀⠘⢻⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⢿⣿⡇⠀⣿⣿⠀⠀⠀⢻⣿⣧⠀⠀⠀⢿⣿⡟⠋⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣹⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 73 | ⠀⠀⢀⣿⠶⠆⠒⢺⡇⠀⠀⢀⡤⠂⠀⠀⠀⠀⢸⣿⣿⣴⣿⡿⠀⠀⠀⠸⣿⣿⡀⠀⠀⢸⣿⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢈⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 74 | ⠀⠀⣿⠗⠀⠀⢉⣹⣧⣤⠟⠃⠀⠀⠀⠀⠀⠀⠘⣿⣿⣿⣿⣥⡀⠀⠀⠀⢿⣿⣷⠀⠀⠈⣿⣿⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠴⠛⠿⣶⣄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 75 | ⠀⠀⣿⡃⠉⢉⣀⣠⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⣿⣿⣷⡄⠀⠀⠘⣿⣿⡀⠀⠀⢸⣿⣧⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 76 | ⠀⠀⣿⣏⣁⣀⠤⠤⣿⡆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣿⣿⠀⢹⣿⣇⠀⠀⠀⢿⣿⣇⠀⠀⠈⣿⣿⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢿⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀ 77 | ⠀⠀⠹⣧⢤⠔⠒⠛⢙⣏⣤⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⡆⠀⣿⣿⡀⠀⠀⢸⣿⢿⣄⠀⠀⢿⣿⣷⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣀⣸⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀ 78 | ⠀⠀⠸⣯⡆⠀⠐⢊⣉⣿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢹⣿⣇⠀⢻⣿⣿⡇⠀⠘⣿⠟⠉⠀⠀⠚⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢠⡏⠙⣇⠀⠀⠀⠀⠀⠀⠀⠀⠀ 79 | ⠀⠀⠘⣿⠀⠈⢠⣭⡤⢿⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠘⣿⣿⡆⠀⠙⠿⠇⠀⠀⠀⠀⠀⠀⠈⠠⢄⠀⠀⠀⠀⠀⠀⠀⠀⠀⣠⡴⠋⠀⠀⣿⡀⠀⠀⠀⠀⠀⠀⠀⠀ 80 | ⠀⠀⠀⣿⡦⠤⠬⢤⣰⣾⣷⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢰⠋⠀⠀⠀⠀⢻⡇⠀⠀⠀⠀⠀⠀⠀⠀ 81 | ⠀⠀⠀⢹⡷⠤⠤⠐⠚⣛⣿⣹⣆⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⠀⠀⠀⠀⠀⢀⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣀⣸⠃⠀⠀⣴⣖⣿⣧⠀⠀⠀⠀⠀⠀⠀⠀ 82 | ⠀⠀⠀⠸⣿⠒⠒⢚⣉⣉⣿⡏⠓⠢⠄⠒⠂⠀⠀⠀⠀⠀⠀⢀⡴⠃⢀⡔⠁⠀⡠⠋⠀⠀⡠⠀⣀⠀⠀⠀⠀⠀⠀⢠⡟⠁⠀⠀⣼⣝⡼⡾⣿⡴⢶⡖⠀⠀⠀⠀⠀ 83 | ⠀⣀⠀⠀⣿⡏⡀⠈⠁⠐⣻⡇⠀⠀⠀⢀⠀⠀⠀⠀⠀⠀⣠⠎⠀⣠⠎⠀⢀⡞⠁⠀⢀⠔⠁⡴⠃⠀⠀⠀⠀⠀⣰⠏⠀⠀⠀⢀⣘⣿⣧⣤⢾⣇⢸⣷⣶⡿⠀⠀⠀ 84 | ⠀⠻⣏⠻⢿⣿⣶⣴⣿⣿⠿⣷⣴⣆⣾⣏⡜⣦⣤⢠⠆⣠⠁⢀⠞⠁⠀⡰⠋⠀⠀⡰⠋⢠⠞⠀⠀⠀⣠⠴⠊⠉⠀⠀⠀⠀⣠⣿⣿⢋⣤⣷⣯⡿⠋⠀⡿⠀⠀⠀⠀ 85 | ⢀⠀⣈⣷⡂⠙⢿⣾⠻⡽⡍⠻⢿⣹⣻⢿⣷⣻⢃⠿⣾⠋⣷⡏⣠⣠⠞⠁⠀⣠⠎⠀⠴⠃⠀⢀⡞⢉⡄⠀⠀⠀⣰⣴⢀⡞⣿⣿⣿⣿⡏⠳⠏⠀⠀⢸⣁⣀⡀⠀⠀ 86 | ⠘⢿⡺⣿⠃⠀⠀⠙⢶⡻⢿⠀⠀⠙⠿⠈⢯⡿⠃⢠⠏⣾⣽⣧⣿⢻⠃⣠⠞⠁⠀⠀⢠⣶⢺⢛⡇⡾⢠⠀⠀⠀⢷⣧⣿⡟⠿⢿⡀⠈⠋⠀⠀⠀⠀⠈⠁⡞⠀⠀⠀ 87 | ⠀⠀⢳⡀⠀⠀⠘⢯⠛⠃⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⠞⢉⣿⣿⢧⡏⣼⣻⢠⢦⠆⣄⣚⢣⡏⡾⢸⠁⡎⠟⠀⠀⢻⣞⠽⣽⠄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠓⠍⠀⠀ 88 | ⠀⠀⠀⠓⠀⣀⣀⣬⡷⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠟⢿⡟⢰⢱⢣⠏⡟⣼⣽⡿⠚⠉⣉⣙⣶⠳⠶⠶⠦⠿⢿⣷⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠀⠀⠀ 89 | ⠀⠀⠀⠀⠀⠈⠙⢦⡄⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠓⠟⠉⠉⠛⠉⠉⢥⣴⠓⠦⠀⠀⠀⠀⠉⠉⠉⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠠⠤⠔⠃ 90 | ⠀⠀⠀⠀⠀⠰⠚⠉⠀⠀⠀⠀⠀⠀⠀⣀⠀⢀⡀⠀⠀⠀⠀⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣀⣀⣀⣠⠔⠒⠒⠒⠚⠛⠓⠦⠤⠄⠀⠀ 91 | ⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠳⠶⠒⠒⠚⠛⠂⠀⠀⠈⠉⠉⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀ 92 | 93 | 94 | Thank you for the adoption and contributions during the past 7 years. 95 | 96 | - Eddie 97 | EOF -------------------------------------------------------------------------------- /scripts/loop.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # This script uses many environment variables, some set from pipeline parameters. See orb yaml for source. 5 | # 6 | 7 | log(){ # add UTC timestamps to log 8 | command echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC')" -- "$@" 9 | } 10 | 11 | respond(){ 12 | command echo "$@" 13 | } 14 | 15 | load_variables(){ 16 | TMP_DIR=$(mktemp -d) 17 | SHALLOW_JOBSTATUS_PATH="$TMP_DIR/jobstatus.json" 18 | AUGMENTED_JOBSTATUS_PATH="$TMP_DIR/augmented_jobstatus.json" 19 | log "Block: $BLOCK_WORKFLOW" 20 | : "${MAX_TIME:?"Required Env Variable not found!"}" 21 | start_time=$(date +%s) 22 | loop_time=11 23 | max_time_seconds=$(( 60 * $MAX_TIME )) 24 | # just confirm our required variables are present 25 | : "${CIRCLE_BUILD_NUM:?"Required Env Variable not found!"}" 26 | : "${CIRCLE_PROJECT_USERNAME:?"Required Env Variable not found!"}" 27 | : "${CIRCLE_PROJECT_REPONAME:?"Required Env Variable not found!"}" 28 | : "${CIRCLE_REPOSITORY_URL:?"Required Env Variable not found!"}" 29 | : "${CIRCLE_JOB:?"Required Env Variable not found!"}" 30 | VCS_TYPE="github" 31 | if [[ "$CIRCLE_REPOSITORY_URL" =~ .*bitbucket\.org.* ]]; then 32 | VCS_TYPE="bitbucket" 33 | log "VCS_TYPE set to bitbucket" 34 | fi 35 | : "${VCS_TYPE:?"Required VCS TYPE not found! This is likely a bug in orb, please report."}" 36 | : "${MY_PIPELINE_NUMBER:?"Required MY_PIPELINE_NUMBER not found! This is likely a bug in orb, please report."}" 37 | 38 | # If a pattern is wrapped with slashes, remove them. 39 | if [[ "$TAG_PATTERN" == /*/ ]]; then 40 | TAG_PATTERN="${TAG_PATTERN:1:-1}" 41 | fi 42 | log "Expecting CCI Personal Access TOKEN Named: $CCI_API_KEY_NAME" 43 | CCI_TOKEN="${!CCI_API_KEY_NAME}" 44 | # Only needed for private projects 45 | if [ -z "$CCI_TOKEN" ]; then 46 | log "CCI_TOKEN not set. Private projects and force cancel will not function." 47 | else 48 | fetch "${CIRCLECI_BASE_URL}/api/v2/me" "$TMP_DIR/me.cci" 49 | me=$(jq -e '.id' "$TMP_DIR/me.cci") 50 | log "Using API key for user: $me on host $CIRCLECI_BASE_URL" 51 | fi 52 | 53 | if [[ $DEBUG != "false" ]]; then 54 | log "Using Temp Dir: $TMP_DIR" 55 | #set 56 | fi 57 | } 58 | 59 | 60 | do_we_run(){ 61 | if [ -n "$CIRCLE_TAG" ] && [ -z "$TAG_PATTERN" ]; then 62 | log "TAG_PATTERN defined, but not on tagged run, skip queueing!" 63 | exit 0 64 | fi 65 | 66 | if [ -n "$CIRCLE_PR_REPONAME" ]; then 67 | log "Queueing on forks is not supported. Skipping queue..." 68 | # It's important that we not fail here because it could cause issues on the main repo's branch 69 | exit 0 70 | fi 71 | if [[ "$ONLY_ON_BRANCH" == "*" ]] || [[ "$ONLY_ON_BRANCH" == "$CIRCLE_BRANCH" ]]; then 72 | log "$CIRCLE_BRANCH matches queueable branch names" 73 | else 74 | log "Queueing only enforced on branch '$ONLY_ON_BRANCH', skipping queue" 75 | exit 0 76 | fi 77 | } 78 | 79 | 80 | update_active_run_data(){ 81 | fetch_filtered_active_builds 82 | augment_jobs_with_pipeline_data 83 | 84 | JOB_NAME="$CIRCLE_JOB" 85 | if [ -n "$JOB_REGEXP" ]; then 86 | JOB_NAME="$JOB_REGEXP" 87 | use_regex=true 88 | else 89 | use_regex=false 90 | fi 91 | 92 | # falsey parameters are empty strings, so always compare against 'true' 93 | if [ "$BLOCK_WORKFLOW" != "false" ]; then 94 | log "Orb parameter block-workflow is true. Any previous (matching) pipelines with running workflows will block this entire workflow." 95 | if [ "$ONLY_ON_WORKFLOW" = "*" ]; then 96 | log "No workflow name filter. This job will block until no previous workflows with *any* name are running, regardless of job name." 97 | oldest_running_build_num=$(jq 'sort_by(.workflows.pipeline_number)| .[0].build_num' "$AUGMENTED_JOBSTATUS_PATH") 98 | front_of_queue_pipeline_number=$(jq -r 'sort_by(.workflows.pipeline_number)| .[0].workflows.pipeline_number // empty' "$AUGMENTED_JOBSTATUS_PATH") 99 | else 100 | log "Orb parameter limit-workflow-name is provided." 101 | log "This job will block until no previous occurrences of workflow $ONLY_ON_WORKFLOW are running, regardless of job name" 102 | oldest_running_build_num=$(jq --arg ONLY_ON_WORKFLOW "$ONLY_ON_WORKFLOW" '. | map(select(.workflows.workflow_name == $ONLY_ON_WORKFLOW)) | sort_by(.workflows.pipeline_number) | .[0].build_num' "$AUGMENTED_JOBSTATUS_PATH") 103 | front_of_queue_pipeline_number=$(jq -r --arg ONLY_ON_WORKFLOW "$ONLY_ON_WORKFLOW" '. | map(select(.workflows.workflow_name == $ONLY_ON_WORKFLOW)) | sort_by(.workflows.pipeline_number) | .[0].workflows.pipeline_number // empty' "$AUGMENTED_JOBSTATUS_PATH") 104 | fi 105 | else 106 | log "Orb parameter block-workflow is false. Use Job level queueing." 107 | log "Only blocking execution if running previous jobs matching this job: $JOB_NAME" 108 | if [ "$use_regex" = true ]; then 109 | oldest_running_build_num=$(jq --arg JOB_NAME "$JOB_NAME" '. | map(select(.workflows.job_name | test($JOB_NAME; "sx"))) | sort_by(.workflows.pipeline_number) | .[0].build_num' "$AUGMENTED_JOBSTATUS_PATH") 110 | front_of_queue_pipeline_number=$(jq -r --arg JOB_NAME "$JOB_NAME" '. | map(select(.workflows.job_name | test($JOB_NAME; "sx"))) | sort_by(.workflows.pipeline_number) | .[0].workflows.pipeline_number // empty' "$AUGMENTED_JOBSTATUS_PATH") 111 | else 112 | oldest_running_build_num=$(jq --arg JOB_NAME "$JOB_NAME" '. | map(select(.workflows.job_name == $JOB_NAME)) | sort_by(.workflows.pipeline_number) | .[0].build_num' "$AUGMENTED_JOBSTATUS_PATH") 113 | front_of_queue_pipeline_number=$(jq -r --arg JOB_NAME "$JOB_NAME" '. | map(select(.workflows.job_name == $JOB_NAME)) | sort_by(.workflows.pipeline_number) | .[0].workflows.pipeline_number // empty' "$AUGMENTED_JOBSTATUS_PATH") 114 | fi 115 | if [[ "$DEBUG" != "false" ]]; then 116 | log "DEBUG: me: $MY_PIPELINE_NUMBER, front: $front_of_queue_pipeline_number" 117 | fi 118 | fi 119 | 120 | if [ -z "$front_of_queue_pipeline_number" ]; then 121 | log "API Call for existing jobs returned no matches. This means job is alone." 122 | if [[ $DEBUG != "false" ]]; then 123 | log "All running jobs:" 124 | cat "$SHALLOW_JOBSTATUS_PATH" || exit 0 125 | log "All running jobs with created_at:" 126 | cat "$AUGMENTED_JOBSTATUS_PATH" || exit 0 127 | log "All workflow details." 128 | cat /tmp/workflow-*.json || log "Could not load workflows.." 129 | exit 1 130 | fi 131 | fi 132 | } 133 | 134 | 135 | fetch_filtered_active_builds(){ 136 | JOB_API_SUFFIX="?filter=running&shallow=true" 137 | jobs_api_url_template="${CIRCLECI_BASE_URL}/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}${JOB_API_SUFFIX}" 138 | if [ "$FILTER_BRANCH" == "false" ]; then 139 | log "Orb parameter 'this-branch-only' is false, will block previous builds on any branch." 140 | else 141 | # branch filter 142 | : "${CIRCLE_BRANCH:?"Required Env Variable not found!"}" 143 | log "Only blocking execution if running previous jobs on branch: $CIRCLE_BRANCH" 144 | jobs_api_url_template="${CIRCLECI_BASE_URL}/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/tree/$(urlencode "$CIRCLE_BRANCH")${JOB_API_SUFFIX}" 145 | fi 146 | 147 | if [ -n "$TESTING_MOCK_RESPONSE" ] && [ -f "$TESTING_MOCK_RESPONSE" ]; then 148 | log "Using test mock response" 149 | cat "$TESTING_MOCK_RESPONSE" > "$SHALLOW_JOBSTATUS_PATH" 150 | else 151 | fetch "$jobs_api_url_template" "$SHALLOW_JOBSTATUS_PATH" 152 | fi 153 | 154 | if [ -n "$CIRCLE_TAG" ] && [ -n "$TAG_PATTERN" ]; then 155 | log "TAG_PATTERN variable non-empty, will only block pipelines with matching tag" 156 | jq "[ .[] | select((.build_num | . == \"${CIRCLE_BUILD_NUM}\") or (.vcs_tag | (. != null and test(\"${TAG_PATTERN}\"))) ) ]" "$SHALLOW_JOBSTATUS_PATH" > /tmp/jobstatus_tag.json 157 | mv /tmp/jobstatus_tag.json "$SHALLOW_JOBSTATUS_PATH" 158 | fi 159 | } 160 | 161 | augment_jobs_with_pipeline_data(){ 162 | log "Getting queue ordering" 163 | cp "$SHALLOW_JOBSTATUS_PATH" "$AUGMENTED_JOBSTATUS_PATH" 164 | for workflow in $(jq -r ".[] | .workflows.workflow_id //empty" "$AUGMENTED_JOBSTATUS_PATH" | uniq); do 165 | # get workflow to get pipeline... 166 | workflow_file="${TMP_DIR}/workflow-${workflow}.json" 167 | if [ -f "$TESTING_MOCK_WORKFLOW_RESPONSES/${workflow}.json" ]; then 168 | log "Using test mock workflow response" 169 | cat "$TESTING_MOCK_WORKFLOW_RESPONSES/${workflow}.json" > "${workflow_file}" 170 | else 171 | fetch "${CIRCLECI_BASE_URL}/api/v2/workflow/${workflow}" "${workflow_file}" 172 | fi 173 | pipeline_number=$(jq -r '.pipeline_number' "${workflow_file}") 174 | log "Workflow: ${workflow} is from pipeline #${pipeline_number}" 175 | jq --arg pipeline_number "${pipeline_number}" --arg workflow "${workflow}" '(.[] | select(.workflows.workflow_id == $workflow) | .workflows) |= . + {pipeline_number:$pipeline_number}' "$AUGMENTED_JOBSTATUS_PATH" > "${TMP_DIR}/augmented_jobstatus-${workflow}.json" 176 | mv "${TMP_DIR}/augmented_jobstatus-${workflow}.json" "$AUGMENTED_JOBSTATUS_PATH" 177 | done 178 | } 179 | 180 | urlencode(){ 181 | LC_WAS="${LC_ALL:-}" 182 | export LC_ALL=C 183 | string="$1" 184 | while [ -n "$string" ]; do 185 | tail="${string#?}" 186 | head="${string%"$tail"}" 187 | case "$head" in 188 | [-_.~A-Za-z0-9]) printf '%c' "$head" ;; 189 | *) printf '%%%02X' "'$head" ;; 190 | esac 191 | string="${tail}" 192 | done 193 | export LC_ALL="${LC_WAS}" 194 | } 195 | 196 | fetch() { 197 | local max_retries=5 198 | local retry_count=0 199 | local backoff=1 200 | 201 | while : ; do 202 | if [[ $DEBUG != "false" ]]; then 203 | log "DEBUG: Making API Call to ${1}" 204 | fi 205 | url="$1" 206 | target="$2" 207 | 208 | response_headers=$(mktemp) 209 | http_response=$(curl -s -X GET -H "Circle-Token:${CCI_TOKEN}" -H "Content-Type: application/json" -D "$response_headers" -o "${target}" -w "%{http_code}" "${url}") 210 | 211 | if [ "$http_response" -eq 200 ]; then 212 | if [[ $DEBUG != "false" ]]; then 213 | log "DEBUG: API Success" 214 | fi 215 | rm -f "$response_headers" 216 | return 0 217 | elif [ "$http_response" -eq 429 ]; then 218 | retry_after=$(grep -i "Retry-After:" "$response_headers" | awk '{print $2}' | tr -d '\r') 219 | if [[ -n "$retry_after" ]]; then 220 | sleep_duration=$((retry_after)) 221 | else 222 | sleep_duration=$((backoff)) 223 | backoff=$((backoff * 2)) 224 | fi 225 | 226 | if (( retry_count >= max_retries )); then 227 | log "ERROR: Maximum retries reached. Exiting." 228 | rm -f "$response_headers" 229 | cat "${target}" 230 | exit 1 231 | fi 232 | 233 | if [[ $DEBUG != "false" ]]; then 234 | log "DEBUG: Rate limit exceeded. Retrying in $sleep_duration seconds..." 235 | fi 236 | sleep "$sleep_duration" 237 | ((retry_count++)) 238 | else 239 | log "ERROR: Server returned error code: $http_response" 240 | rm -f "$response_headers" 241 | cat "${target}" 242 | exit 1 243 | fi 244 | done 245 | } 246 | 247 | 248 | cancel_build_num(){ 249 | BUILD_NUM="$1" 250 | log "Cancelling build ${BUILD_NUM}" 251 | cancel_api_url_template="${CIRCLECI_BASE_URL}/api/v1.1/project/${VCS_TYPE}/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/${BUILD_NUM}/cancel?circle-token=${CCI_TOKEN}" 252 | curl -s -X POST "$cancel_api_url_template" > /dev/null 253 | } 254 | 255 | 256 | get_wait_time() { 257 | local current_time 258 | current_time=$(date +%s) 259 | #log "Calced wait time: $((current_time - start_time))" 1>&2 260 | respond "$((current_time - start_time))" 261 | } 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | # 270 | # MAIN LOGIC STARTS HERE 271 | # 272 | load_variables 273 | do_we_run # exit early if we can 274 | log "Max Queue Time: ${max_time_seconds} seconds." 275 | # 276 | # Queue Loop 277 | # 278 | confidence=0 279 | while true; do 280 | # get running jobs, filtered to branch or tag, with pipeline ID 281 | update_active_run_data 282 | 283 | log "This Job's Pipeline #: $MY_PIPELINE_NUMBER" 284 | log "Front of Queue (fifo) Pipeline #: $front_of_queue_pipeline_number" 285 | wait_time=$(get_wait_time) 286 | # This condition checks if the current job should proceed based on confidence level: 287 | # 1. If 'front_of_queue_pipeline_number' is empty, it means there are no other jobs in the queue, so the current job can proceed. 288 | # 2. If 'MY_PIPELINE_NUMBER' is non-empty and equals 'front_of_queue_pipeline_number', it means the current job is at the front of the queue and can proceed. 289 | # Confidence level is incremented if either of these conditions is true. 290 | if [[ -z "$front_of_queue_pipeline_number" ]] || ([[ -n "$MY_PIPELINE_NUMBER" ]] && [[ "$front_of_queue_pipeline_number" == "$MY_PIPELINE_NUMBER" ]]); then 291 | # recent-jobs API does not include pending, so it is possible we queried in between a workflow transition, and we're NOT really front of line. 292 | if [ $confidence -lt "$CONFIDENCE_THRESHOLD" ]; then 293 | # To grow confidence, we check again with a delay. 294 | confidence=$((confidence+1)) 295 | log "API shows no conflicting jobs/workflows. However it is possible a previous workflow has pending jobs not yet visible in API. To avoid a race condition we will verify our place in queue." 296 | log "Rerunning check ${confidence}/$CONFIDENCE_THRESHOLD" 297 | else 298 | log "Front of the line, WooHoo!, Build continuing" 299 | break 300 | fi 301 | else 302 | # If we fail, reset confidence 303 | confidence=0 304 | log "This build (${CIRCLE_BUILD_NUM}), pipeline (${MY_PIPELINE_NUMBER}) is queued, waiting for build(${oldest_running_build_num}) pipeline (${front_of_queue_pipeline_number}) to complete." 305 | log "Total Queue time: ${wait_time} seconds." 306 | fi 307 | 308 | if [ $wait_time -ge $max_time_seconds ]; then 309 | log "Max wait time exceeded. waited=${wait_time} max=${max_time_seconds}. Fail or force cancel..." 310 | if [ "${DONT_QUIT}" != "false" ]; then 311 | log "Orb parameter dont-quit is set to true, letting this job proceed!" 312 | if [ "${FORCE_CANCEL_PREVIOUS}" != "false" ]; then 313 | log "FEATURE NOT IMPLEMENTED" 314 | exit 1 315 | fi 316 | exit 0 317 | else 318 | if [ "$FAIL_INSTEAD_OF_CANCEL" != "true" ]; then 319 | cancel_build_num "$CIRCLE_BUILD_NUM" 320 | sleep 5 # wait for API to cancel this job, rather than showing as failure 321 | # but just in case, fail job 322 | fi 323 | exit 1 324 | fi 325 | fi 326 | 327 | sleep $loop_time 328 | done 329 | 330 | -------------------------------------------------------------------------------- /src/@orb.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | description: | 3 | DEPRECATED: 4 | 5 | Users should adopt CircleCI's native `serial-group` stanza instead. (See docs). 6 | 7 | 8 | Allows jobs or entire workflows to be queued to ensure they run in serial. 9 | This is ideal for deployments or other activities that must not run concurrently. 10 | May optionally consider branch-level isolation if unique branches should run concurrently. 11 | This orb requires the project to have an **Personal** API key in order to query build states. 12 | It requires a single environment variable CIRCLECI_API_KEY which can be created in account settings - https://circleci.com/account/api. 13 | 14 | 3.3.3/ 15 | 3.3.2: Goodbye world! 16 | Serialization is now native in CIrcleCI's serial-group stanza! 17 | <3 Thanks to all who made that happen, including every user of this orb. 18 | 3.3.1: Calculate wait time via wall clock, PR#140, thanks @jordan-brough 19 | 3.3.0: Add timestamps to output, PR#139, thanks @jordan-brough 20 | 3.2.1: Docs improvement, PR #136 Thanks @RainOfTerra 21 | 3.2.0: Adds back-off support to handle 429 throttling from new API. PR #134 Thanks @rainofterra 22 | 3.1.6: Fix #128 again, this time on JQ side. PR #133 Thanks @RainOfTerra 23 | 3.1.5: Fix #128: allow spaces in job names and other bash bugs in PR #131, thanks @RainOfTerra 24 | 3.1.4: Fix missing branch error when using tags. Issue #126 25 | 3.1.2: Fix github VCS switch (API calls failing) thanks @risinga PR #122 26 | 3.1.1: Boolean fix finally isolated, thanks @@charlescqian 27 | 3.1.0: Force fail, not cancel, when blocked. PR #113, thanks @pguinard-public-com 28 | 3.0.0: [BREAKING CHANGE] Use pipeline.number as authoritative comparison. Deep thanks to @PChambino 29 | 2.2.2: Docs clarity on token needs (@davidjb) 30 | 2.2.1: fixes release version bug 31 | 2.2.0: Adds 'filter-by-workflow' (@soniqua) 32 | 2.0.0: Breaking change fixes dyanamic config, but may break Bitbucket users. 33 | 1.12.0: Adds Server Support (@nanophate) 34 | 1.9.0: Doc update 35 | 1.8.4: Adds urlencode for branch names. (@andrew-barnett) 36 | 1.8.1: Adds content-type header to API calls (@kevinr-electric) and prints message on error (@AlexMeuer) 37 | 1.8.0: minor fix same as version 1.8.0 (missing docs) 38 | 1.7.1: patch fix same as version 1.8.1 to catch folsk who dont update 39 | 1.7.0: adds regexp for job names, collab with @jonesie100 40 | 1.6.5: docs contribution by @pwilczynskiclearcode 41 | 1.6.4: support slashes in Tags, thanks @dunial 42 | 1.6.3: addresses API changes that broke branch-job queueing, adds more API checks 43 | 1.6.1: fixes issue in tag matching , thanks @calvin-summer 44 | 1.6.0: Support Tags, thanks @nikolaik, @dunial 45 | 1.5.0: API variables name as parameter , thanks @philnielson 46 | 1.4.4: Docs improvements, thanks @jordan-brough 47 | 1.4.3: more confident confidence thanks @GreshamDanielStephens 48 | 1.4.2: Doc improvements, thanks @olleolleolle 49 | 1.4.1: fixes bug in block-workflow as job. thanks @mu-bro 50 | 1.4.0: Adds confidence checks to avoid race condition 51 | 1.3.0: use small resource class in job 52 | 53 | 54 | 55 | 56 | display: 57 | home_url: https://eddiewebb.github.io/circleci-queue/ 58 | source_url: https://github.com/eddiewebb/circleci-queue 59 | 60 | examples: 61 | queue_workflow: 62 | description: Used typically as first job and will queue until no previous workflows are running 63 | usage: 64 | version: 2.1 65 | orbs: 66 | queue: eddiewebb/queue@volatile 67 | 68 | workflows: 69 | build_deploy: 70 | jobs: 71 | - queue/block_workflow: 72 | my-pipeline: <> # Required due to orb processing flow 73 | max-wait-time: "10" # max wait, in minutes (default 10) 74 | limit-branch-name: main # restrict queueing to a specific branch (default *) 75 | - some_other_job: 76 | requires: 77 | - queue/block_workflow 78 | 79 | single_concurrency_job: 80 | description: | 81 | Used to ensure that a only single job (deploy) is not run concurrently. 82 | By default will only queue if the same job from previous worfklows is running on the same branch. 83 | This allows safe jobs like build/test to overlap, minimizing overall queue times. 84 | usage: 85 | version: 2.1 86 | orbs: 87 | queue: eddiewebb/queue@volatile 88 | 89 | workflows: 90 | build_deploy: 91 | jobs: 92 | - build 93 | - deploy: 94 | requires: 95 | - build 96 | jobs: 97 | build: 98 | docker: 99 | - image: circleci/node:10 100 | steps: 101 | - run: echo "This job can overlap" 102 | 103 | deploy: 104 | docker: 105 | - image: circleci/node:10 106 | steps: 107 | - queue/until_front_of_line: 108 | my-pipeline: <> # Required due to orb processing flow 109 | max-wait-time: "10" # max wait, in minutes (default 10) 110 | limit-branch-name: main # restrict queueing to a specific branch (default *) 111 | - run: echo "This job will not overlap" 112 | 113 | multiple_job_names_regexp: 114 | description: Use regexp-jobname when you have multiple jobs to block order of. 115 | usage: 116 | version: 2.1 117 | orbs: 118 | queue: eddiewebb/queue@volatile 119 | 120 | workflows: 121 | build_deploy: 122 | jobs: 123 | - build 124 | - DeployStep1: 125 | requires: 126 | - build 127 | - DeployStep2: 128 | requires: 129 | - DeployStep1 130 | jobs: 131 | build: 132 | docker: 133 | - image: circleci/node:10 134 | steps: 135 | - run: echo "This job can overlap" 136 | 137 | DeployStep1: 138 | docker: 139 | - image: circleci/node:10 140 | steps: 141 | - queue/until_front_of_line: 142 | my-pipeline: <> # Required due to orb processing flow 143 | max-wait-time: "10" # max wait, in minutes (default 10) 144 | limit-branch-name: main # restrict queueing to a specific branch (default *) 145 | job-regex: "^DeployStep[0-9]$" #use extendex regexp pattern 146 | - run: echo "This job will not overlap with itself or next similar nameds job" 147 | 148 | DeployStep2: 149 | docker: 150 | - image: circleci/node:10 151 | steps: 152 | - run: echo "This job will block step1 on any further workflows" 153 | 154 | -------------------------------------------------------------------------------- /src/commands/until_front_of_line.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | this-branch-only: 3 | type: boolean 4 | default: true 5 | description: "Should we only consider jobs running on the same branch?" 6 | block-workflow: 7 | type: boolean 8 | # this is false at COMMAND level as intention is to only block CURRENT job. 9 | default: false 10 | description: "If true, this job will block until no other workflows with ANY JOBS with an earlier timestamp are running. Typically used as first job." 11 | max-wait-time: 12 | type: string 13 | default: "10" 14 | description: "How many minutes to wait before giving up." 15 | dont-quit: 16 | type: boolean 17 | default: false 18 | description: "Quitting is for losers. Force job through once time expires instead of failing." 19 | fail-instead-of-cancel: 20 | type: boolean 21 | default: false 22 | description: "Fail this command instead of canceling." 23 | force-cancel-previous: 24 | type: boolean 25 | default: false 26 | description: "No Mercy. Issue cancel commands for any previous competitors (only applies when dont-quit also true)" 27 | limit-branch-name: 28 | type: string 29 | default: "*" 30 | description: "Only apply queue logic on specified branch. " 31 | limit-workflow-name: 32 | type: string 33 | default: "*" 34 | description: "Only queue on a specified workflow. Consider combining this with `this-branch-only`:`false`." 35 | # vcs-type --> pipeline.project.type 36 | confidence: 37 | type: string 38 | default: "1" 39 | description: "Due to scarce API, we need to requery the recent jobs list to ensure we're not just in a pending state for previous jobs. This number indicates the threhold for API returning no previous pending jobs. Default is a single confirmation." 40 | circleci-api-key: 41 | type: env_var_name 42 | default: CIRCLECI_API_KEY 43 | description: "In case you use a different Environment Variable Name than CIRCLECI_API_KEY, supply it here." 44 | tag-pattern: 45 | type: string 46 | default: "" 47 | description: "Set to queue jobs using a regex pattern f.ex '^v[0-9]+\\.[0-9]+\\.[0-9]+$' to filter CIRCLECI_TAG" 48 | job-regex: 49 | type: string 50 | default: "" 51 | description: "Used to selectively block individual jobs in a workflow. ex '^deploy*'" 52 | circleci-hostname: 53 | type: string 54 | default: "circleci.com" 55 | description: "For server user to specifiy custom hostname for their server" 56 | my-pipeline: 57 | type: integer 58 | include-debug: 59 | type: boolean 60 | default: false 61 | 62 | 63 | steps: 64 | - run: 65 | name: Queueing Orb Deprecation Notice - !! IMPORTANT !! 66 | command: <> 67 | shell: bash 68 | - run: 69 | name: 'Queue - Import Parameters' 70 | command: | 71 | echo "export BLOCK_WORKFLOW=<>" >> $BASH_ENV 72 | echo "export CCI_API_KEY_NAME=<< parameters.circleci-api-key >>" >> $BASH_ENV 73 | echo "export CIRCLECI_BASE_URL=https://<>" >> $BASH_ENV 74 | echo "export CONFIDENCE_THRESHOLD=<>" >> $BASH_ENV 75 | echo "export DEBUG=<>" >> $BASH_ENV 76 | echo "export DONT_QUIT=<>" >> $BASH_ENV 77 | echo "export FAIL_INSTEAD_OF_CANCEL=<< parameters.fail-instead-of-cancel >>" >> $BASH_ENV 78 | echo "export FILTER_BRANCH=<< parameters.this-branch-only >>" >> $BASH_ENV 79 | echo "export FORCE_CANCEL_PREVIOUS=<>" >> $BASH_ENV 80 | echo "export JOB_REGEXP=\"<>\"" >> $BASH_ENV 81 | echo "export MAX_TIME='<>'" >> $BASH_ENV 82 | echo "export MY_PIPELINE_NUMBER=<>" >> $BASH_ENV 83 | echo "export ONLY_ON_BRANCH=<>" >> $BASH_ENV 84 | echo "export ONLY_ON_WORKFLOW=<>" >> $BASH_ENV 85 | echo "export TAG_PATTERN=\"<>\"" >> $BASH_ENV 86 | 87 | - run: 88 | name: Queue Until Front of Line 89 | command: <> 90 | shell: bash 91 | -------------------------------------------------------------------------------- /src/jobs/block_workflow.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | this-branch-only: 3 | type: boolean 4 | default: true 5 | description: "Should we only consider jobs running on the same branch?" 6 | block-workflow: 7 | type: boolean 8 | # this is true at JOB level as intention is to block workflow 9 | default: true 10 | description: "If true, this job will block until no other workflows with an earlier timestamp are running. Typically used as first job." 11 | max-wait-time: 12 | type: string 13 | default: "10" 14 | description: "How many minutes to wait before giving up." 15 | dont-quit: 16 | type: boolean 17 | default: false 18 | description: "Quitting is for losers. Force job through once time expires instead of failing." 19 | fail-instead-of-cancel: 20 | type: boolean 21 | default: false 22 | description: "Fail this command instead of canceling." 23 | force-cancel-previous: 24 | type: boolean 25 | default: false 26 | description: "No Mercy. Issue cancel commands for any previous competitors (only applies when dont-quit also true)" 27 | limit-branch-name: 28 | type: string 29 | default: "*" 30 | description: "Only apply queue logic on specified branch. " 31 | limit-workflow-name: 32 | type: string 33 | default: "*" 34 | description: "Only queue on a specified workflow. Consider combining this with `this-branch-only`:`false`." 35 | # vcs-type --> pipeline.project.type 36 | confidence: 37 | type: string 38 | default: "1" 39 | description: "Due to scarce API, we need to requery the recent jobs list to ensure we're not just in a pending state for previous jobs. This number indicates the threhold for API returning no previous pending jobs. Default is a single confirmation." 40 | circleci-api-key: 41 | type: env_var_name 42 | default: CIRCLECI_API_KEY 43 | description: "In case you use a different Environment Variable Name than CIRCLECI_API_KEY, supply it here." 44 | tag-pattern: 45 | type: string 46 | default: "" 47 | description: "Set to queue jobs using a regex pattern f.ex '^v[0-9]+\\.[0-9]+\\.[0-9]+$' to filter CIRCLECI_TAG" 48 | job-regex: 49 | type: string 50 | default: "" 51 | description: "Used to selectively block individual jobs in a workflow. ex '^deploy*'" 52 | circleci-hostname: 53 | type: string 54 | default: "circleci.com" 55 | description: "For server user to specifiy custom hostname for their server" 56 | my-pipeline: 57 | type: integer 58 | include-debug: 59 | type: boolean 60 | default: false 61 | 62 | 63 | docker: 64 | - image: cimg/base:stable 65 | resource_class: small 66 | steps: 67 | - until_front_of_line: 68 | block-workflow: <> 69 | circleci-api-key: <> 70 | circleci-hostname: <> 71 | confidence: <> 72 | dont-quit: <> 73 | fail-instead-of-cancel: << parameters.fail-instead-of-cancel >> 74 | include-debug: <> 75 | job-regex: <> 76 | limit-branch-name: <> 77 | limit-workflow-name: <> 78 | max-wait-time: <> 79 | my-pipeline: <> 80 | tag-pattern: <> 81 | this-branch-only: <> 82 | -------------------------------------------------------------------------------- /test/api/jobs/nopreviousjobs.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "testing-note":"This represents the current build, with a job that should be serial", 5 | "build_num" : 3, 6 | "committer_date": "2019-01-17T11:17:24-05:00", 7 | "branch" : "master", 8 | "build_parameters":{ 9 | }, 10 | "workflows" : { 11 | "job_name" : "singlejob", 12 | "job_id" : "random-hash-2", 13 | "workflow_id" : "second_workflow", 14 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 15 | "upstream_job_ids" : [ ], 16 | "upstream_concurrency_map" : { }, 17 | "workflow_name" : "build-deploy" 18 | }, 19 | "status" : "running" 20 | } 21 | 22 | 23 | ] 24 | -------------------------------------------------------------------------------- /test/api/jobs/onepreviousjob-differentname.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "testing-note":"This represents the current build, with a job that should be serial", 5 | "build_num" : 3, 6 | "committer_date": "2019-01-17T11:17:24-05:00", 7 | "branch" : "master", 8 | "build_parameters":{ 9 | }, 10 | "workflows" : { 11 | "job_name" : "singlejob", 12 | "job_id" : "random-hash-2", 13 | "workflow_id" : "second_workflow", 14 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 15 | "upstream_job_ids" : [ ], 16 | "upstream_concurrency_map" : { }, 17 | "workflow_name" : "build-deploy" 18 | }, 19 | "status" : "running" 20 | }, 21 | 22 | { 23 | "testing-note":"This represents a previous deploy job with max concurreny of 1, Delete this block while local testing is queued to mimic previous build completing", 24 | "build_num" : 2, 25 | "committer_date": "2019-01-17T10:17:24-05:00", 26 | "branch" : "master", 27 | "build_parameters":{ 28 | }, 29 | "workflows" : { 30 | "job_name" : "adifferentjob", 31 | "job_id" : "random-hash-2", 32 | "workflow_id" : "first_workflow", 33 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 34 | "upstream_job_ids" : [ ], 35 | "upstream_concurrency_map" : { }, 36 | "workflow_name" : "build-deploy" 37 | }, 38 | "status" : "running" 39 | } 40 | 41 | 42 | ] 43 | -------------------------------------------------------------------------------- /test/api/jobs/onepreviousjobsamename.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "testing-note":"This represents the current build, with a job that should be serial", 5 | "build_num" : 3, 6 | "committer_date": "2019-01-17T11:17:24-05:00", 7 | "branch" : "master", 8 | "build_parameters":{ 9 | }, 10 | "workflows" : { 11 | "job_name" : "singlejob", 12 | "job_id" : "random-hash-2", 13 | "workflow_id" : "second_workflow", 14 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 15 | "upstream_job_ids" : [ ], 16 | "upstream_concurrency_map" : { }, 17 | "workflow_name" : "build-deploy" 18 | }, 19 | "status" : "running" 20 | }, 21 | 22 | { 23 | "testing-note":"This represents a previous deploy job with max concurreny of 1, Delete this block while local testing is queued to mimic previous build completing", 24 | "build_num" : 2, 25 | "committer_date": "2019-01-17T10:17:24-05:00", 26 | "branch" : "master", 27 | "build_parameters":{ 28 | }, 29 | "workflows" : { 30 | "job_name" : "singlejob", 31 | "job_id" : "random-hash-2", 32 | "workflow_id" : "first_workflow", 33 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 34 | "upstream_job_ids" : [ ], 35 | "upstream_concurrency_map" : { }, 36 | "workflow_name" : "build-deploy" 37 | }, 38 | "status" : "running" 39 | } 40 | 41 | 42 | ] 43 | -------------------------------------------------------------------------------- /test/api/jobs/regex-matches.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "testing-note":"This represents the current build, with a job that should be serial", 5 | "build_num" : 3, 6 | "committer_date": "2019-01-17T11:17:24-05:00", 7 | "branch" : "main", 8 | "build_parameters":{ 9 | }, 10 | "workflows" : { 11 | "job_name" : "DeployStep1", 12 | "job_id" : "random-hash-2", 13 | "workflow_id" : "second_workflow", 14 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 15 | "upstream_job_ids" : [ ], 16 | "upstream_concurrency_map" : { }, 17 | "workflow_name" : "build-deploy" 18 | }, 19 | "status" : "running" 20 | }, 21 | 22 | { 23 | "testing-note":"This represents a previous deploy job with max concurreny of 1, Delete this block while local testing is queued to mimic previous build completing", 24 | "build_num" : 2, 25 | "committer_date": "2019-01-17T10:17:24-05:00", 26 | "branch" : "main", 27 | "build_parameters":{ 28 | }, 29 | "workflows" : { 30 | "job_name" : "DeployStep2", 31 | "job_id" : "random-hash-3", 32 | "workflow_id" : "first_workflow", 33 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 34 | "upstream_job_ids" : [ ], 35 | "upstream_concurrency_map" : { }, 36 | "workflow_name" : "build-deploy" 37 | }, 38 | "status" : "running" 39 | } 40 | , 41 | 42 | { 43 | "testing-note":"This represents a previous deploy job with max concurreny of 1, Delete this block while local testing is queued to mimic previous build completing", 44 | "build_num" : 1, 45 | "committer_date": "2019-01-17T10:17:25-05:00", 46 | "branch" : "main", 47 | "build_parameters":{ 48 | }, 49 | "workflows" : { 50 | "job_name" : "SafeBuildJob", 51 | "job_id" : "random-hash-4", 52 | "workflow_id" : "first_workflow", 53 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 54 | "upstream_job_ids" : [ ], 55 | "upstream_concurrency_map" : { }, 56 | "workflow_name" : "build-deploy" 57 | }, 58 | "status" : "running" 59 | } 60 | 61 | 62 | ] 63 | -------------------------------------------------------------------------------- /test/api/jobs/regex-no-matches.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "testing-note":"This represents the current build, with a job that should be serial", 5 | "build_num" : 3, 6 | "committer_date": "2019-01-17T11:17:24-05:00", 7 | "branch" : "main", 8 | "build_parameters":{ 9 | }, 10 | "workflows" : { 11 | "job_name" : "DeployStep1", 12 | "job_id" : "random-hash-2", 13 | "workflow_id" : "second_workflow", 14 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 15 | "upstream_job_ids" : [ ], 16 | "upstream_concurrency_map" : { }, 17 | "workflow_name" : "build-deploy" 18 | }, 19 | "status" : "running" 20 | }, 21 | 22 | { 23 | "testing-note":"This represents a previous job bu diff name", 24 | "build_num" : 2, 25 | "committer_date": "2019-01-17T10:17:24-05:00", 26 | "branch" : "main", 27 | "build_parameters":{ 28 | }, 29 | "workflows" : { 30 | "job_name" : "RandomStep2", 31 | "job_id" : "random-hash-3", 32 | "workflow_id" : "first_workflow", 33 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 34 | "upstream_job_ids" : [ ], 35 | "upstream_concurrency_map" : { }, 36 | "workflow_name" : "build-deploy" 37 | }, 38 | "status" : "running" 39 | } 40 | , 41 | 42 | { 43 | "testing-note":"This represents a previous deploy job with max concurreny of 1, Delete this block while local testing is queued to mimic previous build completing", 44 | "build_num" : 1, 45 | "committer_date": "2019-01-17T10:17:25-05:00", 46 | "branch" : "main", 47 | "build_parameters":{ 48 | }, 49 | "workflows" : { 50 | "job_name" : "SafeBuildJob", 51 | "job_id" : "random-hash-4", 52 | "workflow_id" : "first_workflow", 53 | "workspace_id" : "a1892dee-6e9e-4f2e-adbc-a5629a97b483", 54 | "upstream_job_ids" : [ ], 55 | "upstream_concurrency_map" : { }, 56 | "workflow_name" : "build-deploy" 57 | }, 58 | "status" : "running" 59 | } 60 | 61 | 62 | ] 63 | -------------------------------------------------------------------------------- /test/api/workflows/first_workflow.json: -------------------------------------------------------------------------------- 1 | { 2 | "stopped_at" : null, 3 | "name" : "build-deploy", 4 | "project_slug" : "test/project", 5 | "pipeline_number" : 1, 6 | "status" : "running" 7 | } 8 | -------------------------------------------------------------------------------- /test/api/workflows/second_workflow.json: -------------------------------------------------------------------------------- 1 | { 2 | "stopped_at" : null, 3 | "name" : "build-deploy", 4 | "project_slug" : "test/project", 5 | "pipeline_number" : 2, 6 | "status" : "running" 7 | } 8 | -------------------------------------------------------------------------------- /test/bats_helper.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | function process_config_with { 5 | append_project_configuration $1 > $INPUT_PROJECT_CONFIG 6 | circleci config process $INPUT_PROJECT_CONFIG > ${PROCESSED_PROJECT_CONFIG} 7 | yq eval -o=j ${PROCESSED_PROJECT_CONFIG} > ${JSON_PROJECT_CONFIG} 8 | 9 | #assertions use output, tests can override outptu to test additional commands beyond parsing. 10 | output=`cat ${PROCESSED_PROJECT_CONFIG}` 11 | } 12 | 13 | function append_project_configuration { 14 | if [ -z "$BATS_IMPORT_DEV_ORB" ]; then 15 | assemble_inline $1 16 | else 17 | assemble_external $1 18 | fi 19 | } 20 | 21 | # 22 | # USes circleci config pack, but indents everything under an `orbs.ORBNAME` element so it may be inlined. 23 | # 24 | function assemble_inline { 25 | CONFIG=$1 26 | echo "version: 2.1" 27 | echo "orbs:" 28 | echo " ${INLINE_ORB_NAME}:" 29 | circleci orb pack src | sed -e 's/^/ /' 30 | if [ -s $CONFIG ];then 31 | cat $CONFIG 32 | fi 33 | } 34 | 35 | 36 | # 37 | # Adds `orbs:` section referencing the provided dev orb 38 | # 39 | function assemble_external { 40 | CONFIG=$1 41 | echo "version: 2.1" 42 | echo "orbs:" 43 | echo " ${INLINE_ORB_NAME}: $BATS_IMPORT_DEV_ORB" 44 | if [ -s $CONFIG ];then 45 | cat $CONFIG 46 | fi 47 | } 48 | 49 | 50 | 51 | # 52 | # Add assertions for use in BATS tests 53 | # 54 | 55 | function assert_contains_text { 56 | TEXT=$1 57 | if [[ "$output" != *"${TEXT}"* ]]; then 58 | echo "Expected text \`$TEXT\`, not found in output (printed below)" 59 | echo $output 60 | return 1 61 | fi 62 | } 63 | 64 | function assert_text_not_found { 65 | TEXT=$1 66 | if [[ "$output" == *"${TEXT}"* ]]; then 67 | echo "Forbidden text \`$TEXT\`, was found in output.." 68 | echo $output 69 | return 1 70 | fi 71 | } 72 | 73 | function assert_matches_file { 74 | FILE=$1 75 | echo "${output}" | sed '/# Original config.yml file:/q' | sed '$d' | diff -B $FILE - 76 | return $? 77 | } 78 | 79 | function assert_jq_match { 80 | MATCH=$2 81 | RES=$(jq -r "$1" ${JSON_PROJECT_CONFIG}) 82 | if [[ "$RES" != "$MATCH" ]];then 83 | echo "Expected match "'"'"$MATCH"'"'" was not found in "'"'"$RES"'"' 84 | return 1 85 | fi 86 | } 87 | 88 | function assert_jq_contains { 89 | MATCH=$2 90 | RES=$(jq -r "$1" ${JSON_PROJECT_CONFIG}) 91 | if [[ "$RES" != *"$MATCH"* ]];then 92 | echo "Expected string "'"'"$MATCH"'"'" was not found in "'"'"$RES"'"' 93 | return 1 94 | fi 95 | } 96 | 97 | function load_config_parameters { 98 | NAME="${1:-build}" 99 | #echo $JSON_PROJECT_CONFIG > $ENV_STAGING_PATH 100 | jq -r '.jobs["'"${NAME}"'"].steps[1].run.command' $JSON_PROJECT_CONFIG > $ENV_STAGING_PATH-input 101 | >$ENV_STAGING_PATH 102 | export BASH_ENV=$ENV_STAGING_PATH 103 | export CIRCLE_BRANCH="main" 104 | export CIRCLE_BUILD_NUM=3 105 | export CIRCLE_PROJECT_USERNAME=eddie 106 | export CIRCLE_PROJECT_REPONAME=queue 107 | export CIRCLE_REPOSITORY_URL="https://github.com/somthh" 108 | export CIRCLE_JOB=singlejob 109 | bash "$ENV_STAGING_PATH-input" 110 | } 111 | 112 | -------------------------------------------------------------------------------- /test/inputs/command-anybranch.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | build: 3 | docker: 4 | - image: circleci/node:10 5 | working_directory: ~/repo 6 | steps: 7 | - queue/until_front_of_line: 8 | max-wait-time: "1/10" 9 | my-pipeline: 2 10 | include-debug: true 11 | this-branch-only: false 12 | -------------------------------------------------------------------------------- /test/inputs/command-defaults.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | build: 3 | docker: 4 | - image: circleci/node:10 5 | working_directory: ~/repo 6 | steps: 7 | - queue/until_front_of_line: 8 | max-wait-time: "1/10" 9 | my-pipeline: 2 10 | include-debug: true 11 | -------------------------------------------------------------------------------- /test/inputs/command-dont-quit.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | build: 3 | docker: 4 | - image: circleci/node:10 5 | working_directory: ~/repo 6 | steps: 7 | - queue/until_front_of_line: 8 | max-wait-time: "1/10" 9 | my-pipeline: 2 10 | include-debug: true 11 | dont-quit: true 12 | -------------------------------------------------------------------------------- /test/inputs/command-job-regex.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | build: 3 | docker: 4 | - image: circleci/node:10 5 | working_directory: ~/repo 6 | steps: 7 | - queue/until_front_of_line: 8 | max-wait-time: "1/10" 9 | limit-branch-name: "main" 10 | job-regex: "^DeployStep[0-9]$" 11 | my-pipeline: 2 12 | include-debug: true -------------------------------------------------------------------------------- /test/inputs/command-non-default.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | build: 3 | docker: 4 | - image: circleci/node:10 5 | working_directory: ~/repo 6 | steps: 7 | - queue/until_front_of_line: 8 | #all non default values to assert passthrough no typos, etc 9 | this-branch-only: false 10 | block-workflow: true 11 | max-wait-time: "1/10" 12 | dont-quit: true 13 | force-cancel-previous: true 14 | limit-branch-name: 'unique-branch-name' 15 | limit-workflow-name: 'unique-workflow-name' 16 | confidence: "100" 17 | circleci-api-key: 'ABC_123' 18 | tag-pattern: 'unique-tag-pattern' 19 | job-regex: 'unique-job-regex' 20 | circleci-hostname: 'unique-hostname' 21 | my-pipeline: 2 22 | -------------------------------------------------------------------------------- /test/inputs/fulljob-noblock.yml: -------------------------------------------------------------------------------- 1 | workflows: 2 | single_file: 3 | jobs: 4 | - queue/block_workflow: 5 | name: "Single File" 6 | max-wait-time: "1/10" 7 | my-pipeline: 2 8 | block-workflow: false #make sure false vlaues work! -------------------------------------------------------------------------------- /test/inputs/fulljob.yml: -------------------------------------------------------------------------------- 1 | workflows: 2 | single_file: 3 | jobs: 4 | - queue/block_workflow: 5 | name: "Single File" 6 | max-wait-time: "1/10" 7 | my-pipeline: 2 -------------------------------------------------------------------------------- /test/test_expansion.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # load custom assertions and functions 4 | load bats_helper 5 | 6 | # setup is run beofre each test 7 | function setup { 8 | INPUT_PROJECT_CONFIG=${BATS_TMPDIR}/input_config-${BATS_TEST_NUMBER} 9 | PROCESSED_PROJECT_CONFIG=${BATS_TMPDIR}/packed_config-${BATS_TEST_NUMBER} 10 | JSON_PROJECT_CONFIG=${BATS_TMPDIR}/json_config-${BATS_TEST_NUMBER} 11 | ENV_STAGING_PATH=${BATS_TMPDIR}/env-${BATS_TEST_NUMBER}.sh 12 | #echo "#using temp file ${BATS_TMPDIR}" 13 | BASH_ENV="$ENV_STAGING_PATH" 14 | 15 | # the name used in example config files. 16 | INLINE_ORB_NAME="queue" 17 | 18 | 19 | #if [ -z "$BATS_IMPORT_DEV_ORB" ]; then 20 | #echo "#Using \`inline\` orb assembly, to test against published orb, set BATS_IMPORT_DEV_ORB to fully qualified path" >&3 21 | #else 22 | #echo "#BATS_IMPORT_DEV_ORB env var is set, all config will be tested against imported orb $BATS_IMPORT_DEV_ORB" >&3 23 | #fi 24 | 25 | 26 | 27 | } 28 | 29 | @test "Bitbucket switch works properly based on VCS" { 30 | # given 31 | process_config_with test/inputs/command-defaults.yml 32 | export TESTING_MOCK_RESPONSE=test/api/jobs/nopreviousjobs.json 33 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 34 | 35 | load_config_parameters 36 | export CIRCLE_REPOSITORY_URL="git@bitbucket.org:deedee/deeedee.git" 37 | run bash scripts/loop.bash 38 | echo $output 39 | 40 | assert_contains_text "VCS_TYPE set to bitbucket" 41 | } 42 | 43 | @test "Non Bitbucket switch works properly based on VCS" { 44 | # given 45 | process_config_with test/inputs/command-defaults.yml 46 | export TESTING_MOCK_RESPONSE=test/api/jobs/nopreviousjobs.json 47 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 48 | 49 | load_config_parameters 50 | export CIRCLE_REPOSITORY_URL="git@bebop.org:deedee/deeedee.git" 51 | run bash scripts/loop.bash 52 | echo $output 53 | 54 | assert_text_not_found "VCS_TYPE set to bitbucket" 55 | } 56 | 57 | @test "Default job sets block workflow properly" { 58 | # given 59 | export TESTING_MOCK_RESPONSE=test/api/jobs/onepreviousjob-differentname.json 60 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 61 | process_config_with test/inputs/fulljob.yml 62 | 63 | # when 64 | assert_jq_match '.jobs | length' 1 #only 1 job 65 | assert_jq_match '.jobs["Single File"].steps | length' 3 #only 1 steps 66 | 67 | 68 | export CIRCLE_BRANCH="main" 69 | load_config_parameters "Single File" 70 | run bash scripts/loop.bash 71 | echo $output 72 | 73 | 74 | assert_contains_text "Orb parameter block-workflow is true." 75 | } 76 | 77 | @test "Default job sets can NOT block workflow if configured" { 78 | # given 79 | export TESTING_MOCK_RESPONSE=test/api/jobs/onepreviousjob-differentname.json 80 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 81 | process_config_with test/inputs/fulljob-noblock.yml 82 | 83 | # when 84 | assert_jq_match '.jobs | length' 1 #only 1 job 85 | assert_jq_match '.jobs["Single File"].steps | length' 3 #only 1 steps 86 | 87 | 88 | export CIRCLE_BRANCH="main" 89 | load_config_parameters "Single File" 90 | run bash scripts/loop.bash 91 | echo $output 92 | 93 | 94 | assert_contains_text "Orb parameter block-workflow is false" 95 | } 96 | 97 | 98 | @test "Command: script will WAIT with previous job of similar name used in regexp" { 99 | # given 100 | 101 | process_config_with test/inputs/command-job-regex.yml 102 | # load any parameters provided as envars. 103 | 104 | export CIRCLE_BRANCH="main" 105 | load_config_parameters 106 | export TESTING_MOCK_RESPONSE=test/api/jobs/regex-matches.json 107 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 108 | export CIRCLE_JOB=DeployStep1 109 | run bash scripts/loop.bash 110 | echo $output 111 | 112 | assert_contains_text "Max Queue Time: 6 seconds" 113 | assert_contains_text "Max wait time exceeded" 114 | assert_contains_text "Cancelling build 3" 115 | [[ "$status" == "1" ]] 116 | } 117 | 118 | 119 | @test "Command: script will NOT WAIT with previous job of non matching names when using regexp" { 120 | # given 121 | process_config_with test/inputs/command-job-regex.yml 122 | export TESTING_MOCK_RESPONSE=test/api/jobs/regex-no-matches.json 123 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 124 | 125 | 126 | export CIRCLE_BRANCH="main" 127 | load_config_parameters 128 | export CIRCLE_JOB="DeployStep1" 129 | run bash scripts/loop.bash 130 | echo $output 131 | 132 | 133 | assert_contains_text "Max Queue Time: 6 seconds" 134 | assert_text_not_found "Max wait time exceeded" 135 | assert_contains_text "Front of the line, WooHoo!, Build continuing" 136 | [[ "$status" == "0" ]] 137 | } 138 | 139 | 140 | 141 | @test "Command: script will proceed with no previous jobs" { 142 | # given 143 | process_config_with test/inputs/command-defaults.yml 144 | export TESTING_MOCK_RESPONSE=test/api/jobs/nopreviousjobs.json 145 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 146 | 147 | 148 | load_config_parameters 149 | run bash scripts/loop.bash 150 | echo $output 151 | 152 | 153 | assert_contains_text "Max Queue Time: 6 seconds" 154 | assert_text_not_found "Max wait time exceeded" 155 | assert_contains_text "Front of the line, WooHoo!, Build continuing" 156 | [[ "$status" == "0" ]] 157 | 158 | } 159 | 160 | @test "Command: script will proceed with previous job of different name" { 161 | # given 162 | process_config_with test/inputs/command-defaults.yml 163 | export TESTING_MOCK_RESPONSE=test/api/jobs/onepreviousjob-differentname.json 164 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 165 | 166 | 167 | export CIRCLE_BRANCH="main" 168 | load_config_parameters 169 | run bash scripts/loop.bash 170 | echo $output 171 | 172 | 173 | assert_contains_text "Max Queue Time: 6 seconds" 174 | assert_contains_text "Front of the line, WooHoo!, Build continuing" 175 | [[ "$status" == "0" ]] 176 | } 177 | 178 | @test "Command: script will WAIT with previous job of same name" { 179 | # given 180 | process_config_with test/inputs/command-defaults.yml 181 | export TESTING_MOCK_RESPONSE=test/api/jobs/onepreviousjobsamename.json 182 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 183 | 184 | 185 | export CIRCLE_BRANCH="main" 186 | load_config_parameters 187 | run bash scripts/loop.bash 188 | echo $output 189 | 190 | 191 | assert_contains_text "Max Queue Time: 6 seconds" 192 | assert_contains_text "Max wait time exceeded" 193 | assert_contains_text "Cancelling build 3" 194 | [[ "$status" == "1" ]] 195 | } 196 | 197 | 198 | @test "Command: script with dont-quit will not fail current job" { 199 | # given 200 | process_config_with test/inputs/command-dont-quit.yml 201 | export TESTING_MOCK_RESPONSE=test/api/jobs/onepreviousjobsamename.json 202 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 203 | 204 | 205 | export CIRCLE_BRANCH="main" 206 | load_config_parameters 207 | run bash scripts/loop.bash 208 | echo $output 209 | 210 | 211 | assert_contains_text "Max Queue Time: 6 seconds" 212 | assert_contains_text "Max wait time exceeded" 213 | assert_contains_text "Orb parameter dont-quit is set to true, letting this job proceed!" 214 | [[ "$status" == "0" ]] 215 | } 216 | 217 | @test "Command: script will NOT consider branch" { 218 | # given 219 | process_config_with test/inputs/command-anybranch.yml 220 | export TESTING_MOCK_RESPONSE=test/api/jobs/nopreviousjobs.json 221 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 222 | 223 | # when 224 | 225 | export CIRCLE_BRANCH="main" 226 | load_config_parameters 227 | run bash scripts/loop.bash 228 | echo $output 229 | 230 | assert_contains_text "Max Queue Time: 6 seconds" 231 | assert_contains_text "Orb parameter 'this-branch-only' is false, will block previous builds on any branch" 232 | assert_contains_text "Front of the line, WooHoo!, Build continuing" 233 | [[ "$status" == "0" ]] 234 | 235 | } 236 | 237 | 238 | @test "Command: script will consider branch default" { 239 | # given 240 | process_config_with test/inputs/command-defaults.yml 241 | export TESTING_MOCK_RESPONSE=test/api/jobs/nopreviousjobs.json #branch filtering handled by API, so return no matching builds 242 | export TESTING_MOCK_WORKFLOW_RESPONSES=test/api/workflows 243 | 244 | # when 245 | assert_jq_match '.jobs | length' 1 #only 1 job 246 | assert_jq_match '.jobs["build"].steps | length' 3 #only 1 steps 247 | 248 | 249 | export CIRCLE_BRANCH="main" 250 | load_config_parameters 251 | run bash scripts/loop.bash 252 | echo $output 253 | 254 | 255 | assert_contains_text "${CIRCLE_BRANCH} matches queueable branch names" 256 | assert_contains_text "Max Queue Time: 6 seconds" 257 | assert_contains_text "Only blocking execution if running previous jobs on branch: ${CIRCLE_BRANCH}" 258 | assert_contains_text "Front of the line, WooHoo!, Build continuing" 259 | [[ "$status" == "0" ]] 260 | 261 | } 262 | 263 | 264 | 265 | 266 | @test "Command: script will skip queueing on forks" { 267 | # given 268 | process_config_with test/inputs/command-defaults.yml 269 | 270 | # when 271 | assert_jq_match '.jobs | length' 1 #only 1 job 272 | assert_jq_match '.jobs["build"].steps | length' 3 #only 1 steps 273 | 274 | 275 | export CIRCLE_BRANCH="main" 276 | load_config_parameters 277 | export CIRCLE_PR_REPONAME="this/was/forked" 278 | export TRIGGER_SOURCE="1" 279 | run bash scripts/loop.bash 280 | echo $output 281 | 282 | 283 | assert_contains_text "Queueing on forks is not supported. Skipping queue..." 284 | 285 | } 286 | 287 | --------------------------------------------------------------------------------