├── .github └── workflows │ └── validate.yml ├── .prettierrc ├── LICENSE ├── README.md ├── changelog.md ├── docker-compose.yml ├── hooks ├── lib │ ├── ecr-registry-provider.bash │ ├── gcr-registry-provider.bash │ ├── stdlib.bash │ └── stub-registry-provider.bash └── pre-command ├── plugin.yml └── tests ├── ecr-registry-provider.bats ├── gcr-registry-provider.bats ├── pre-command.bats └── stdlib.bats /.github/workflows/validate.yml: -------------------------------------------------------------------------------- 1 | name: Validate 2 | 3 | on: 4 | - pull_request 5 | - push 6 | 7 | permissions: {} 8 | 9 | jobs: 10 | core: 11 | name: Lint & Test 12 | permissions: 13 | checks: write 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v3 18 | 19 | - name: Test 20 | run: docker compose run --rm tests 21 | 22 | - name: Lint 23 | run: docker compose run --rm lint 24 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true 3 | } 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The Clear BSD License 2 | 3 | Copyright (c) 2018 SEEK Pty Ltd 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted (subject to the limitations in the disclaimer 8 | below) provided that the following conditions are met: 9 | 10 | * Redistributions of source code must retain the above copyright notice, 11 | this list of conditions and the following disclaimer. 12 | 13 | * Redistributions in binary form must reproduce the above copyright 14 | notice, this list of conditions and the following disclaimer in the 15 | documentation and/or other materials provided with the distribution. 16 | 17 | * Neither the name of the copyright holder nor the names of its 18 | contributors may be used to endorse or promote products derived from this 19 | software without specific prior written permission. 20 | 21 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY 22 | THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 23 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 25 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 26 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 27 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 28 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 29 | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 30 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 | POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker ECR Cache Buildkite Plugin 2 | 3 | [![GitHub Release](https://img.shields.io/github/release/seek-oss/docker-ecr-cache-buildkite-plugin.svg)](https://github.com/seek-oss/docker-ecr-cache-buildkite-plugin/releases) 4 | 5 | A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) to cache 6 | Docker images in Amazon ECR or Google Container Registry. 7 | 8 | This allows you to define a Dockerfile for your build-time dependencies without 9 | worrying about the time it takes to build the image. It allows you to re-use 10 | entire Docker images without worrying about layer caching, and/or pruning layers 11 | as changes are made to your containers. 12 | 13 | An ECR repository to store the built Docker image will be created for you, if 14 | one doesn't already exist. 15 | 16 | ## Example 17 | 18 | ### Basic usage 19 | 20 | ```dockerfile 21 | FROM bash 22 | 23 | RUN echo 'my expensive build step' 24 | ``` 25 | 26 | ```yaml 27 | steps: 28 | - command: echo wow 29 | plugins: 30 | - seek-oss/docker-ecr-cache#v2.2.1 31 | - docker#v5.10.0 32 | ``` 33 | 34 | ### Caching npm packages 35 | 36 | This plugin can be used to effectively cache `node_modules` between builds 37 | without worrying about Docker layer cache invalidation. You do this by hinting 38 | when the image should be re-built. 39 | 40 | ```dockerfile 41 | FROM node:20-alpine 42 | 43 | WORKDIR /workdir 44 | 45 | COPY package.json pnpm-lock.yaml /workdir 46 | 47 | # this step downloads the internet 48 | RUN pnpm install 49 | ``` 50 | 51 | ```yaml 52 | steps: 53 | - command: pnpm test 54 | plugins: 55 | - seek-oss/docker-ecr-cache#v2.2.1: 56 | cache-on: 57 | - package.json # avoid cache hits on stale lockfiles 58 | - pnpm-lock.yaml 59 | - docker#v5.10.0: 60 | volumes: 61 | - /workdir/node_modules 62 | ``` 63 | 64 | The `cache-on` property also supports Bash globbing with `globstar`: 65 | 66 | ```yaml 67 | steps: 68 | - command: pnpm test 69 | plugins: 70 | - seek-oss/docker-ecr-cache#v2.2.1: 71 | cache-on: 72 | - '**/package.json' # monorepo with multiple manifest files 73 | - pnpm-lock.yaml 74 | - docker#v5.10.0: 75 | volumes: 76 | - /workdir/node_modules 77 | ``` 78 | 79 | It also supports caching on specific JSON keys which can be specified following a `#` character using [jq syntax](https://jqlang.github.io/jq/manual/#object-identifier-index). This requires [jq](https://jqlang.github.io/jq/) to be installed on the build agent. This implementation works by matching on the first `.json#` substring. 80 | 81 | A given entry cannot contain both bash globbing and a jq path. 82 | 83 | ```yaml 84 | steps: 85 | - command: pnpm test 86 | plugins: 87 | - seek-oss/docker-ecr-cache#v2.2.1: 88 | cache-on: 89 | - .npmrc 90 | - package.json#.dependencies 91 | - package.json#.devDependencies 92 | - package.json#.packageManager 93 | - package.json#.pnpm.overrides 94 | - pnpm-lock.yaml 95 | - docker#v5.10.0: 96 | volumes: 97 | - /workdir/node_modules 98 | ``` 99 | 100 | ### Using another Dockerfile 101 | 102 | It's possible to specify the Dockerfile to use by: 103 | 104 | ```yaml 105 | steps: 106 | - command: echo wow 107 | plugins: 108 | - seek-oss/docker-ecr-cache#v2.2.1: 109 | dockerfile: my-dockerfile 110 | - docker#v5.10.0 111 | ``` 112 | 113 | Alternatively, Dockerfile can be embedded inline: 114 | 115 | ```yaml 116 | steps: 117 | - command: echo wow 118 | plugins: 119 | - seek-oss/docker-ecr-cache#v2.2.1: 120 | dockerfile-inline: | 121 | FROM node:20-alpine 122 | WORKDIR /workdir 123 | COPY package.json pnpm-lock.yaml /workdir 124 | RUN pnpm install 125 | 126 | - docker#v5.10.0 127 | ``` 128 | 129 | ### Building on the resulting image 130 | 131 | The resulting image are exported as environment variables: 132 | 133 | - `BUILDKITE_PLUGIN_DOCKER_IMAGE` (or whatever is specified per [changing the name of exported variable](#changing-the-name-of-exported-variable)) for the combined `image:tag` value 134 | - `BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE` for the `image` by itself 135 | - `BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG` for the `tag` by itself 136 | 137 | These variables can be used by subsequent plugins and commands in the same build step. 138 | For example, you may have a command that propagates these variables to another Docker build command: 139 | 140 | ```yaml 141 | steps: 142 | - command: >- 143 | docker build 144 | --build-arg BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE 145 | --build-arg BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG 146 | --file Dockerfile.secondary 147 | plugins: 148 | - seek-oss/docker-ecr-cache#v2.2.1 149 | ``` 150 | 151 | Your `Dockerfile.secondary` can then [dynamically use these args](https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact): 152 | 153 | ```dockerfile 154 | ARG BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE 155 | ARG BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG 156 | 157 | FROM ${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE}:${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG} 158 | 159 | RUN echo wow 160 | ``` 161 | 162 | ### Specifying a target step 163 | 164 | A [multi-stage Docker build] can be used to reduce an application container to 165 | just its runtime dependencies. However, this stripped down container may not 166 | have the environment necessary for running CI commands such as tests or linting. 167 | Instead, the `target` property can be used to specify an intermediate build 168 | stage to run commands against: 169 | 170 | [multi-stage docker build]: https://docs.docker.com/develop/develop-images/multistage-build/ 171 | 172 | ```yaml 173 | steps: 174 | - command: cargo test 175 | plugins: 176 | - seek-oss/docker-ecr-cache#v2.2.1: 177 | target: build-deps 178 | - docker#v5.10.0 179 | ``` 180 | 181 | ### Specifying build context 182 | 183 | The subdirectory containing the Dockerfile is the path used for the build's context by default. 184 | 185 | The `context` property can be used to specify a different path. 186 | 187 | ```yaml 188 | steps: 189 | - command: cargo test 190 | plugins: 191 | - seek-oss/docker-ecr-cache#v2.2.1: 192 | dockerfile: dockerfiles/test/Dockerfile 193 | context: '.' 194 | - docker#v5.10.0 195 | ``` 196 | 197 | ### Specifying build args 198 | 199 | [Build-time variables] are supported, either with an explicit value, or without 200 | one to propagate an environment variable from the pipeline step: 201 | 202 | [build-time variables]: https://docs.docker.com/engine/reference/commandline/build/#set-build-time-variables---build-arg 203 | 204 | ```dockerfile 205 | FROM bash 206 | 207 | ARG ARG_1 208 | ARG ARG_2 209 | 210 | RUN echo "${ARG_1}" 211 | RUN echo "${ARG_2}" 212 | ``` 213 | 214 | ```yaml 215 | steps: 216 | - command: echo amaze 217 | env: 218 | ARG_1: wow 219 | plugins: 220 | - seek-oss/docker-ecr-cache#v2.2.1: 221 | build-args: 222 | - ARG_1 223 | - ARG_2=such 224 | - docker#v5.10.0 225 | ``` 226 | 227 | Additional `docker build` arguments be passed via the `additional-build-args` setting: 228 | 229 | ```yaml 230 | steps: 231 | - command: echo amaze 232 | env: 233 | ARG_1: wow 234 | plugins: 235 | - seek-oss/docker-ecr-cache#v2.2.1: 236 | additional-build-args: '--ssh= default=\$SSH_AUTH_SOCK' 237 | - docker#v5.10.0 238 | ``` 239 | 240 | ### Specifying secrets 241 | 242 | [Build-time variables] can be extracted from a pulled image, so when passing 243 | sensitive data, [secrets] should be used instead. 244 | 245 | [secrets]: https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information 246 | 247 | To use environment variables (perhaps fetched by another plugin) as secrets: 248 | 249 | ```dockerfile 250 | # syntax=docker/dockerfile:1.2 251 | 252 | FROM bash 253 | 254 | RUN --mount=type=secret,id=SECRET cat /run/secrets/SECRET 255 | ``` 256 | 257 | ```yaml 258 | steps: 259 | - command: echo amaze 260 | env: 261 | SECRET: wow 262 | plugins: 263 | - seek-oss/docker-ecr-cache#v2.2.1: 264 | secrets: 265 | - SECRET 266 | - docker#v5.10.0 267 | ``` 268 | 269 | You can also specify the full `--secret` flag value if you need more control: 270 | 271 | ```yaml 272 | steps: 273 | - command: echo amaze 274 | env: 275 | SECRET: wow 276 | plugins: 277 | - seek-oss/private-npm#v1.2.0: 278 | env: SECRET 279 | - seek-oss/docker-ecr-cache#v2.2.1: 280 | secrets: 281 | - id=npmrc,src=.npmrc 282 | - docker#v5.10.0 283 | ``` 284 | 285 | You must have a recent version of Docker with BuildKit support to use secrets. 286 | This plugin will automatically enable BuildKit via the `DOCKER_BUILDKIT` 287 | environment variable if any secrets are present in the configuration. 288 | 289 | ### Changing the max cache time 290 | 291 | By default images are kept in ECR for up to 30 days. This can be changed by specifying a `max-age-days` parameter: 292 | 293 | ```yaml 294 | steps: 295 | - command: echo wow 296 | plugins: 297 | - seek-oss/docker-ecr-cache#v2.2.1: 298 | max-age-days: 7 299 | - docker#v5.10.0 300 | ``` 301 | 302 | ### Changing the name of exported variable 303 | 304 | By default, image name and computed tag are exported to the Docker buildkite plugin env variable `BUILDKITE_PLUGIN_DOCKER_IMAGE`. In order to chain the plugin with a different plugin, this can be changed by specifying a `export-env-variable` parameter: 305 | 306 | ```yaml 307 | steps: 308 | - command: echo wow 309 | plugins: 310 | - seek-oss/docker-ecr-cache#v2.2.1: 311 | export-env-variable: BUILDKITE_PLUGIN_MY_CUSTOM_PLUGIN_CACHE_IMAGE 312 | - my-custom-plugin#v1.0.0: 313 | ``` 314 | 315 | ### Skipping image pull from cache 316 | 317 | By default, this plugin will pull the image when a cache hit is found. In scenarios where you may be using a caching step to ensure that an image exists for future steps, this may not be required. You can use `skip-pull-from-cache` to allow the plugin to exit early without pulling the image. 318 | 319 | ```yaml 320 | steps: 321 | - label: Build Cache 322 | command: ':' 323 | plugins: 324 | - seek-oss/docker-ecr-cache#v2.2.1: 325 | skip-pull-from-cache: true 326 | ``` 327 | 328 | ### AWS ECR specific configuration 329 | 330 | #### Specifying an ECR repository name 331 | 332 | The plugin pushes and pulls Docker images to and from an ECR repository named 333 | `build-cache/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}`. You can 334 | optionally use a custom repository name: 335 | 336 | ```yaml 337 | steps: 338 | - command: echo wow 339 | plugins: 340 | - seek-oss/docker-ecr-cache#v2.2.1: 341 | ecr-name: my-unique-repository-name 342 | ecr-tags: 343 | Key: Value 344 | Key2: Value2 345 | - docker#v5.10.0 346 | ``` 347 | 348 | #### Specifying a region 349 | 350 | By default, the plugin uses the region specified in the `AWS_DEFAULT_REGION` environment variable. If this environment variable is not present, it defaults to the `eu-west-1` region. You can optionally specify the region in which you would like your cache to reside in: 351 | 352 | ```yaml 353 | steps: 354 | - command: echo wow 355 | plugins: 356 | - seek-oss/docker-ecr-cache#v2.2.1: 357 | region: ap-southeast-2 358 | - docker#v5.10.0 359 | ``` 360 | 361 | #### Required permissions 362 | 363 | Below is a sample set of IAM policy statements that will allow this plugin to work: 364 | 365 | ```yaml 366 | - Sid: AllowRepositoryActions 367 | Action: 368 | - ecr:BatchCheckLayerAvailability 369 | - ecr:BatchGetImage 370 | - ecr:CompleteLayerUpload 371 | - ecr:CreateRepository 372 | - ecr:DescribeImages 373 | - ecr:DescribeRepositories 374 | - ecr:InitiateLayerUpload 375 | - ecr:PutImage 376 | - ecr:PutLifecyclePolicy 377 | - ecr:SetRepositoryPolicy 378 | - ecr:UploadLayerPart 379 | Effect: Allow 380 | Resource: 381 | - Fn::Sub: arn:aws:ecr:*:${AWS::AccountId}:repository/build-cache/${YourOrganisationSlug}/${YourPipelineSlug} 382 | - Sid: AllowGetAuthorizationToken 383 | Action: 384 | - ecr:GetAuthorizationToken 385 | Resource: '*' 386 | Effect: Allow 387 | ``` 388 | 389 | ### GCP GCR specific configuration 390 | 391 | [Overview of Google Container Registry](https://cloud.google.com/container-registry/docs/overview) 392 | 393 | Example: 394 | 395 | ```yaml 396 | - command: echo wow 397 | plugins: 398 | - seek-oss/docker-ecr-cache#v2.2.1: 399 | registry-provider: gcr 400 | gcp-project: foo-bar-123456 401 | ``` 402 | 403 | #### Required GCR configuration 404 | 405 | - `registry-provider`: this must be `gcr` to aim at a google container registry. 406 | - `gcp-project`: this must be supplied. It is the GCP project your GCR is set up inside. 407 | 408 | #### Optional GCR configuration 409 | 410 | - `registry-hostname` (default: `gcr.io`). The location your image will be stored. [See upstream docs](https://cloud.google.com/container-registry/docs/overview#registry_name) for options. 411 | 412 | ## Design 413 | 414 | The plugin derives a checksum from: 415 | 416 | - The argument names and values specified in the `build-args` property 417 | - The files specified in the `cache-on` and `dockerfile` properties 418 | 419 | This checksum is used as the Docker image tag to find and pull an existing 420 | cached image from ECR, or to build and push a new image for subsequent builds to 421 | use. 422 | 423 | The plugin handles the creation of a dedicated ECR repository for the pipeline 424 | it runs in. To save on [ECR storage costs] and give images a chance to update/patch, a [lifecycle policy] is 425 | automatically applied to expire images after 30 days (configurable via `max-age-days`). 426 | 427 | [ecr storage costs]: https://aws.amazon.com/ecr/pricing/ 428 | [lifecycle policy]: https://docs.aws.amazon.com/AmazonECR/latest/userguide/LifecyclePolicies.html 429 | 430 | ## Tests 431 | 432 | To run the tests of this plugin, run 433 | 434 | ```bash 435 | docker-compose run --rm tests 436 | ``` 437 | 438 | ## License 439 | 440 | MIT (see [LICENSE](LICENSE)) 441 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seek-oss/docker-ecr-cache-buildkite-plugin/dd59aacb8653d3a0fa6e32b822dd40ff92990b58/changelog.md -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | tests: 3 | image: buildkite/plugin-tester:v4.1.0 4 | volumes: 5 | - '.:/plugin:ro' 6 | environment: 7 | - BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_TAGS_MYKEY=mykeyvalue 8 | lint: 9 | image: buildkite/plugin-linter 10 | command: ['--name', 'seek-oss/docker-ecr-cache'] 11 | volumes: 12 | - '.:/plugin:ro' 13 | -------------------------------------------------------------------------------- /hooks/lib/ecr-registry-provider.bash: -------------------------------------------------------------------------------- 1 | login() { 2 | local account_id 3 | local region 4 | 5 | account_id=$(aws sts get-caller-identity --query Account --output text) 6 | region=$(get_ecr_region) 7 | 8 | aws ecr get-login-password \ 9 | --region "${region}" \ 10 | | docker login \ 11 | --username AWS \ 12 | --password-stdin "${account_id}.dkr.ecr.${region}.amazonaws.com" 13 | } 14 | 15 | get_ecr_region() { 16 | echo "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGION:-${AWS_DEFAULT_REGION:-eu-west-1}}" 17 | } 18 | 19 | get_registry_url() { 20 | local repository_name 21 | repository_name="$(get_ecr_repository_name)" 22 | aws ecr describe-repositories \ 23 | --repository-names "${repository_name}" \ 24 | --output text \ 25 | --query 'repositories[0].repositoryUri' 26 | } 27 | 28 | ecr_exists() { 29 | local repository_name="${1}" 30 | aws ecr describe-repositories \ 31 | --repository-names "${repository_name}" \ 32 | --output text \ 33 | --query 'repositories[0].registryId' 34 | } 35 | 36 | image_exists() { 37 | local repository_name="$(get_ecr_repository_name)" 38 | local image_tag="${1}" 39 | local image_meta="$(aws ecr list-images \ 40 | --repository-name "${repository_name}" \ 41 | --query "imageIds[?imageTag=='${image_tag}'].imageTag" \ 42 | --output text)" 43 | 44 | if [ "$image_meta" == "$image_tag" ]; then 45 | true 46 | else 47 | false 48 | fi 49 | } 50 | 51 | get_ecr_arn() { 52 | local repository_name="${1}" 53 | aws ecr describe-repositories \ 54 | --repository-names "${repository_name}" \ 55 | --output text \ 56 | --query 'repositories[0].repositoryArn' 57 | } 58 | 59 | get_ecr_tags() { 60 | local result=$(cat <&2; 3 | } 4 | 5 | log_fatal() { 6 | echoerr "In $(pwd)" 7 | echoerr "${@}" 8 | # use the last argument as the exit code 9 | exit_code="${*: -1}" 10 | if [[ "${exit_code}" =~ ^[0-9]+$ ]]; then 11 | exit "${exit_code}" 12 | fi 13 | exit 1 14 | } 15 | 16 | read_build_args() { 17 | read_list_property 'BUILD_ARGS' 18 | for arg in ${result[@]+"${result[@]}"}; do 19 | build_args+=("--build-arg=${arg}") 20 | done 21 | } 22 | 23 | read_secrets() { 24 | read_list_property 'SECRETS' 25 | for arg in ${result[@]+"${result[@]}"}; do 26 | secrets_args+=("--secret") 27 | if [[ "${arg}" =~ ^id= ]]; then 28 | # Assume this is a full argument like id=123,src=path/to/file 29 | secrets_args+=("${arg}") 30 | else 31 | # Assume this is environment variable shorthand like SECRET_ENV 32 | secrets_args+=("id=${arg},env=${arg}") 33 | fi 34 | done 35 | } 36 | 37 | read_secrets_with_output() { 38 | read_secrets 39 | 40 | echo "${secrets_args[@]}" 41 | } 42 | 43 | # read a plugin property of type [array, string] into a Bash array. Buildkite 44 | # exposes a string value at BUILDKITE_PLUGIN_{NAME}_{KEY}, and array values at 45 | # BUILDKITE_PLUGIN_{NAME}_{KEY}_{IDX}. 46 | read_list_property() { 47 | local base_name="BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_${1}" 48 | 49 | result=() 50 | 51 | if [[ -n ${!base_name:-} ]]; then 52 | result+=("${!base_name}") 53 | fi 54 | 55 | while IFS='=' read -r item_name _; do 56 | if [[ ${item_name} =~ ^(${base_name}_[0-9]+) ]]; then 57 | result+=("${!item_name}") 58 | fi 59 | done < <(env | sort) 60 | } 61 | 62 | get_default_image_name() { 63 | echo "build-cache/${BUILDKITE_ORGANIZATION_SLUG}/${BUILDKITE_PIPELINE_SLUG}" 64 | } 65 | 66 | compute_tag() { 67 | local docker_file="$1" 68 | local sums 69 | 70 | echoerr '--- Computing tag' 71 | 72 | echoerr 'DOCKERFILE' 73 | echoerr "+ ${docker_file}:${target:-""}" 74 | sums="$(cd ${docker_file_dir}; sha1sum $(basename ${docker_file}))" 75 | sums+="$(echo "${target}" | sha1sum)" 76 | 77 | echoerr 'ARCHITECTURE' 78 | echoerr "+ $(uname -m)" 79 | sums+="$(uname -m | sha1sum)" 80 | 81 | echoerr 'BUILD_ARGS' 82 | read_list_property 'BUILD_ARGS' 83 | for arg in ${result[@]+"${result[@]}"}; do 84 | echoerr "+ ${arg}" 85 | 86 | # include underlying environment variable after echo 87 | if [[ ${arg} != *=* ]]; then 88 | arg+="=${!arg:-}" 89 | fi 90 | 91 | sums+="$(echo "${arg}" | sha1sum)" 92 | done 93 | 94 | if [[ -n "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ADDITIONAL_BUILD_ARGS:-}" ]]; then 95 | echoerr 'ADDITIONAL_BUILD_ARGS' 96 | sums+="$(echo "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ADDITIONAL_BUILD_ARGS}" | sha1sum)" 97 | fi 98 | 99 | # expand ** in cache-on properties 100 | shopt -s globstar 101 | 102 | echoerr 'CACHE_ON' 103 | read_list_property 'CACHE_ON' 104 | for glob in ${result[@]+"${result[@]}"}; do 105 | echoerr "${glob}" 106 | for file in ${glob}; do 107 | echoerr "+ ${file}" 108 | if [[ "${file}" == *.json#* ]]; then 109 | # Extract the file path and keys from the pattern 110 | file_path="${file%%#*}" 111 | keys=${file#*#} 112 | 113 | # Read the JSON file and calculate sha1sum only for the specified keys 114 | value=$(jq -r "${keys}" "${file_path}") 115 | sums+="$(echo -n "${value}" | sha1sum)" 116 | else 117 | # Calculate sha1sum for the whole file 118 | sums+="$(sha1sum "${file}")" 119 | fi 120 | done 121 | done 122 | 123 | echo "${sums}" | sha1sum | cut -c-7 124 | } 125 | -------------------------------------------------------------------------------- /hooks/lib/stub-registry-provider.bash: -------------------------------------------------------------------------------- 1 | login() { 2 | echo "stubbed login" 3 | } 4 | 5 | configure_registry_for_image_if_necessary() { 6 | echo "stubbed configure_registry_for_image_if_necessary" 7 | } 8 | 9 | get_registry_url() { 10 | echo "pretend.host/path/segment/image" 11 | } 12 | 13 | # BATS/bats-mock does not allow stubbing a function, currently. 14 | # So, override it to avoid needing to repeat all the stub'd sha1sum etc inside the end-to-end tests. 15 | compute_tag() { 16 | echo "stubbed-computed-tag" 17 | } 18 | 19 | image_exists() { 20 | echo "stubbed image_exists" 21 | } 22 | -------------------------------------------------------------------------------- /hooks/pre-command: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | # shellcheck source=lib/stdlib.bash 5 | source "$(dirname "${BASH_SOURCE[0]}")/lib/stdlib.bash" || exit 67 6 | source "$(dirname "${BASH_SOURCE[0]}")/lib/${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER:-"ecr"}-registry-provider.bash" || 7 | log_fatal "Failed to source registry-provider. BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER must be set to one of [ecr, gcr]" 67 8 | 9 | login 10 | configure_registry_for_image_if_necessary 11 | image="$(get_registry_url)" 12 | if [ -n "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_DOCKERFILE_INLINE:-}" ] 13 | then 14 | [ -n "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_DOCKERFILE:-}" ] && 15 | log_fatal "Cannot specify both 'dockerfile' and 'dockerfile-inline'." 16 | # Put the Dockerfile into a temporary directory to work around 17 | # https://github.com/docker/cli/issues/2249 18 | docker_file_dir="$(mktemp -d)" 19 | docker_file="${docker_file_dir}/Dockerfile" 20 | echo "$BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_DOCKERFILE_INLINE" > "$docker_file" 21 | context_dir="." 22 | else 23 | docker_file="${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_DOCKERFILE:-"Dockerfile"}" 24 | context_dir="$(dirname "${docker_file}")" 25 | docker_file_dir="${context_dir}" 26 | fi 27 | target="${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_TARGET:-}" 28 | export_env_variable="${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_ENV_VARIABLE:-"BUILDKITE_PLUGIN_DOCKER_IMAGE"}" 29 | exec 3>&1 30 | tag="$(compute_tag "${docker_file}" 2>&3)" 31 | context="${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_CONTEXT:-"${context_dir}"}" 32 | 33 | build_args=() 34 | read_build_args 35 | 36 | secrets_args=() 37 | read_secrets 38 | 39 | if [ "${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SKIP_PULL_FROM_CACHE:-}" == "true" ] && image_exists "$tag"; then 40 | echo "Image exists, skipping pull" 41 | else 42 | echo "--- Pulling image" 43 | 44 | if ! docker pull "${image}:${tag}"; then 45 | echo '--- Building image' 46 | image_build_args=( 47 | "build" 48 | "--file=${docker_file}" 49 | "--progress=plain" 50 | "--tag=${image}:${tag}" 51 | ) 52 | if [[ -n "${target:-}" ]]; then 53 | image_build_args+=( 54 | "--target=${target}" 55 | ) 56 | fi 57 | if [[ "${#build_args[@]}" -gt 0 ]]; then 58 | for ba in "${build_args[@]}"; do 59 | image_build_args+=( 60 | "${ba}" 61 | ) 62 | done 63 | fi 64 | if [[ "${#secrets_args[@]}" -gt 0 ]]; then 65 | export DOCKER_BUILDKIT=1 66 | for sa in "${secrets_args[@]}"; do 67 | image_build_args+=( 68 | "${sa}" 69 | ) 70 | done 71 | fi 72 | 73 | echo "Inside $(pwd), running \`docker ${image_build_args[*]} ${context}\`" 74 | # We can't quote BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ADDITIONAL_BUILD_ARGS, because it's passed here as a string instead of a bash array. 75 | # shellcheck disable=SC2086 76 | docker "${image_build_args[@]}" ${BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ADDITIONAL_BUILD_ARGS:-} "${context}" || 77 | log_fatal "^^^ +++" 1 78 | 79 | docker tag "${image}:${tag}" "${image}:latest" 80 | 81 | echo "--- Pushing tag ${tag}" 82 | docker push "${image}:${tag}" 83 | 84 | echo "--- Pushing tag latest" 85 | docker push "${image}:latest" 86 | fi || echo "Not found" 87 | fi 88 | 89 | # Support using https://github.com/buildkite-plugins/docker-buildkite-plugin without an image by default 90 | export ${export_env_variable}="${image}:${tag}" 91 | 92 | # Support programmatic use of cache image and tag values 93 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_IMAGE="${image}" 94 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_EXPORT_TAG="${tag}" 95 | -------------------------------------------------------------------------------- /plugin.yml: -------------------------------------------------------------------------------- 1 | name: Docker ECR Cache 2 | description: Cache Docker images in Amazon ECR 3 | author: https://github.com/seek-oss 4 | requirements: 5 | - docker 6 | configuration: 7 | properties: 8 | build-args: 9 | type: [array, string] 10 | additional-build-args: 11 | type: string 12 | cache-on: 13 | type: [array, string] 14 | dockerfile: 15 | type: string 16 | ecr-name: 17 | type: string 18 | ecr-tags: 19 | type: object 20 | target: 21 | type: string 22 | max-age-days: 23 | type: number 24 | export-env-variable: 25 | type: string 26 | context: 27 | type: string 28 | registry-provider: 29 | type: string 30 | gcp-project: 31 | type: string 32 | registry-hostname: 33 | type: string 34 | region: 35 | type: string 36 | skip-pull-from-cache: 37 | type: boolean 38 | required: [] 39 | -------------------------------------------------------------------------------- /tests/ecr-registry-provider.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # export AWS_STUB_DEBUG=/dev/tty 4 | # export DOCKER_STUB_DEBUG=/dev/tty 5 | 6 | load "$BATS_PLUGIN_PATH/load.bash" 7 | load "$PWD/hooks/lib/stdlib.bash" 8 | load "$PWD/hooks/lib/ecr-registry-provider.bash" 9 | 10 | pre_command_hook="$PWD/hooks/pre-command" 11 | 12 | @test "ECR: Applies lifecycle policy to existing repositories" { 13 | export AWS_DEFAULT_REGION="ap-southeast-2" 14 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 15 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 16 | local expected_repository_name="build-cache/example-org/example-pipeline" 17 | 18 | stub aws \ 19 | "sts get-caller-identity --query Account --output text : echo 1234567891012" \ 20 | "ecr get-login-password --region ap-southeast-2 : echo secure-ecr-password" \ 21 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].registryId : echo looked up repository" \ 22 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryArn : echo arn:aws:ecr:ap-southeast-2:1234567891012:repository/${expected_repository_name}" \ 23 | "ecr tag-resource --resource-arn arn:aws:ecr:ap-southeast-2:1234567891012:repository/build-cache/example-org/example-pipeline --cli-input-json \* : echo tag existing resource" \ 24 | "ecr put-lifecycle-policy --repository-name build-cache/example-org/example-pipeline --lifecycle-policy-text \* : echo put lifecycle policy" \ 25 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryUri : echo https://1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com" 26 | 27 | stub docker \ 28 | "login --username AWS --password-stdin 1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com : echo logging in to docker" \ 29 | "pull https://1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com:sha1sum : echo pulled image" 30 | 31 | stub sha1sum \ 32 | "Dockerfile : echo 'sha1sum(Dockerfile)'" \ 33 | ": echo sha1sum" \ 34 | ": echo sha1sum" \ 35 | ": echo sha1sum" 36 | 37 | run "${pre_command_hook}" 38 | 39 | assert_success 40 | assert_output --partial "logging in to docker" 41 | assert_output --partial "pulled image" 42 | assert_output --partial "looked up repository" 43 | assert_output --partial "tag existing resource" 44 | assert_output --partial "put lifecycle policy" 45 | 46 | unstub aws 47 | unstub docker 48 | unstub sha1sum 49 | } 50 | 51 | @test "ECR: Builds new images with tags" { 52 | export AWS_DEFAULT_REGION="ap-southeast-2" 53 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 54 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 55 | local expected_repository_name="build-cache/example-org/example-pipeline" 56 | local repository_uri="1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com/${expected_repository_name}" 57 | 58 | stub aws \ 59 | "sts get-caller-identity --query Account --output text : echo 1234567891012" \ 60 | "ecr get-login-password --region ap-southeast-2 : echo secure-ecr-password" \ 61 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].registryId : echo looked up repository" \ 62 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryArn : echo arn:aws:ecr:ap-southeast-2:1234567891012:repository/${expected_repository_name}" \ 63 | "ecr tag-resource --resource-arn arn:aws:ecr:ap-southeast-2:1234567891012:repository/build-cache/example-org/example-pipeline --cli-input-json \* : echo tag existing resource" \ 64 | "ecr put-lifecycle-policy --repository-name build-cache/example-org/example-pipeline --lifecycle-policy-text \* : echo put lifecycle policy" \ 65 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryUri : echo ${repository_uri}" \ 66 | 67 | stub docker \ 68 | "login --username AWS --password-stdin 1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com : echo logging in to docker" \ 69 | "pull 1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com/build-cache/example-org/example-pipeline:deadbee : echo not found && false" \ 70 | "build --file=Dockerfile --progress=plain --tag=1234567891012.dkr.ecr.ap-southeast-2.amazonaws.com/build-cache/example-org/example-pipeline:deadbee . : echo building docker image" \ 71 | "tag ${repository_uri}:deadbee ${repository_uri}:latest : echo tagged latest" \ 72 | "push ${repository_uri}:deadbee : echo pushed deadbee" \ 73 | "push ${repository_uri}:latest : echo pushed latest" 74 | 75 | stub sha1sum \ 76 | "Dockerfile : echo 'sha1sum(Dockerfile)'" \ 77 | ": echo sha1sum" \ 78 | ": echo sha1sum" \ 79 | ": echo deadbee" 80 | 81 | run "${pre_command_hook}" 82 | 83 | assert_success 84 | assert_output --partial "logging in to docker" 85 | assert_output --partial "looked up repository" 86 | assert_output --partial "building docker image" 87 | assert_output --partial "tag existing resource" 88 | assert_output --partial "put lifecycle policy" 89 | assert_output --partial "tagged latest" 90 | assert_output --partial "pushed deadbee" 91 | assert_output --partial "pushed latest" 92 | 93 | unstub aws 94 | unstub docker 95 | unstub sha1sum 96 | } 97 | 98 | @test "ECR: Uses correct region when region not specified and AWS_DEFAULT_REGION not set" { 99 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 100 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 101 | local expected_repository_name="build-cache/example-org/example-pipeline" 102 | local repository_uri="1234567891012.dkr.ecr.eu-west-1.amazonaws.com/${expected_repository_name}" 103 | 104 | stub aws \ 105 | "sts get-caller-identity --query Account --output text : echo 1234567891012" \ 106 | "ecr get-login-password --region eu-west-1 : echo secure-ecr-password" \ 107 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].registryId : echo looked up repository" \ 108 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryArn : echo arn:aws:ecr:eu-west-1:1234567891012:repository/${expected_repository_name}" \ 109 | "ecr tag-resource --resource-arn arn:aws:ecr:eu-west-1:1234567891012:repository/build-cache/example-org/example-pipeline --cli-input-json \* : echo tag existing resource" \ 110 | "ecr put-lifecycle-policy --repository-name build-cache/example-org/example-pipeline --lifecycle-policy-text \* : echo put lifecycle policy" \ 111 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryUri : echo ${repository_uri}" \ 112 | 113 | stub docker \ 114 | "login --username AWS --password-stdin 1234567891012.dkr.ecr.eu-west-1.amazonaws.com : echo logging in to docker" \ 115 | "pull 1234567891012.dkr.ecr.eu-west-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee : echo not found && false" \ 116 | "build --file=Dockerfile --progress=plain --tag=1234567891012.dkr.ecr.eu-west-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee . : echo building docker image" \ 117 | "tag ${repository_uri}:deadbee ${repository_uri}:latest : echo tagged latest" \ 118 | "push ${repository_uri}:deadbee : echo pushed deadbee" \ 119 | "push ${repository_uri}:latest : echo pushed latest" 120 | 121 | stub sha1sum \ 122 | "Dockerfile : echo 'sha1sum(Dockerfile)'" \ 123 | ": echo sha1sum" \ 124 | ": echo sha1sum" \ 125 | ": echo deadbee" 126 | 127 | run "${pre_command_hook}" 128 | 129 | assert_success 130 | assert_output --partial "logging in to docker" 131 | assert_output --partial "looked up repository" 132 | assert_output --partial "building docker image" 133 | assert_output --partial "tag existing resource" 134 | assert_output --partial "put lifecycle policy" 135 | assert_output --partial "tagged latest" 136 | assert_output --partial "pushed deadbee" 137 | assert_output --partial "pushed latest" 138 | 139 | unstub aws 140 | unstub docker 141 | unstub sha1sum 142 | } 143 | 144 | @test "ECR: Uses correct region when region is specified" { 145 | export AWS_DEFAULT_REGION="ap-southeast-2" 146 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGION="ap-southeast-1" 147 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 148 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 149 | local expected_repository_name="build-cache/example-org/example-pipeline" 150 | local repository_uri="1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/${expected_repository_name}" 151 | 152 | stub aws \ 153 | "sts get-caller-identity --query Account --output text : echo 1234567891012" \ 154 | "ecr get-login-password --region ap-southeast-1 : echo secure-ecr-password" \ 155 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].registryId : echo looked up repository" \ 156 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryArn : echo arn:aws:ecr:ap-southeast-1:1234567891012:repository/${expected_repository_name}" \ 157 | "ecr tag-resource --resource-arn arn:aws:ecr:ap-southeast-1:1234567891012:repository/build-cache/example-org/example-pipeline --cli-input-json \* : echo tag existing resource" \ 158 | "ecr put-lifecycle-policy --repository-name build-cache/example-org/example-pipeline --lifecycle-policy-text \* : echo put lifecycle policy" \ 159 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryUri : echo ${repository_uri}" \ 160 | 161 | stub docker \ 162 | "login --username AWS --password-stdin 1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com : echo logging in to docker" \ 163 | "pull 1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee : echo not found && false" \ 164 | "build --file=Dockerfile --progress=plain --tag=1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/build-cache/example-org/example-pipeline:deadbee . : echo building docker image" \ 165 | "tag ${repository_uri}:deadbee ${repository_uri}:latest : echo tagged latest" \ 166 | "push ${repository_uri}:deadbee : echo pushed deadbee" \ 167 | "push ${repository_uri}:latest : echo pushed latest" 168 | 169 | stub sha1sum \ 170 | "Dockerfile : echo 'sha1sum(Dockerfile)'" \ 171 | ": echo sha1sum" \ 172 | ": echo sha1sum" \ 173 | ": echo deadbee" 174 | 175 | run "${pre_command_hook}" 176 | 177 | assert_success 178 | assert_output --partial "logging in to docker" 179 | assert_output --partial "looked up repository" 180 | assert_output --partial "building docker image" 181 | assert_output --partial "tag existing resource" 182 | assert_output --partial "put lifecycle policy" 183 | assert_output --partial "tagged latest" 184 | assert_output --partial "pushed deadbee" 185 | assert_output --partial "pushed latest" 186 | 187 | unstub aws 188 | unstub docker 189 | unstub sha1sum 190 | } 191 | 192 | @test "ECR: Calls list-images to check existence of cache" { 193 | export AWS_DEFAULT_REGION="ap-southeast-2" 194 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGION="ap-southeast-1" 195 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 196 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 197 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SKIP_PULL_FROM_CACHE="true" 198 | local expected_repository_name="build-cache/example-org/example-pipeline" 199 | local repository_uri="1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com/${expected_repository_name}" 200 | 201 | stub aws \ 202 | "sts get-caller-identity --query Account --output text : echo 1234567891012" \ 203 | "ecr get-login-password --region ap-southeast-1 : echo secure-ecr-password" \ 204 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].registryId : echo looked up repository" \ 205 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryArn : echo arn:aws:ecr:ap-southeast-1:1234567891012:repository/${expected_repository_name}" \ 206 | "ecr tag-resource --resource-arn arn:aws:ecr:ap-southeast-1:1234567891012:repository/build-cache/example-org/example-pipeline --cli-input-json \* : echo tag existing resource" \ 207 | "ecr put-lifecycle-policy --repository-name build-cache/example-org/example-pipeline --lifecycle-policy-text \* : echo put lifecycle policy" \ 208 | "ecr describe-repositories --repository-names ${expected_repository_name} --output text --query repositories[0].repositoryUri : echo ${repository_uri}" \ 209 | "ecr list-images --repository-name ${expected_repository_name} --query imageIds[?imageTag==\'deadbee\'].imageTag --output text : echo 'deadbee'" 210 | 211 | stub docker \ 212 | "login --username AWS --password-stdin 1234567891012.dkr.ecr.ap-southeast-1.amazonaws.com : echo logging in to docker" 213 | 214 | stub sha1sum \ 215 | "Dockerfile : echo 'sha1sum(Dockerfile)'" \ 216 | ": echo sha1sum" \ 217 | ": echo sha1sum" \ 218 | ": echo deadbee" 219 | 220 | run "${pre_command_hook}" 221 | 222 | assert_success 223 | assert_output --partial "logging in to docker" 224 | assert_output --partial "looked up repository" 225 | assert_output --partial "Image exists, skipping pull" 226 | unstub aws 227 | unstub docker 228 | unstub sha1sum 229 | } 230 | -------------------------------------------------------------------------------- /tests/gcr-registry-provider.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "$BATS_PLUGIN_PATH/load.bash" 4 | load "$PWD/hooks/lib/stdlib.bash" 5 | load "$PWD/hooks/lib/gcr-registry-provider.bash" 6 | 7 | @test "GCR: Can login" { 8 | run login 9 | 10 | assert_success 11 | assert_output --partial "Plugin currently assumes" 12 | } 13 | 14 | @test "GCR: Can configure registry for image if necessary" { 15 | # Currently a no-op for GCR. 16 | run configure_registry_for_image_if_necessary 17 | 18 | assert_success 19 | assert_output "" 20 | } 21 | 22 | @test "GCR: get_registry_url fail when no gcp-project" { 23 | run get_registry_url 24 | 25 | assert_failure 26 | assert_output --partial "gcp-project" 27 | } 28 | 29 | @test "GCR: get_registry_url uses defaults when no registry-hostname or ecr-name" { 30 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 31 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 32 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_GCP_PROJECT="rusty-beaver-23452" 33 | 34 | run get_registry_url 35 | 36 | assert_success 37 | assert_line "gcr.io/rusty-beaver-23452/build-cache/example-org/example-pipeline" 38 | } 39 | 40 | @test "GCR: get_registry_url uses overrides when supplied" { 41 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_GCP_PROJECT="rusty-beaver-23452" 42 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_HOSTNAME="eu.gcr.io" 43 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ECR_NAME="my-dam" 44 | 45 | run get_registry_url 46 | 47 | assert_success 48 | assert_line "eu.gcr.io/rusty-beaver-23452/my-dam" 49 | } 50 | -------------------------------------------------------------------------------- /tests/pre-command.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load "$BATS_PLUGIN_PATH/load.bash" 4 | 5 | # export DOCKER_STUB_DEBUG=/dev/tty 6 | 7 | pre_command_hook="$PWD/hooks/pre-command" 8 | 9 | @test "Fails hard if bad registry-provider" { 10 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="utter-fantasy" 11 | 12 | run "${pre_command_hook}" 13 | 14 | assert_failure 15 | assert_line --partial "Failed to source registry-provider." 16 | } 17 | 18 | @test "Skips build if pull succeeds" { 19 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="stub" 20 | local repository_uri="pretend.host/path/segment/image" 21 | 22 | stub docker \ 23 | "pull pretend.host/path/segment/image:stubbed-computed-tag : true" 24 | 25 | run "${pre_command_hook}" 26 | 27 | assert_success 28 | assert_line "--- Pulling image" 29 | 30 | unstub docker 31 | } 32 | 33 | @test "Exits 1 if docker build fails" { 34 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="stub" 35 | 36 | stub docker \ 37 | "pull pretend.host/path/segment/image:stubbed-computed-tag : false" \ 38 | "build --file=Dockerfile --progress=plain --tag=pretend.host/path/segment/image:stubbed-computed-tag . : exit 242" 39 | 40 | run "${pre_command_hook}" 41 | 42 | assert_failure 43 | assert_line "--- Pulling image" 44 | assert_line "--- Building image" 45 | refute_line --partial "--- Pushing tag" 46 | 47 | unstub docker 48 | } 49 | 50 | @test "Tags and pushes computed tag and latest if build succeeds" { 51 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="stub" 52 | local repository_uri="pretend.host/path/segment/image" 53 | 54 | stub docker \ 55 | "pull pretend.host/path/segment/image:stubbed-computed-tag : false" \ 56 | "build --file=Dockerfile --progress=plain --tag=pretend.host/path/segment/image:stubbed-computed-tag . : echo building docker image" \ 57 | "tag ${repository_uri}:stubbed-computed-tag ${repository_uri}:latest : echo tagged latest" \ 58 | "push ${repository_uri}:stubbed-computed-tag : echo pushed stubbed-computed-tag" \ 59 | "push ${repository_uri}:latest : echo pushed latest" 60 | run "${pre_command_hook}" 61 | 62 | assert_success 63 | assert_line "--- Pulling image" 64 | assert_line "--- Building image" 65 | assert_line "--- Pushing tag stubbed-computed-tag" 66 | assert_line "--- Pushing tag latest" 67 | 68 | unstub docker 69 | } 70 | 71 | @test "Tags and pushes with inline Dockerfile" { 72 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="stub" 73 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_DOCKERFILE_INLINE="FROM stub" 74 | local repository_uri="pretend.host/path/segment/image" 75 | 76 | one_time_mktemp=$(mktemp -d) 77 | 78 | stub mktemp \ 79 | "-d : echo $one_time_mktemp" 80 | 81 | stub docker \ 82 | "pull pretend.host/path/segment/image:stubbed-computed-tag : false" \ 83 | "build --file=$one_time_mktemp/Dockerfile --progress=plain --tag=pretend.host/path/segment/image:stubbed-computed-tag . : echo building docker image" \ 84 | "tag ${repository_uri}:stubbed-computed-tag ${repository_uri}:latest : echo tagged latest" \ 85 | "push ${repository_uri}:stubbed-computed-tag : echo pushed stubbed-computed-tag" \ 86 | "push ${repository_uri}:latest : echo pushed latest" 87 | run "${pre_command_hook}" 88 | 89 | assert_success 90 | assert_line "--- Pulling image" 91 | assert_line "--- Building image" 92 | assert_line "--- Pushing tag stubbed-computed-tag" 93 | assert_line "--- Pushing tag latest" 94 | 95 | assert_equal "FROM stub" "$(cat ${one_time_mktemp}/Dockerfile)" 96 | 97 | unstub mktemp 98 | unstub docker 99 | } 100 | 101 | @test "Exits 0 if skip-pull-on-cache and image exists" { 102 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_REGISTRY_PROVIDER="stub" 103 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SKIP_PULL_FROM_CACHE="true" 104 | local repository_uri="pretend.host/path/segment/image" 105 | 106 | run "${pre_command_hook}" 107 | 108 | assert_success 109 | assert_line "Image exists, skipping pull" 110 | } 111 | -------------------------------------------------------------------------------- /tests/stdlib.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | # export UNAME_STUB_DEBUG=/dev/tty 4 | # export SHA1SUM_STUB_DEBUG=/dev/tty 5 | # export JQ_STUB_DEBUG=/dev/tty 6 | 7 | load "$BATS_PLUGIN_PATH/load.bash" 8 | load "$PWD/hooks/lib/stdlib.bash" 9 | 10 | pre_command_hook="$PWD/hooks/pre-command" 11 | 12 | @test "Can read build-args from array" { 13 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_BUILD_ARGS_1="foo=1" 14 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_BUILD_ARGS_2="bar=2" 15 | 16 | run read_build_args 17 | 18 | assert_success 19 | # cannot assert, here, because function does not emit output, and populates build_args var in outer scope. 20 | # coverage happens via later tests of compute_tag. 21 | } 22 | 23 | @test "Can read secrets from array" { 24 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SECRETS_1="FOO" 25 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SECRETS_2="id=1,env=BAR" 26 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_SECRETS_3="id=2,src=path/to/secret.txt" 27 | 28 | run read_secrets_with_output 29 | 30 | assert_success 31 | assert_output "--secret id=FOO,env=FOO --secret id=1,env=BAR --secret id=2,src=path/to/secret.txt" 32 | } 33 | 34 | @test "Can get default image name" { 35 | export BUILDKITE_ORGANIZATION_SLUG="example-org" 36 | export BUILDKITE_PIPELINE_SLUG="example-pipeline" 37 | 38 | run get_default_image_name 39 | 40 | assert_success 41 | assert_output "build-cache/example-org/example-pipeline" 42 | } 43 | 44 | @test "Can compute image tag, no target, no build-args, no cache-on" { 45 | # TODO: this var leaks in via pre-command. Fix at some point by adding a function arg. 46 | target="" 47 | 48 | stub uname \ 49 | "-m : echo my-architecture" \ 50 | "-m : echo my-architecture" 51 | stub sha1sum \ 52 | "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \ 53 | ": echo sha1sum(target: )" \ 54 | ": echo sha1sum(my-architecture)" \ 55 | ": echo sha1sum(hashes so far)" 56 | 57 | run compute_tag "pretend-dockerfile" 58 | 59 | assert_success 60 | assert_line "--- Computing tag" 61 | assert_line "DOCKERFILE" 62 | assert_line "+ pretend-dockerfile:" 63 | assert_line "ARCHITECTURE" 64 | assert_line "+ my-architecture" 65 | assert_line "BUILD_ARGS" 66 | refute_line "ADDITIONAL_BUILD_ARGS" 67 | assert_line "CACHE_ON" 68 | 69 | unstub uname 70 | unstub sha1sum 71 | } 72 | 73 | @test "Can compute image tag, with target" { 74 | # this var leaks in via pre-command 75 | target="my-multi-stage-container" 76 | 77 | stub uname \ 78 | "-m : echo my-architecture" \ 79 | "-m : echo my-architecture" 80 | stub sha1sum \ 81 | "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \ 82 | ": echo sha1sum(target: my-multi-stage-container)" \ 83 | ": echo sha1sum(uname: my-architecture)" \ 84 | ": echo sha1sum(hashes so far)" 85 | 86 | run compute_tag "pretend-dockerfile" 87 | 88 | assert_success 89 | assert_line "--- Computing tag" 90 | assert_line "DOCKERFILE" 91 | assert_line "+ pretend-dockerfile:my-multi-stage-container" 92 | assert_line "ARCHITECTURE" 93 | assert_line "+ my-architecture" 94 | assert_line "BUILD_ARGS" 95 | refute_line "ADDITIONAL_BUILD_ARGS" 96 | assert_line "CACHE_ON" 97 | 98 | unstub uname 99 | unstub sha1sum 100 | } 101 | 102 | @test "Can compute image tag, with target, build-args" { 103 | # this var leaks in via pre-command 104 | target="my-multi-stage-container" 105 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_BUILD_ARGS_1="foo=1" 106 | 107 | stub uname \ 108 | "-m : echo my-architecture" \ 109 | "-m : echo my-architecture" 110 | stub sha1sum \ 111 | "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \ 112 | ": echo sha1sum(target: my-multi-stage-container)" \ 113 | ": echo sha1sum(uname: my-architecture)" \ 114 | ": echo sha1sum(build-arg: foo=1)" \ 115 | ": echo sha1sum(hashes so far)" 116 | 117 | run compute_tag "pretend-dockerfile" 118 | 119 | assert_success 120 | assert_line "--- Computing tag" 121 | assert_line "DOCKERFILE" 122 | assert_line "+ pretend-dockerfile:my-multi-stage-container" 123 | assert_line "ARCHITECTURE" 124 | assert_line "+ my-architecture" 125 | assert_line "BUILD_ARGS" 126 | assert_line "+ foo=1" 127 | refute_line "ADDITIONAL_BUILD_ARGS" 128 | assert_line "CACHE_ON" 129 | 130 | unstub uname 131 | unstub sha1sum 132 | } 133 | 134 | @test "Can compute image tag, with target, build-args, cache-on" { 135 | # this var leaks in via pre-command 136 | target="my-multi-stage-container" 137 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_BUILD_ARGS_1="foo=1" 138 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_CACHE_ON_1="amazing-content.txt" 139 | 140 | stub uname \ 141 | "-m : echo my-architecture" \ 142 | "-m : echo my-architecture" 143 | stub sha1sum \ 144 | "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \ 145 | ": echo sha1sum(target: my-multi-stage-container)" \ 146 | ": echo sha1sum(uname: my-architecture)" \ 147 | ": echo sha1sum(build-arg: foo=1)" \ 148 | "amazing-content.txt : echo sha1sum(cache-on: amazing-content.txt)" \ 149 | ": echo sha1sum(hashes so far)" 150 | 151 | run compute_tag "pretend-dockerfile" 152 | 153 | assert_success 154 | assert_line "--- Computing tag" 155 | assert_line "DOCKERFILE" 156 | assert_line "+ pretend-dockerfile:my-multi-stage-container" 157 | assert_line "ARCHITECTURE" 158 | assert_line "+ my-architecture" 159 | assert_line "BUILD_ARGS" 160 | assert_line "+ foo=1" 161 | refute_line "ADDITIONAL_BUILD_ARGS" 162 | assert_line "CACHE_ON" 163 | 164 | unstub uname 165 | unstub sha1sum 166 | } 167 | 168 | @test "Can compute image tag with cache-on on specific json keys" { 169 | # this var leaks in via pre-command 170 | target="my-multi-stage-container" 171 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_CACHE_ON_1="test-package.json#.dependencies" 172 | 173 | stub uname \ 174 | "-m : echo my-architecture" \ 175 | "-m : echo my-architecture" 176 | stub jq \ 177 | "-r .dependencies test-package.json : echo '{\"test\":\"123\"}'" 178 | stub sha1sum \ 179 | "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \ 180 | ": echo sha1sum(target: my-multi-stage-container)" \ 181 | ": echo sha1sum(uname: my-architecture)" \ 182 | ": echo sha1sum(jq: .dependencies)" \ 183 | ": echo sha1sum(hashes so far)" 184 | 185 | run compute_tag "pretend-dockerfile" 186 | 187 | assert_success 188 | assert_line "--- Computing tag" 189 | assert_line "DOCKERFILE" 190 | assert_line "+ pretend-dockerfile:my-multi-stage-container" 191 | assert_line "ARCHITECTURE" 192 | assert_line "+ my-architecture" 193 | assert_line "BUILD_ARGS" 194 | refute_line "ADDITIONAL_BUILD_ARGS" 195 | assert_line "CACHE_ON" 196 | 197 | unstub uname 198 | unstub jq 199 | unstub sha1sum 200 | } 201 | 202 | @test "Can compute image tag with additional-build-args" { 203 | # this var leaks in via pre-command 204 | target="my-multi-stage-container" 205 | export BUILDKITE_PLUGIN_DOCKER_ECR_CACHE_ADDITIONAL_BUILD_ARGS='--platform=linux/amd64,linux/arm64' 206 | 207 | stub uname \ 208 | "-m : echo my-architecture" \ 209 | "-m : echo my-architecture" 210 | stub sha1sum \ 211 | "pretend-dockerfile : echo sha1sum(pretend-dockerfile)" \ 212 | ": echo sha1sum(target: my-multi-stage-container)" \ 213 | ": echo sha1sum(uname: my-architecture)" \ 214 | ": echo sha1sum(--platform=linux/amd64,linux/arm64)" \ 215 | ": echo sha1sum(hashes so far)" 216 | 217 | run compute_tag "pretend-dockerfile" 218 | 219 | assert_success 220 | assert_line "--- Computing tag" 221 | assert_line "DOCKERFILE" 222 | assert_line "+ pretend-dockerfile:my-multi-stage-container" 223 | assert_line "ARCHITECTURE" 224 | assert_line "+ my-architecture" 225 | assert_line "BUILD_ARGS" 226 | assert_line "ADDITIONAL_BUILD_ARGS" 227 | assert_line "CACHE_ON" 228 | 229 | unstub uname 230 | unstub sha1sum 231 | } 232 | --------------------------------------------------------------------------------