├── docker-compose.yml ├── hooks ├── pre-command └── post-command ├── plugin.yml ├── lib └── shared.bash ├── .github └── workflows │ └── tests.yml ├── LICENSE ├── README.md └── tests ├── pre-command.bats └── post-command.bats /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | tests: 4 | image: "buildkite/plugin-tester" 5 | volumes: 6 | - ".:/plugin:ro" 7 | lint: 8 | image: "buildkite/plugin-linter" 9 | command: ["--id", "envato/aws-s3-sync"] 10 | volumes: 11 | - ".:/plugin:ro" 12 | -------------------------------------------------------------------------------- /hooks/pre-command: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" 5 | 6 | # shellcheck source=lib/shared.bash 7 | source "$DIR/../lib/shared.bash" 8 | 9 | if [[ $BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE == s3://* ]] && [[ $BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION != s3://* ]] ;then 10 | aws_s3_sync "$BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE" "$BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION" 11 | fi 12 | 13 | -------------------------------------------------------------------------------- /hooks/post-command: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | 4 | DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" 5 | 6 | # shellcheck source=lib/shared.bash 7 | source "$DIR/../lib/shared.bash" 8 | 9 | if [[ ${BUILDKITE_COMMAND_EXIT_STATUS:-0} != '0' ]]; then 10 | echo '~~~ Skipping S3 sync because the command failed' 11 | exit 0 12 | fi 13 | 14 | if [[ $BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE != s3://* ]] ;then 15 | aws_s3_sync "$BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE" "$BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION" 16 | fi 17 | 18 | -------------------------------------------------------------------------------- /plugin.yml: -------------------------------------------------------------------------------- 1 | name: AWS S3 Sync 2 | description: A Buildkite plugin syncs files to the AWS Simple Storage Service (S3) 3 | author: https://github.com/envato 4 | requirements: 5 | - aws 6 | configuration: 7 | properties: 8 | source: 9 | type: string 10 | destination: 11 | type: string 12 | delete: 13 | type: boolean 14 | follow-symlinks: 15 | type: boolean 16 | cache-control: 17 | type: string 18 | endpoint-url: 19 | type: string 20 | required: 21 | - source 22 | - destination 23 | -------------------------------------------------------------------------------- /lib/shared.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function aws_s3_sync() { 4 | local source=$1 5 | local destination=$2 6 | 7 | params=() 8 | 9 | if [[ "${BUILDKITE_PLUGIN_AWS_S3_SYNC_DELETE:-false}" == "true" ]]; then 10 | params+=(--delete) 11 | fi 12 | 13 | if [[ "${BUILDKITE_PLUGIN_AWS_S3_SYNC_FOLLOW_SYMLINKS:-true}" == "false" ]]; then 14 | params+=(--no-follow-symlinks) 15 | fi 16 | 17 | if [[ -n "${BUILDKITE_PLUGIN_AWS_S3_SYNC_CACHE_CONTROL:-}" ]] && [[ $destination == s3://* ]]; then 18 | params+=("--cache-control=${BUILDKITE_PLUGIN_AWS_S3_SYNC_CACHE_CONTROL/\ /}") 19 | fi 20 | 21 | if [[ -n "${BUILDKITE_PLUGIN_AWS_S3_SYNC_ENDPOINT_URL:-}" ]]; then 22 | params+=("--endpoint-url=${BUILDKITE_PLUGIN_AWS_S3_SYNC_ENDPOINT_URL/\ /}") 23 | fi 24 | 25 | params+=("$source") 26 | params+=("$destination") 27 | 28 | echo "~~~ :s3: Syncing $source to $destination" 29 | aws s3 sync "${params[@]}" 30 | } 31 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: tests 3 | on: [push, pull_request] 4 | jobs: 5 | plugin-tests: 6 | name: Tests 7 | runs-on: ubuntu-latest 8 | container: 9 | image: buildkite/plugin-tester:latest 10 | volumes: 11 | - "${{github.workspace}}:/plugin" 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: tests 15 | run: bats tests/ 16 | plugin-lint: 17 | name: Lint 18 | runs-on: ubuntu-latest 19 | container: 20 | image: buildkite/plugin-linter:latest 21 | volumes: 22 | - "${{github.workspace}}:/plugin" 23 | steps: 24 | - uses: actions/checkout@v2 25 | - name: lint 26 | run: lint --id envato/aws-s3-sync 27 | plugin-shellcheck: 28 | name: Shellcheck 29 | runs-on: ubuntu-latest 30 | steps: 31 | - uses: actions/checkout@v2 32 | - name: Run ShellCheck 33 | uses: ludeeus/action-shellcheck@1.1.0 34 | with: 35 | check_together: 'yes' 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Envato 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | of the Software, and to permit persons to whom the Software is furnished to do 10 | so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS S3 Sync Buildkite Plugin 2 | 3 | [![tests](https://github.com/envato/aws-s3-sync-buildkite-plugin/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/envato/aws-s3-sync-buildkite-plugin/actions/workflows/tests.yml) 4 | [![MIT License](https://img.shields.io/badge/License-MIT-brightgreen.svg)](LICENSE) 5 | 6 | A [Buildkite plugin] that syncs files to the AWS Simple Storage Service (S3). It automatically detects when the local path is the source or destination, then syncs after or before the step `command` respectively. 7 | 8 | ## Example 9 | 10 | Sync local files source with s3 destination. This is run after the main command in a [`post-command` job hook](https://buildkite.com/docs/agent/v3/hooks#job-lifecycle-hooks). 11 | 12 | ```yml 13 | steps: 14 | - label: "Generate files and push to S3" 15 | command: bin/command-that-generates-files 16 | plugins: 17 | - envato/aws-s3-sync#v0.5.0: 18 | source: local-directory/ 19 | destination: s3://example-bucket/directory/ 20 | ``` 21 | 22 | Sync s3 files source with a local path destination. This is run before the main command in a [`pre-command` job hook](https://buildkite.com/docs/agent/v3/hooks#job-lifecycle-hooks). 23 | 24 | ```yml 25 | steps: 26 | - label: "Pull files from S3 and execute task" 27 | command: bin/command-that-uses-files 28 | plugins: 29 | - envato/aws-s3-sync#v0.5.0: 30 | source: s3://example-bucket/directory/ 31 | destination: local-directory/ 32 | ``` 33 | 34 | ## Configuration 35 | 36 | ### `source` 37 | 38 | The source location containing the local path or the s3 uri. 39 | 40 | ### `destination` 41 | 42 | The destination location containing the local path or the s3 uri. 43 | 44 | ### `delete` 45 | 46 | Defaults to `false` 47 | 48 | Files that exist in the destination but not in the source are deleted during sync. 49 | 50 | ### `follow-symlinks` 51 | 52 | Defaults to `true` 53 | 54 | Symbolic links are followed only when uploading to S3 from the local filesystem. 55 | 56 | ### `cache-control` 57 | 58 | Defaults to `null` 59 | 60 | Specify the cache control metadata value for all syncable objects 61 | 62 | ### `endpoint-url` 63 | 64 | Defaults to `null` 65 | 66 | Optionally specify an s3 endpoint URL to override the default. This option can be used to integrate with services that provide a s3 compatible api like CloudFlare R2. 67 | 68 | ## Development 69 | 70 | To run the tests: 71 | 72 | ```sh 73 | docker-compose run --rm tests 74 | ``` 75 | 76 | To run the [Buildkite Plugin Linter]: 77 | 78 | ```sh 79 | docker-compose run --rm lint 80 | ``` 81 | 82 | [Buildkite plugin]: https://buildkite.com/docs/agent/v3/plugins 83 | [Buildkite Plugin Linter]: https://github.com/buildkite-plugins/buildkite-plugin-linter 84 | -------------------------------------------------------------------------------- /tests/pre-command.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load '/usr/local/lib/bats/load.bash' 4 | 5 | # Uncomment the following to get more detail on failures of stubs 6 | # export AWS_STUB_DEBUG=/dev/tty 7 | 8 | @test "Syncs files from the source directory to the destination S3 bucket" { 9 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source 10 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=destination/ 11 | 12 | stub aws "s3 sync s3://source destination/ : echo s3 sync" 13 | 14 | run $PWD/hooks/pre-command 15 | 16 | assert_success 17 | assert_output --partial "s3 sync" 18 | unstub aws 19 | } 20 | 21 | @test "Syncs and deletes files" { 22 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source 23 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=destination/ 24 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DELETE=true 25 | 26 | stub aws "s3 sync --delete s3://source destination/ : echo s3 sync --delete" 27 | 28 | run $PWD/hooks/pre-command 29 | 30 | assert_success 31 | assert_output --partial "s3 sync --delete" 32 | unstub aws 33 | } 34 | 35 | @test "Doesn't follow symlinks" { 36 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source 37 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=destination/ 38 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_FOLLOW_SYMLINKS=false 39 | 40 | stub aws "s3 sync --no-follow-symlinks s3://source destination/ : echo s3 sync --no-follow-symlinks" 41 | 42 | run $PWD/hooks/pre-command 43 | 44 | assert_success 45 | assert_output --partial "s3 sync --no-follow-symlinks" 46 | unstub aws 47 | } 48 | 49 | @test "Uses endpoint URL" { 50 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source 51 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=destination/ 52 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_ENDPOINT_URL=https://myaccountid.r2.cloudflarestorage.com 53 | 54 | stub aws "s3 sync --endpoint-url=https://myaccountid.r2.cloudflarestorage.com s3://source destination/ : echo s3 sync --endpoint-url=https://myaccountid.r2.cloudflarestorage.com" 55 | 56 | run $PWD/hooks/pre-command 57 | 58 | assert_success 59 | assert_output --partial "s3 sync --endpoint-url=https://myaccountid.r2.cloudflarestorage.com" 60 | unstub aws 61 | } 62 | 63 | @test "Ignores cache control option" { 64 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source 65 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=destination/ 66 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_CACHE_CONTROL="public,max-age=315360000" 67 | 68 | stub aws "s3 sync s3://source destination/ : echo s3 sync" 69 | 70 | run $PWD/hooks/pre-command 71 | 72 | assert_success 73 | assert_output --partial "s3 sync" 74 | unstub aws 75 | } 76 | 77 | @test "Skips pre command when source is local" { 78 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 79 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 80 | 81 | run $PWD/hooks/pre-command 82 | 83 | assert_success 84 | } 85 | 86 | @test "Skips pre command when source and destination are both s3" { 87 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source/ 88 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 89 | 90 | run $PWD/hooks/pre-command 91 | 92 | assert_success 93 | } 94 | -------------------------------------------------------------------------------- /tests/post-command.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load '/usr/local/lib/bats/load.bash' 4 | 5 | # Uncomment the following to get more detail on failures of stubs 6 | # export AWS_STUB_DEBUG=/dev/tty 7 | 8 | @test "Syncs files from the source directory to the destination S3 bucket" { 9 | export BUILDKITE_COMMAND_EXIT_STATUS=0 10 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 11 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 12 | 13 | stub aws "s3 sync source/ s3://destination : echo s3 sync" 14 | 15 | run $PWD/hooks/post-command 16 | 17 | assert_success 18 | assert_output --partial "s3 sync" 19 | unstub aws 20 | } 21 | 22 | @test "Syncs and deletes files" { 23 | export BUILDKITE_COMMAND_EXIT_STATUS=0 24 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 25 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 26 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DELETE=true 27 | 28 | stub aws "s3 sync --delete source/ s3://destination : echo s3 sync --delete" 29 | 30 | run $PWD/hooks/post-command 31 | 32 | assert_success 33 | assert_output --partial "s3 sync --delete" 34 | unstub aws 35 | } 36 | 37 | @test "Syncs with cache control metadata" { 38 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 39 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 40 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_CACHE_CONTROL="public, max-age=315360000" 41 | 42 | stub aws "s3 sync --cache-control=public,max-age=315360000 source/ s3://destination : echo s3 sync --cache-control=public,max-age=315360000" 43 | 44 | run $PWD/hooks/post-command 45 | 46 | assert_success 47 | assert_output --partial "s3 sync --cache-control=public,max-age=315360000" 48 | unstub aws 49 | } 50 | 51 | @test "Doesn't follow symlinks" { 52 | export BUILDKITE_COMMAND_EXIT_STATUS=0 53 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 54 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 55 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_FOLLOW_SYMLINKS=false 56 | 57 | stub aws "s3 sync --no-follow-symlinks source/ s3://destination : echo s3 sync --no-follow-symlinks" 58 | 59 | run $PWD/hooks/post-command 60 | 61 | assert_success 62 | assert_output --partial "s3 sync --no-follow-symlinks" 63 | unstub aws 64 | } 65 | 66 | @test "Uses endpoint URL" { 67 | export BUILDKITE_COMMAND_EXIT_STATUS=0 68 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 69 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 70 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_ENDPOINT_URL=https://myaccountid.r2.cloudflarestorage.com 71 | 72 | stub aws "s3 sync --endpoint-url=https://myaccountid.r2.cloudflarestorage.com source/ s3://destination : echo s3 sync --endpoint-url=https://myaccountid.r2.cloudflarestorage.com" 73 | 74 | run $PWD/hooks/post-command 75 | 76 | assert_success 77 | assert_output --partial "s3 sync --endpoint-url=https://myaccountid.r2.cloudflarestorage.com" 78 | unstub aws 79 | } 80 | 81 | @test "Doesn't attempt to sync files if the step command fails" { 82 | export BUILDKITE_COMMAND_EXIT_STATUS=1 83 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=source/ 84 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=s3://destination 85 | 86 | run $PWD/hooks/post-command 87 | 88 | assert_output --partial "Skipping S3 sync because the command failed" 89 | assert_success 90 | } 91 | 92 | @test "Skips post command when source is s3" { 93 | export BUILDKITE_COMMAND_EXIT_STATUS=0 94 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_SOURCE=s3://source 95 | export BUILDKITE_PLUGIN_AWS_S3_SYNC_DESTINATION=destination/ 96 | 97 | run $PWD/hooks/post-command 98 | 99 | assert_success 100 | } 101 | --------------------------------------------------------------------------------