├── CODEOWNERS ├── shared └── scripts │ ├── apt-install-jq.sh │ ├── apt-update.sh │ ├── data-directories.sh │ ├── gvisor.sh │ └── apt-install-docker.sh ├── aws ├── scripts │ ├── dnf-update.sh │ ├── docker.sh │ ├── system-deps.sh │ ├── ssm-agent.sh │ └── cloudwatch-agent.sh ├── README.md └── configs │ └── amazon-cloudwatch-agent.json ├── .gitignore ├── gcp └── README.md ├── README.md ├── azure ├── README.md └── scripts │ └── azure-cli.sh ├── .github ├── pull_request_template.md └── workflows │ ├── ci.yml │ └── build_scheduled.yml ├── LICENSE ├── gcp.pkr.hcl ├── azure.pkr.hcl └── aws.pkr.hcl /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @spacelift-io/foundations-backend 2 | -------------------------------------------------------------------------------- /shared/scripts/apt-install-jq.sh: -------------------------------------------------------------------------------- 1 | # This script installs jq. 2 | 3 | sudo apt-get -y install jq 4 | -------------------------------------------------------------------------------- /aws/scripts/dnf-update.sh: -------------------------------------------------------------------------------- 1 | # This script updates the installed packages and package cache on the instance. 2 | 3 | sudo dnf update -y 4 | -------------------------------------------------------------------------------- /aws/scripts/docker.sh: -------------------------------------------------------------------------------- 1 | # This script installs and starts Docker. 2 | 3 | sudo dnf install -y docker 4 | sudo systemctl enable --now docker 5 | -------------------------------------------------------------------------------- /shared/scripts/apt-update.sh: -------------------------------------------------------------------------------- 1 | # This script updates the installed packages and package cache on the instance. 2 | 3 | sudo apt-get -y update 4 | -------------------------------------------------------------------------------- /aws/scripts/system-deps.sh: -------------------------------------------------------------------------------- 1 | # We need it for binary verification. 2 | sudo dnf swap -y gnupg2-minimal gnupg2-full 3 | 4 | # We need it for service management. 5 | sudo dnf install -y chkconfig 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Cache objects 2 | packer_cache/ 3 | 4 | # Crash log 5 | crash.log 6 | 7 | # For built boxes 8 | *.box 9 | 10 | # Ignore pkrvars files used for local testing 11 | *.pkrvars.hcl 12 | -------------------------------------------------------------------------------- /shared/scripts/data-directories.sh: -------------------------------------------------------------------------------- 1 | # This script creates directories Spacelift needs to work. 2 | 3 | # /opt/spacelift is Spacelift launcher's scratch space. 4 | sudo mkdir -p /opt/spacelift 5 | 6 | # /var/log/spacelift is where Spacelift launcher sends its logs. 7 | sudo mkdir -p /var/log/spacelift 8 | -------------------------------------------------------------------------------- /gcp/README.md: -------------------------------------------------------------------------------- 1 | # GCP Image 2 | 3 | ## Usage 4 | 5 | ### Build your own image 6 | 7 | ```shell 8 | git clone git@github.com:spacelift-io/spacelift-worker-image.git 9 | cd spacelift-worker-image 10 | packer build gcp.pkr.hcl 11 | ``` 12 | 13 | Override the defaults using `-var="zone=europe-west1-d"` 14 | 15 | The variables are located in the `gcp.pkr.hcl` file. 16 | -------------------------------------------------------------------------------- /aws/scripts/ssm-agent.sh: -------------------------------------------------------------------------------- 1 | CURRENTARCH=$(uname -m) 2 | if [ "$CURRENTARCH" = "aarch64" ] || [ "$CURRENTARCH" = "arm64" ]; then 3 | CURRENTARCH="arm64" 4 | else 5 | CURRENTARCH="amd64" 6 | fi 7 | 8 | sudo dnf install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_${CURRENTARCH}/amazon-ssm-agent.rpm 9 | 10 | sudo systemctl status amazon-ssm-agent 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spacelift-worker-image 2 | 3 | Building Spacelift-friendly images for private workers. 4 | 5 | ## Images 6 | 7 | You can find the list of published AMIs, Azure and GCP images on the [releases](https://github.com/spacelift-io/spacelift-worker-image/releases) page. 8 | 9 | ## Structure 10 | 11 | This repo contains packer templates for building AWS, Azure and GCP images. See the relevant section 12 | depending on the type of image you want to build: 13 | 14 | - [AWS](./aws/README.md) 15 | - [Azure](./azure/README.md) 16 | - [GCP](./gcp/README.md) 17 | -------------------------------------------------------------------------------- /shared/scripts/gvisor.sh: -------------------------------------------------------------------------------- 1 | # This script installs the gVisor. 2 | set -e 3 | 4 | CURRENTARCH=$(uname -m) 5 | URL=https://storage.googleapis.com/gvisor/releases/release/latest/${CURRENTARCH} 6 | 7 | curl --remote-name-all ${URL}/{runsc,runsc.sha512,containerd-shim-runsc-v1,containerd-shim-runsc-v1.sha512} 8 | sha512sum -c runsc.sha512 -c containerd-shim-runsc-v1.sha512 9 | rm -f *.sha512 10 | 11 | chmod a+rx runsc containerd-shim-runsc-v1 12 | sudo mv runsc containerd-shim-runsc-v1 /usr/local/bin 13 | 14 | sudo /usr/local/bin/runsc install -- --fsgofer-host-uds 15 | sudo systemctl restart docker 16 | -------------------------------------------------------------------------------- /azure/README.md: -------------------------------------------------------------------------------- 1 | # Azure Image 2 | 3 | ## Usage 4 | 5 | ### Build your own image 6 | 7 | ```shell 8 | git clone git@github.com:spacelift-io/spacelift-worker-image.git 9 | cd spacelift-worker-image 10 | packer build azure.pkr.hcl 11 | ``` 12 | 13 | Override the defaults using `-var="variable-name=variable-value"` 14 | 15 | The variables are located in the `azure.pkr.hcl` file. 16 | 17 | ### Shared Image Gallery 18 | 19 | By default, the template creates a Managed image in the resource group defined by the `image_resource_group` 20 | variable. The image can optionally be published to a shared image gallery by setting the `gallery_*` 21 | variables. 22 | -------------------------------------------------------------------------------- /shared/scripts/apt-install-docker.sh: -------------------------------------------------------------------------------- 1 | # This script installs and starts Docker. 2 | 3 | sudo apt-get -y install \ 4 | apt-transport-https \ 5 | ca-certificates \ 6 | curl \ 7 | gnupg \ 8 | lsb-release 9 | 10 | # Add Docker's GPG key 11 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg 12 | 13 | # Setup the Docker APT repository 14 | echo \ 15 | "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ 16 | $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 17 | 18 | sudo apt-get -y update 19 | 20 | sudo apt-get -y install docker-ce docker-ce-cli containerd.io 21 | -------------------------------------------------------------------------------- /azure/scripts/azure-cli.sh: -------------------------------------------------------------------------------- 1 | # This script installs the Azure CLI 2 | 3 | sudo apt-get install \ 4 | ca-certificates \ 5 | curl \ 6 | apt-transport-https \ 7 | lsb-release \ 8 | gnupg 9 | 10 | sudo mkdir -p /etc/apt/keyrings 11 | 12 | curl -sLS https://packages.microsoft.com/keys/microsoft.asc | 13 | gpg --dearmor | sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null 14 | 15 | sudo chmod go+r /etc/apt/keyrings/microsoft.gpg 16 | 17 | AZ_DIST=$(lsb_release -cs) 18 | echo "Types: deb 19 | URIs: https://packages.microsoft.com/repos/azure-cli/ 20 | Suites: ${AZ_DIST} 21 | Components: main 22 | Architectures: $(dpkg --print-architecture) 23 | Signed-by: /etc/apt/keyrings/microsoft.gpg" | sudo tee /etc/apt/sources.list.d/azure-cli.sources 24 | 25 | sudo apt-get update 26 | sudo apt-get install azure-cli 27 | -------------------------------------------------------------------------------- /aws/scripts/cloudwatch-agent.sh: -------------------------------------------------------------------------------- 1 | CONFIG_DESTINATION=/opt/aws/amazon-cloudwatch-agent/bin/config.json 2 | CONFIG_SOURCE=/tmp/amazon-cloudwatch-agent.json 3 | 4 | CURRENTARCH=$(uname -m) 5 | if [ "$CURRENTARCH" = "aarch64" ] || [ "$CURRENTARCH" = "arm64" ]; then 6 | CURRENTARCH="arm64" 7 | else 8 | CURRENTARCH="amd64" 9 | fi 10 | 11 | DOWNLOAD_URL=https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/${CURRENTARCH}/latest/amazon-cloudwatch-agent.rpm 12 | RPM_PATH=/tmp/amazon-cloudwatch-agent.rpm 13 | 14 | sudo touch /var/log/spacelift/{info,error}.log 15 | 16 | curl $DOWNLOAD_URL --output $RPM_PATH 17 | sudo rpm -U $RPM_PATH 18 | rm $RPM_PATH 19 | sudo mv ${CONFIG_SOURCE} ${CONFIG_DESTINATION} 20 | sudo /opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:${CONFIG_DESTINATION} 21 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description of the change 2 | 3 | > Description here 4 | 5 | ## Type of change 6 | 7 | - [ ] Bug fix (non-breaking change that fixes an issue); 8 | - [ ] New feature (non-breaking change that adds functionality); 9 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected); 10 | - [ ] Documentation (a documentation or example fix not affecting the infrastructure managed by this module); 11 | 12 | ## Checklists 13 | 14 | ### Development 15 | 16 | - [ ] All necessary variables have been defined, with defaults if applicable; 17 | - [ ] The HCL code is formatted; 18 | - [ ] An AMI has been created in some AWS account, and the AMI is working as expected; 19 | 20 | ### Code review 21 | 22 | - [ ] This pull request has a descriptive title and information useful to a reviewer. There may be a screenshot or screencast attached; 23 | - [ ] This pull request is no longer marked as "draft"; 24 | - [ ] Reviewers have been assigned; 25 | - [ ] Changes have been reviewed by at least one other engineer; 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Spacelift 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /aws/README.md: -------------------------------------------------------------------------------- 1 | # AWS Image 2 | 3 | ## Usage 4 | 5 | ### Use an official AMI 6 | 7 | Find the latest AMIs in the [releases](https://github.com/spacelift-io/spacelift-worker-image/releases) section 8 | 9 | #### awscli 10 | 11 | Use the `awscli` to get the latest AMI 12 | 13 | ```shell 14 | aws ec2 describe-images \ 15 | --owners 643313122712 \ 16 | --filters "Name=name,Values=spacelift-*" "Name=architecture,Values=x86_64" \ 17 | --query 'sort_by(Images, &CreationDate)[-1]' 18 | ``` 19 | 20 | Architecture could be either `x86_64` or `arm64`. 21 | 22 | #### Terraform 23 | 24 | Use a terraform data source to retrieve the latest AMI 25 | 26 | ```hcl 27 | provider "aws" { 28 | region = "us-east-2" 29 | } 30 | 31 | data "aws_ami" "spacelift" { 32 | most_recent = true 33 | owners = ["643313122712"] # spacelift owner 34 | 35 | filter { 36 | name = "name" 37 | values = ["spacelift-*"] 38 | } 39 | 40 | filter { 41 | name = "root-device-type" 42 | values = ["ebs"] 43 | } 44 | 45 | filter { 46 | name = "virtualization-type" 47 | values = ["hvm"] 48 | } 49 | } 50 | 51 | output "ami" { 52 | value = data.aws_ami.spacelift.image_id 53 | } 54 | ``` 55 | 56 | ### Build your own AMI 57 | 58 | ```shell 59 | git clone git@github.com:spacelift-io/spacelift-worker-image.git 60 | cd spacelift-worker-image 61 | packer build aws.pkr.hcl 62 | ``` 63 | 64 | Override the defaults using `-var="region=us-east-2"` 65 | 66 | The variables are located in the `aws.pkr.hcl` file. 67 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Validate the Packer templates 2 | 3 | on: 4 | push: 5 | branches-ignore: [main] 6 | 7 | concurrency: 8 | group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} 9 | cancel-in-progress: true 10 | 11 | jobs: 12 | build: 13 | name: 👷 ${{ matrix.cloud }} 14 | strategy: 15 | matrix: 16 | cloud: [aws, azure, gcp] 17 | runs-on: ubuntu-latest 18 | permissions: 19 | id-token: write 20 | contents: read 21 | env: 22 | # AWS 23 | PKR_VAR_encrypt_boot: false 24 | # GCP 25 | PKR_VAR_project_id: spacelift-workers 26 | PKR_VAR_credentials_json: ${{ secrets.GCP_CREDENTIALS_JSON }} 27 | PKR_VAR_image_base_name: spacelift-worker 28 | PKR_VAR_image_family: spacelift-worker 29 | # Azure 30 | PKR_VAR_client_id: "433d3ca3-1866-4dfa-b9bf-65d6c4391ec7" 31 | PKR_VAR_tenant_id: ${{ secrets.AZURE_TENANT_ID }} 32 | PKR_VAR_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 33 | PKR_VAR_image_resource_group: rg-worker_images-public-westeurope 34 | PKR_VAR_packer_work_group: rg-worker_images_packer-public-westeurope 35 | PKR_VAR_gallery_resource_group: rg-worker_images-public-westeurope 36 | PKR_VAR_gallery_name: spacelift_worker_images_public 37 | PKR_VAR_gallery_image_name: spacelift_worker_image_ubu_lts 38 | PKR_VAR_gallery_replication_regions: '["westeurope", "northeurope", "northcentralus", "centralus", "eastus", "eastus2", "westus2", "westus3", "australiaeast", "southcentralus"]' 39 | PKR_VAR_gallery_image_version: 3.0.${{ github.run_number }} 40 | 41 | steps: 42 | - name: Check out the source code 43 | uses: actions/checkout@main 44 | 45 | - name: Export suffix for GCP 46 | if: matrix.cloud == 'gcp' 47 | run: | 48 | echo "PKR_VAR_suffix=$(date +%s)-$(cat /dev/urandom | tr -dc 'a-z0-9' | head -c 8)" >> $GITHUB_ENV 49 | 50 | - name: Setup packer 51 | uses: hashicorp/setup-packer@main 52 | with: 53 | version: latest 54 | 55 | - name: Initialize Packer 56 | run: packer init ${{ matrix.cloud }}.pkr.hcl 57 | env: 58 | PACKER_GITHUB_API_TOKEN: "${{ github.token }}" 59 | 60 | - name: Validate the Packer template 61 | id: validate 62 | run: packer validate ${{ matrix.cloud }}.pkr.hcl 63 | -------------------------------------------------------------------------------- /gcp.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | googlecompute = { 4 | source = "github.com/hashicorp/googlecompute" 5 | version = "~> 1" 6 | } 7 | } 8 | } 9 | 10 | variable "credentials_json" { 11 | type = string 12 | default = null 13 | } 14 | 15 | variable "project_id" { 16 | type = string 17 | default = null 18 | } 19 | 20 | variable "image_base_name" { 21 | type = string 22 | default = "spacelift-private-worker" 23 | } 24 | 25 | variable "image_family" { 26 | type = string 27 | default = "spacelift-private-worker" 28 | } 29 | 30 | variable "image_storage_location" { 31 | type = string 32 | default = "us" 33 | } 34 | 35 | variable "source_image_family" { 36 | type = string 37 | default = "ubuntu-2404-lts-amd64" 38 | } 39 | 40 | variable "suffix" { 41 | type = string 42 | description = "A suffix to add to image names to ensure each version is unique. For example a timestamp or version number." 43 | } 44 | 45 | variable "machine_type" { 46 | type = string 47 | default = "n1-standard-2" 48 | } 49 | 50 | variable "additional_labels" { 51 | type = map(string) 52 | default = {} 53 | } 54 | 55 | variable "zone" { 56 | type = string 57 | default = "us-central1-a" 58 | } 59 | 60 | source "googlecompute" "spacelift" { 61 | project_id = var.project_id 62 | source_image_family = var.source_image_family 63 | ssh_username = "spacelift" 64 | ssh_clear_authorized_keys = true 65 | zone = var.zone 66 | disk_size = 50 67 | machine_type = var.machine_type 68 | credentials_json = var.credentials_json 69 | 70 | image_name = "${var.image_base_name}-${var.image_storage_location}-${var.suffix}" 71 | image_family = var.image_family 72 | image_storage_locations = [var.image_storage_location] 73 | } 74 | 75 | build { 76 | sources = ["source.googlecompute.spacelift"] 77 | 78 | provisioner "shell" { 79 | scripts = [ 80 | "shared/scripts/data-directories.sh", 81 | "shared/scripts/apt-update.sh", 82 | "shared/scripts/apt-install-docker.sh", 83 | "shared/scripts/gvisor.sh", 84 | "shared/scripts/apt-install-jq.sh", 85 | ] 86 | 87 | env = { 88 | DEBIAN_FRONTEND = "noninteractive" 89 | } 90 | } 91 | 92 | post-processor "manifest" { 93 | output = "manifest_gcp.json" 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /azure.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | azure = { 4 | source = "github.com/hashicorp/azure" 5 | version = "~> 2" 6 | } 7 | } 8 | } 9 | 10 | variable "client_id" { 11 | type = string 12 | default = "" 13 | } 14 | 15 | variable "oidc_request_url" { 16 | type = string 17 | default = env("ACTIONS_ID_TOKEN_REQUEST_URL") // Github built-in variable 18 | } 19 | 20 | variable "oidc_request_token" { 21 | type = string 22 | default = env("ACTIONS_ID_TOKEN_REQUEST_TOKEN") // Github built-in variable 23 | } 24 | 25 | variable "subscription_id" { 26 | type = string 27 | } 28 | 29 | variable "tenant_id" { 30 | type = string 31 | default = "" 32 | } 33 | 34 | variable "image_name" { 35 | type = string 36 | default = "spacelift-{{ timestamp }}" 37 | } 38 | 39 | variable "image_resource_group" { 40 | type = string 41 | } 42 | 43 | variable "gallery_resource_group" { 44 | type = string 45 | default = null 46 | } 47 | 48 | variable "gallery_name" { 49 | type = string 50 | } 51 | 52 | variable "gallery_image_name" { 53 | type = string 54 | default = null 55 | } 56 | 57 | variable "gallery_image_version" { 58 | type = string 59 | default = null 60 | } 61 | 62 | variable "gallery_replication_regions" { 63 | type = list(string) 64 | default = null 65 | } 66 | 67 | variable "source_image_publisher" { 68 | type = string 69 | default = "Canonical" 70 | } 71 | 72 | variable "source_image_offer" { 73 | type = string 74 | default = "ubuntu-24_04-lts" 75 | } 76 | 77 | variable "source_image_sku" { 78 | type = string 79 | default = "minimal" 80 | } 81 | 82 | variable "vm_size" { 83 | type = string 84 | default = "Standard_B2S" 85 | } 86 | 87 | variable "additional_tags" { 88 | type = map(string) 89 | default = {} 90 | } 91 | 92 | variable "packer_work_group" { 93 | type = string 94 | default = "" 95 | description = "The resource group for Packer to use while building the VM" 96 | } 97 | 98 | source "azure-arm" "spacelift" { 99 | client_id = var.client_id 100 | subscription_id = var.subscription_id 101 | tenant_id = var.tenant_id 102 | oidc_request_url = var.oidc_request_url 103 | oidc_request_token = var.oidc_request_token 104 | 105 | managed_image_name = var.image_name 106 | managed_image_resource_group_name = var.image_resource_group 107 | 108 | shared_image_gallery_destination { 109 | subscription = var.subscription_id 110 | resource_group = var.gallery_resource_group 111 | gallery_name = var.gallery_name 112 | image_name = var.gallery_image_name 113 | image_version = var.gallery_image_version 114 | 115 | dynamic target_region { 116 | for_each = var.gallery_replication_regions 117 | 118 | content { 119 | name = target_region.value 120 | } 121 | } 122 | } 123 | 124 | os_type = "Linux" 125 | 126 | image_publisher = var.source_image_publisher 127 | image_offer = var.source_image_offer 128 | image_sku = var.source_image_sku 129 | 130 | build_resource_group_name = var.packer_work_group 131 | 132 | vm_size = var.vm_size 133 | 134 | ssh_clear_authorized_keys = true 135 | 136 | azure_tags = merge(var.additional_tags, { 137 | Name = "Spacelift Worker Image" 138 | SourceImagePublisher = var.source_image_publisher 139 | SourceImageOffer = var.source_image_offer 140 | SourceImageSku = var.source_image_sku 141 | CreatedAt = "{{ timestamp }}" 142 | }) 143 | } 144 | 145 | build { 146 | sources = ["source.azure-arm.spacelift"] 147 | 148 | provisioner "shell" { 149 | scripts = [ 150 | "shared/scripts/data-directories.sh", 151 | "shared/scripts/apt-update.sh", 152 | "shared/scripts/apt-install-docker.sh", 153 | "shared/scripts/gvisor.sh", 154 | "shared/scripts/apt-install-jq.sh", 155 | "azure/scripts/azure-cli.sh", 156 | ] 157 | 158 | env = { 159 | DEBIAN_FRONTEND = "noninteractive" 160 | } 161 | } 162 | 163 | # Deprovision VM 164 | provisioner "shell" { 165 | execute_command = "chmod +x {{ .Path }}; {{ .Vars }} sudo -E sh '{{ .Path }}'" 166 | inline = [ 167 | "/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync" 168 | ] 169 | inline_shebang = "/bin/sh -x" 170 | } 171 | 172 | post-processor "manifest" { 173 | output = "manifest_azure.json" 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /aws.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | amazon = { 4 | source = "github.com/hashicorp/amazon" 5 | version = "~> 1" 6 | } 7 | 8 | amazon-ami-management = { 9 | version = "= 1.6.1" 10 | source = "github.com/wata727/amazon-ami-management" 11 | } 12 | } 13 | } 14 | 15 | variable "ami_name_prefix" { 16 | type = string 17 | default = "spacelift-{{timestamp}}" 18 | } 19 | 20 | variable "ami_regions" { 21 | type = list(string) 22 | default = [ 23 | "ap-northeast-1", 24 | "ap-northeast-2", 25 | "ap-northeast-3", 26 | "ap-southeast-1", 27 | "ap-southeast-2", 28 | "ap-south-1", 29 | "ca-central-1", 30 | "eu-central-1", 31 | "eu-central-2", 32 | "eu-west-1", 33 | "eu-west-2", 34 | "eu-west-3", 35 | "eu-north-1", 36 | "sa-east-1", 37 | "us-east-1", 38 | "us-east-2", 39 | "us-west-1", 40 | "us-west-2", 41 | ] 42 | } 43 | 44 | variable "source_ami_architecture" { 45 | type = string 46 | default = "x86_64" 47 | } 48 | 49 | variable "source_ami_owners" { 50 | type = list(string) 51 | default = ["137112412989"] # defaults to Amazon for Amazon Linux, see https://docs.aws.amazon.com/AmazonECR/latest/userguide/amazon_linux_container_image.html 52 | } 53 | 54 | variable "ami_groups" { 55 | type = list(string) 56 | default = ["all"] 57 | } 58 | 59 | variable "instance_type" { 60 | type = string 61 | default = "t3.micro" 62 | } 63 | 64 | variable "encrypt_boot" { 65 | type = bool 66 | default = true 67 | } 68 | 69 | variable "shared_credentials_file" { 70 | type = string 71 | default = null 72 | } 73 | 74 | variable "subnet_filter" { 75 | type = map(string) 76 | default = null 77 | } 78 | 79 | variable "additional_tags" { 80 | type = map(string) 81 | default = {} 82 | } 83 | 84 | variable "region" { 85 | type = string 86 | default = "us-east-1" 87 | } 88 | 89 | variable "vpc_id" { 90 | type = string 91 | default = null 92 | } 93 | 94 | source "amazon-ebs" "spacelift" { 95 | source_ami_filter { 96 | filters = { 97 | virtualization-type = "hvm" 98 | name = "al2023-ami-minimal-*-kernel-6.1-${var.source_ami_architecture}" 99 | root-device-type = "ebs" 100 | architecture = var.source_ami_architecture 101 | } 102 | owners = var.source_ami_owners 103 | most_recent = true 104 | } 105 | 106 | launch_block_device_mappings { 107 | device_name = "/dev/xvda" 108 | volume_size = 8 109 | volume_type = "gp3" 110 | delete_on_termination = true 111 | } 112 | 113 | ami_name = "${var.ami_name_prefix}-${var.source_ami_architecture}" 114 | ami_regions = var.ami_regions 115 | ami_groups = var.ami_groups 116 | ami_description = <> $GITHUB_OUTPUT 19 | 20 | aws: 21 | # Since we run in parallel, let's make sure we use the same timestamp for all jobs 22 | needs: timestamp 23 | strategy: 24 | matrix: 25 | arch: [x86_64, arm64] 26 | name: Build the AWS AMI using Packer 27 | runs-on: ubuntu-latest 28 | permissions: 29 | id-token: write 30 | contents: read 31 | steps: 32 | - name: Check out the source code 33 | uses: actions/checkout@main 34 | 35 | - name: Configure AWS credentials 36 | uses: aws-actions/configure-aws-credentials@v5 37 | with: 38 | aws-region: ${{ secrets.AWS_REGION }} 39 | role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} 40 | role-duration-seconds: 3600 41 | 42 | - name: Setup packer 43 | uses: hashicorp/setup-packer@main 44 | with: 45 | version: latest 46 | 47 | - name: Initialize Packer 48 | run: packer init aws.pkr.hcl 49 | env: 50 | PACKER_GITHUB_API_TOKEN: "${{ github.token }}" 51 | 52 | - name: Build the AWS AMI using Packer (${{ matrix.arch }}) 53 | # We only run the cleanup postprocessor for one of them, to avoid race conditions 54 | run: packer build ${{ matrix.arch == 'x86_64' && '--except=amazon-ami-management' || '' }} aws.pkr.hcl 55 | env: 56 | PKR_VAR_encrypt_boot: false 57 | PKR_VAR_ami_name_prefix: spacelift-${{ needs.timestamp.outputs.timestamp }} 58 | PKR_VAR_source_ami_architecture: ${{ matrix.arch }} 59 | PKR_VAR_instance_type: ${{ matrix.arch == 'x86_64' && 't3.micro' || 't4g.micro' }} 60 | 61 | - name: Upload manifest 62 | uses: actions/upload-artifact@v5 63 | with: 64 | path: manifest_aws_${{ matrix.arch }}.json 65 | name: manifest_aws_${{ matrix.arch }}.json 66 | retention-days: 5 67 | 68 | aws-govcloud: 69 | # Since we run in parallel, let's make sure we use the same timestamp for all jobs 70 | needs: timestamp 71 | strategy: 72 | matrix: 73 | arch: [x86_64, arm64] 74 | name: Build the AWS (GovCloud) AMI using Packer 75 | runs-on: ubuntu-latest 76 | permissions: 77 | id-token: write 78 | contents: read 79 | steps: 80 | - name: Check out the source code 81 | uses: actions/checkout@main 82 | 83 | - name: Configure GovCloud AWS credentials 84 | uses: aws-actions/configure-aws-credentials@v5 85 | with: 86 | aws-region: ${{ secrets.GOVCLOUD_AWS_REGION }} 87 | role-to-assume: ${{ secrets.GOVCLOUD_AWS_ROLE_ARN }} 88 | role-duration-seconds: 3600 89 | 90 | - name: Setup packer 91 | uses: hashicorp/setup-packer@main 92 | with: 93 | version: latest 94 | 95 | - name: Initialize Packer 96 | run: packer init aws.pkr.hcl 97 | env: 98 | PACKER_GITHUB_API_TOKEN: "${{ github.token }}" 99 | 100 | - name: Build the GovCloud AWS AMI using Packer (${{ matrix.arch }}) 101 | # We only run the cleanup postprocessor for one of them, to avoid race conditions 102 | run: packer build ${{ matrix.arch == 'x86_64' && '--except=amazon-ami-management' || '' }} aws.pkr.hcl 103 | env: 104 | PKR_VAR_source_ami_owners: '["045324592363"]' 105 | PKR_VAR_region: us-gov-east-1 106 | PKR_VAR_ami_regions: '["us-gov-east-1", "us-gov-west-1"]' 107 | PKR_VAR_encrypt_boot: false 108 | PKR_VAR_ami_name_prefix: spacelift-${{ needs.timestamp.outputs.timestamp }} 109 | PKR_VAR_source_ami_architecture: ${{ matrix.arch }} 110 | PKR_VAR_instance_type: ${{ matrix.arch == 'x86_64' && 't3.micro' || 't4g.micro' }} 111 | 112 | - name: Rename the manifest file 113 | run: | 114 | mv manifest_aws_${{ matrix.arch }}.json manifest_aws_govcloud_${{ matrix.arch }}.json 115 | 116 | - name: Upload manifest 117 | uses: actions/upload-artifact@v5 118 | with: 119 | path: manifest_aws_govcloud_${{ matrix.arch }}.json 120 | name: manifest_aws_govcloud_${{ matrix.arch }}.json 121 | retention-days: 5 122 | 123 | azure: 124 | name: Build Azure AMI using Packer 125 | runs-on: ubuntu-latest 126 | permissions: 127 | id-token: write 128 | contents: read 129 | env: 130 | PKR_VAR_client_id: "976e4a6e-c619-417e-9add-50e2d674e2db" 131 | PKR_VAR_tenant_id: ${{ secrets.AZURE_TENANT_ID }} 132 | PKR_VAR_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 133 | PKR_VAR_image_resource_group: rg-worker_images-public-westeurope 134 | PKR_VAR_packer_work_group: rg-worker_images_packer-public-westeurope 135 | PKR_VAR_gallery_resource_group: rg-worker_images-public-westeurope 136 | PKR_VAR_gallery_name: spacelift_worker_images_public 137 | PKR_VAR_gallery_image_name: spacelift_worker_image_ubu_lts 138 | PKR_VAR_gallery_replication_regions: '["westeurope", "northeurope", "northcentralus", "centralus", "eastus", "eastus2", "westus2", "westus3", "australiaeast", "southcentralus"]' 139 | PKR_VAR_gallery_image_version: 3.0.${{ github.run_number }} 140 | 141 | steps: 142 | - name: Check out the source code 143 | uses: actions/checkout@main 144 | 145 | - name: Setup packer 146 | uses: hashicorp/setup-packer@main 147 | with: 148 | version: latest 149 | 150 | - name: Initialize Packer 151 | run: packer init azure.pkr.hcl 152 | env: 153 | PACKER_GITHUB_API_TOKEN: "${{ github.token }}" 154 | 155 | - name: Azure => Build the AMI using Packer 156 | run: packer build azure.pkr.hcl 157 | 158 | - name: Upload manifest 159 | uses: actions/upload-artifact@v5 160 | with: 161 | path: manifest_azure.json 162 | name: manifest_azure.json 163 | retention-days: 5 164 | 165 | - name: Export Azure version number 166 | id: export_azure_version 167 | run: | 168 | echo "azure_version=$PKR_VAR_gallery_image_version" >> $GITHUB_OUTPUT 169 | 170 | outputs: 171 | azure_version: ${{ steps.export_azure_version.outputs.azure_version }} 172 | 173 | gcp: 174 | name: Build GCP AMI using Packer 175 | runs-on: ubuntu-latest 176 | permissions: 177 | id-token: write 178 | contents: read 179 | env: 180 | PKR_VAR_project_id: spacelift-workers 181 | PKR_VAR_credentials_json: ${{ secrets.GCP_CREDENTIALS_JSON }} 182 | PKR_VAR_image_base_name: spacelift-worker 183 | PKR_VAR_image_family: spacelift-worker 184 | 185 | steps: 186 | - name: Check out the source code 187 | uses: actions/checkout@main 188 | 189 | - name: Set up Google Cloud SDK 190 | uses: google-github-actions/setup-gcloud@v3 191 | 192 | - name: Authenticate with GCP 193 | uses: google-github-actions/auth@v3 194 | with: 195 | credentials_json: ${{ secrets.GCP_CREDENTIALS_JSON }} 196 | 197 | - name: Export suffix for GCP 198 | run: | 199 | echo "PKR_VAR_suffix=$(date +%s)-$(cat /dev/urandom | tr -dc 'a-z0-9' | head -c 8)" >> $GITHUB_ENV 200 | 201 | - name: Setup packer 202 | uses: hashicorp/setup-packer@main 203 | with: 204 | version: latest 205 | 206 | - name: Initialize Packer 207 | run: packer init gcp.pkr.hcl 208 | env: 209 | PACKER_GITHUB_API_TOKEN: "${{ github.token }}" 210 | 211 | - name: GCP => Build the AMI using Packer for US 212 | run: packer build gcp.pkr.hcl 213 | env: 214 | PKR_VAR_image_storage_location: us 215 | PKR_VAR_zone: us-central1-a 216 | 217 | - name: GCP => Build the AMI using Packer for EU 218 | run: packer build gcp.pkr.hcl 219 | env: 220 | PKR_VAR_image_storage_location: eu 221 | PKR_VAR_zone: europe-west1-d 222 | 223 | - name: GCP => Build the AMI using Packer for Asia 224 | run: packer build gcp.pkr.hcl 225 | env: 226 | PKR_VAR_image_storage_location: asia 227 | PKR_VAR_zone: asia-northeast2-a 228 | 229 | - name: GCP => Add IAM policy binding to the Compute Engine images 230 | run: | 231 | gcloud compute images add-iam-policy-binding ${PKR_VAR_image_base_name}-us-${PKR_VAR_suffix} --member='allAuthenticatedUsers' --role='roles/compute.imageUser' 232 | gcloud compute images add-iam-policy-binding ${PKR_VAR_image_base_name}-eu-${PKR_VAR_suffix} --member='allAuthenticatedUsers' --role='roles/compute.imageUser' 233 | gcloud compute images add-iam-policy-binding ${PKR_VAR_image_base_name}-asia-${PKR_VAR_suffix} --member='allAuthenticatedUsers' --role='roles/compute.imageUser' 234 | 235 | - name: Upload manifest 236 | uses: actions/upload-artifact@v5 237 | with: 238 | path: manifest_gcp.json 239 | name: manifest_gcp.json 240 | retention-days: 5 241 | 242 | gh-release: 243 | needs: [aws, aws-govcloud, azure, gcp] 244 | name: Create tag & publish GitHub release 245 | runs-on: ubuntu-latest 246 | steps: 247 | # Technically, we don't need the source code but the git tagging action requires it 248 | - name: Checkout source code 249 | uses: actions/checkout@main 250 | 251 | - name: Bump version and push tag 252 | id: tag_version 253 | uses: mathieudutour/github-tag-action@v6.2 254 | with: 255 | github_token: ${{ secrets.GITHUB_TOKEN }} 256 | create_annotated_tag: true 257 | tag_prefix: "" 258 | default_bump: minor 259 | 260 | - name: Download AWS x64 manifest 261 | uses: actions/download-artifact@v6 262 | with: 263 | name: manifest_aws_x86_64.json 264 | 265 | - name: Download AWS arm64 manifest 266 | uses: actions/download-artifact@v6 267 | with: 268 | name: manifest_aws_arm64.json 269 | 270 | # The manifest file look like this: 271 | # "builds": [ 272 | # { 273 | # "name": "spacelift", 274 | # "builder_type": "amazon-ebs", 275 | # "build_time": 1698670371, 276 | # "files": null, 277 | # "artifact_id": "ap-northeast-1:ami-0facbd2b91807c339,ap-northeast-2:ami-03849b8d23619dfb2,... 278 | # } 279 | # ] 280 | 281 | - name: Write AWS AMI IDs to a markdown file 282 | uses: actions/github-script@v8 283 | with: 284 | script: | 285 | const fs = require("fs"); 286 | 287 | var content = fs.readFileSync("./manifest_aws_arm64.json", "utf8"); 288 | var manifest = JSON.parse(content); 289 | 290 | const toPrint = []; 291 | manifest["builds"].forEach((build) => { 292 | const regionToAmi = build["artifact_id"].split(","); 293 | regionToAmi.forEach((regionToAmi) => { 294 | const [region, ami] = regionToAmi.split(":"); 295 | toPrint.push(`| ${region} | ${ami} |`); 296 | }); 297 | }); 298 | 299 | content = fs.readFileSync("./manifest_aws_x86_64.json", "utf8"); 300 | manifest = JSON.parse(content); 301 | 302 | manifest["builds"].forEach((build) => { 303 | const regionToAmi = build["artifact_id"].split(","); 304 | regionToAmi.forEach((regionToAmi, i) => { 305 | const [region, ami] = regionToAmi.split(":"); 306 | toPrint[i] = toPrint[i] + ` ${ami} |`; 307 | }); 308 | }); 309 | 310 | const header = [ 311 | "## AWS", 312 | "", 313 | "| AWS Region | AMI ID (ARM64) | AMI ID (x86_64) |", 314 | "|------------------|-------------------------|-------------------------|", 315 | ] 316 | fs.writeFileSync("./body.md", header.join("\n") + "\n" + toPrint.join("\n")); 317 | 318 | - name: Download GovCloud AWS x64 manifest 319 | uses: actions/download-artifact@v6 320 | with: 321 | name: manifest_aws_govcloud_x86_64.json 322 | 323 | - name: Download GovCloud AWS arm64 manifest 324 | uses: actions/download-artifact@v6 325 | with: 326 | name: manifest_aws_govcloud_arm64.json 327 | 328 | - name: Write GovCloud AWS AMI IDs to the markdown file 329 | uses: actions/github-script@v8 330 | with: 331 | script: | 332 | const fs = require("fs"); 333 | 334 | var content = fs.readFileSync("./manifest_aws_govcloud_arm64.json", "utf8"); 335 | var manifest = JSON.parse(content); 336 | 337 | const toPrint = []; 338 | manifest["builds"].forEach((build) => { 339 | const regionToAmi = build["artifact_id"].split(","); 340 | regionToAmi.forEach((regionToAmi) => { 341 | const [region, ami] = regionToAmi.split(":"); 342 | toPrint.push(`| ${region} | ${ami} |`); 343 | }); 344 | }); 345 | 346 | content = fs.readFileSync("./manifest_aws_govcloud_x86_64.json", "utf8"); 347 | manifest = JSON.parse(content); 348 | 349 | manifest["builds"].forEach((build) => { 350 | const regionToAmi = build["artifact_id"].split(","); 351 | regionToAmi.forEach((regionToAmi, i) => { 352 | const [region, ami] = regionToAmi.split(":"); 353 | toPrint[i] = toPrint[i] + ` ${ami} |`; 354 | }); 355 | }); 356 | 357 | const header = [ 358 | "## AWS GovCloud", 359 | "", 360 | "| AWS Region | AMI ID (ARM64) | AMI ID (x86_64) |", 361 | "|------------|-------------------------|-------------------------|", 362 | ] 363 | fs.appendFileSync("./body.md", "\n\n" + header.join("\n") + "\n" + toPrint.join("\n")); 364 | 365 | - name: Download Google Cloud manifest 366 | uses: actions/download-artifact@v6 367 | with: 368 | name: manifest_gcp.json 369 | 370 | # The GCP manifest file look like this: 371 | # "builds": [ 372 | # { 373 | # "name": "spacelift", 374 | # "builder_type": "googlecompute", 375 | # "build_time": 1700479054, 376 | # "files": null, 377 | # "artifact_id": "spacelift-worker-us-1700478848-305dsvij", 378 | # "packer_run_uuid": "cdc82943-986b-5ab9-6ce1-9024ca0ebb6a", 379 | # "custom_data": null 380 | # }, 381 | # { 382 | # "name": "spacelift", 383 | # "builder_type": "googlecompute", 384 | # "build_time": 1700479263, 385 | # "files": null, 386 | # "artifact_id": "spacelift-worker-eu-1700478848-305dsvij", 387 | # "packer_run_uuid": "2440c9b1-a342-3606-2661-6e5389bdffc6", 388 | # "custom_data": null 389 | # } 390 | # ] 391 | 392 | - name: Write Azure and GCP AMI IDs to the markdown file 393 | uses: actions/github-script@v8 394 | env: 395 | AZURE_VERSION: ${{ needs.azure.outputs.azure_version }} 396 | with: 397 | script: | 398 | const fs = require("fs"); 399 | 400 | content = fs.readFileSync("./manifest_gcp.json", "utf8"); 401 | manifest = JSON.parse(content); 402 | 403 | const gcpLinesToPrint = []; 404 | 405 | manifest["builds"].forEach((build) => { 406 | artifact = build["artifact_id"]; 407 | if (artifact.indexOf("-us-") > 0) { 408 | gcpLinesToPrint.push(` - United States | \`${artifact}\``); 409 | } 410 | if (artifact.indexOf("-eu-") > 0) { 411 | gcpLinesToPrint.push(` - Europe | \`${artifact}\``); 412 | } 413 | if (artifact.indexOf("-asia-") > 0) { 414 | gcpLinesToPrint.push(` - Asia | \`${artifact}\``); 415 | } 416 | }); 417 | 418 | azureLines = [ 419 | "## Azure", 420 | "", 421 | "- Community Gallery Name | `spacelift-40913cda-9bf9-4bcb-bf90-78fd83f30079`", 422 | "- Image name | `spacelift_worker_image_ubu_lts`", 423 | `- Version | \`${process.env.AZURE_VERSION}\``, 424 | `- Resource ID | \`/CommunityGalleries/spacelift-40913cda-9bf9-4bcb-bf90-78fd83f30079/Images/spacelift_worker_image_ubu_lts/Versions/${process.env.AZURE_VERSION}\``, 425 | "", 426 | "## Google Cloud Platform", 427 | "", 428 | ]; 429 | fs.appendFileSync("./body.md", "\n\n" + azureLines.join("\n") + "\n" + gcpLinesToPrint.join("\n")); 430 | 431 | - name: Create GitHub release 432 | uses: ncipollo/release-action@v1 433 | with: 434 | tag: ${{ steps.tag_version.outputs.new_tag }} 435 | name: ${{ steps.tag_version.outputs.new_tag }} 436 | bodyFile: ./body.md 437 | --------------------------------------------------------------------------------