├── terraform.tfvars ├── Dockerfile ├── README.md └── main.tf /terraform.tfvars: -------------------------------------------------------------------------------- 1 | github_org = "" 2 | github_repo = "" -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # Set environment variables 4 | ENV DEBIAN_FRONTEND=noninteractive 5 | 6 | # Update and install common packages (same as base) 7 | RUN apt-get update && apt-get install -y \ 8 | python3.10 \ 9 | python3-pip \ 10 | curl \ 11 | wget \ 12 | git \ 13 | && apt-get clean \ 14 | && rm -rf /var/lib/apt/lists/* 15 | 16 | # Install common Python packages (same as base) 17 | RUN pip3 install --no-cache-dir --upgrade pip && \ 18 | pip3 install --no-cache-dir \ 19 | numpy \ 20 | pandas \ 21 | scikit-learn \ 22 | matplotlib 23 | 24 | # Install additional packages 25 | RUN apt-get update && apt-get install -y \ 26 | nodejs \ 27 | npm \ 28 | && apt-get clean \ 29 | && rm -rf /var/lib/apt/lists/* 30 | 31 | # Install TensorFlow and Keras 32 | RUN pip3 install --no-cache-dir tensorflow keras 33 | 34 | # Install PyTorch 35 | RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu 36 | 37 | # Install some Node.js packages globally 38 | RUN npm install -g \ 39 | express \ 40 | lodash \ 41 | moment 42 | 43 | # Create large files to simulate longer build time 44 | RUN dd if=/dev/urandom of=/large_file_1 bs=1M count=500 && \ 45 | dd if=/dev/urandom of=/large_file_2 bs=1M count=500 && \ 46 | rm /large_file_1 /large_file_2 47 | 48 | # Set working directory 49 | WORKDIR /app 50 | 51 | # Create a simple Python script that uses one of the new packages 52 | RUN echo 'import tensorflow as tf; print(f"TensorFlow version: {tf.__version__}")' > tf_version.py 53 | 54 | CMD ["python3", "tf_version.py"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Remote BuildKit Instance for Docker Builds 2 | 3 | [Read more on our blog](https://www.blacksmith.sh/blog/faster-docker-builds-using-a-remote-buildkit-instance). 4 | 5 | This repository contains Terraform scripts and GitHub Actions workflow to set up and use a remote BuildKit instance for running Docker builds. 6 | 7 | ## Overview 8 | 9 | The main components of this setup are: 10 | 11 | 1. `main.tf`: Terraform script to create an EC2 instance with BuildKit installed. 12 | 2. `terraform.tfvars`: Variables file for Terraform (you need to populate this). 13 | 3. `Dockerfile`: A sample Dockerfile for testing. 14 | 4. `buildkit-test.yml`: Example GitHub Actions workflow to use the remote BuildKit instance. 15 | 16 | ## Quick Start 17 | 18 | 1. Clone this repository. 19 | 2. Configure AWS CLI with your credentials. 20 | 3. Populate `terraform.tfvars` with your GitHub organization and repository name: 21 | 22 | ``` 23 | github_org = "your-org-name" 24 | github_repo = "your-repo-name" 25 | ``` 26 | 4. Run Terraform: 27 | 28 | ``` 29 | terraform init 30 | terraform plan 31 | terraform apply 32 | ``` 33 | 34 | 5. After `terraform apply` is successful, you'll get the public IP of your BuildKit instance. Use this to set the `BUILDKIT_HOST` environment variable. 35 | 36 | 6. Set the following environment secrets variables in your GitHub repo: 37 | - `BUILDKIT_HOST`: The public IP address of your EC2 instance (you'll get this after applying Terraform). 38 | - `AWS_ACCOUNT_ID`: Your AWS account ID. 39 | 40 | ## Terraform Configuration Details 41 | 42 | The `main.tf` file sets up the following resources: 43 | 44 | - EC2 instance with BuildKit installed 45 | - Instance type: c5a.4xlarge (16 vCPUs, 32 GiB Memory) 46 | - AMI: Amazon Linux 2 (ami-05c3dc660cb6907f0 in us-east-2) 47 | - Root volume: 100 GB gp3 EBS volume 48 | - User data script installs Docker, BuildKit, and sets up a systemd service for BuildKit 49 | - BuildKit is configured to listen on port 9999 50 | - Security group for the EC2 instance 51 | - IAM role and instance profile for the EC2 instance 52 | - OIDC provider for GitHub Actions 53 | - IAM role for GitHub Actions with necessary permissions 54 | 55 | The EC2 instance is configured with user data to install and set up BuildKit. It exposes BuildKit on port 9999 by default. 56 | 57 | ## GitHub Actions Workflow 58 | 59 | The included workflow file (`buildkit-test.yml`) demonstrates how to use the remote BuildKit instance in your CI/CD pipeline. It sets up the AWS credentials, configures BuildKit to use the remote instance, and runs a Docker build. 60 | 61 | ## Important Security Note 62 | 63 | ⚠️ **Warning**: The current setup does not exclusively whitelist GitHub Actions runners' IPs. For production use, it's strongly recommended to restrict the security group ingress rules to only allow traffic from [GitHub Actions IP ranges](https://api.github.com/meta) and trusted sources. 64 | 65 | ## Customization 66 | 67 | - You can change the BuildKit port (default: 9999) in the EC2 instance user data script and the GitHub Actions workflow file. 68 | - Adjust the EC2 instance type and region in main.tf as needed. If you modify the region, make sure to update it in your GitHub Actions workflow file as well (.github/workflows/build.yml). 69 | - Modify the IAM permissions in `main.tf` if you need additional AWS services access. 70 | - Update the AMI ID in main.tf if you want to use a different base image or if you're changing regions, as AMI IDs are region-specific. 71 | 72 | ## Contributing 73 | 74 | Contributions to improve this setup are welcome! Please submit a pull request or open an issue to discuss proposed changes. 75 | 76 | ## License 77 | 78 | [MIT License](LICENSE) 79 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | # Provider configuration 2 | provider "aws" { 3 | region = "us-east-2" # Change this to your preferred region 4 | } 5 | 6 | # Variables 7 | variable "github_org" { 8 | description = "GitHub organization name" 9 | type = string 10 | } 11 | 12 | variable "github_repo" { 13 | description = "GitHub repository name" 14 | type = string 15 | } 16 | 17 | # EC2 Instance for BuildKit 18 | resource "aws_instance" "buildkit" { 19 | ami = "ami-05c3dc660cb6907f0" 20 | instance_type = "c5a.4xlarge" 21 | availability_zone = "us-east-2a" 22 | 23 | vpc_security_group_ids = [aws_security_group.buildkit.id] 24 | iam_instance_profile = aws_iam_instance_profile.buildkit_instance_profile.name 25 | 26 | root_block_device { 27 | volume_type = "gp3" 28 | volume_size = 100 29 | } 30 | 31 | user_data = <<-EOF 32 | #!/bin/bash 33 | set -e 34 | 35 | # Update and install dependencies 36 | yum update -y 37 | yum install -y docker git 38 | 39 | # Start and enable Docker 40 | systemctl start docker 41 | systemctl enable docker 42 | 43 | # Add ec2-user to the docker group 44 | usermod -aG docker ec2-user 45 | 46 | # Download and install BuildKit 47 | export BUILDKIT_VERSION=0.12.0 48 | curl -sSL "https://github.com/moby/buildkit/releases/download/v$${BUILDKIT_VERSION}/buildkit-v$${BUILDKIT_VERSION}.linux-amd64.tar.gz" -o buildkit.tar.gz 49 | tar -xzf buildkit.tar.gz -C /usr/local/bin --strip-components=1 50 | 51 | # Create buildkitd systemd service 52 | cat < /etc/systemd/system/buildkitd.service 53 | [Unit] 54 | Description=BuildKit daemon 55 | After=network.target 56 | 57 | [Service] 58 | ExecStart=/usr/local/bin/buildkitd --addr tcp://0.0.0.0:9999 --addr unix:///run/buildkit/buildkitd.sock --debug 59 | Restart=always 60 | 61 | [Install] 62 | WantedBy=multi-user.target 63 | EOT 64 | 65 | # Enable and start buildkitd service 66 | systemctl daemon-reload 67 | systemctl enable buildkitd 68 | systemctl start buildkitd 69 | EOF 70 | } 71 | 72 | # Security Group for BuildKit Instance 73 | resource "aws_security_group" "buildkit" { 74 | name = "buildkit-sg" 75 | description = "Security group for BuildKit instance" 76 | 77 | ingress { 78 | from_port = 9999 79 | to_port = 9999 80 | protocol = "tcp" 81 | cidr_blocks = ["0.0.0.0/0"] # Restrict this to GA runners range 82 | } 83 | 84 | egress { 85 | from_port = 0 86 | to_port = 0 87 | protocol = "-1" 88 | cidr_blocks = ["0.0.0.0/0"] 89 | } 90 | } 91 | 92 | # IAM Role for EC2 Instance 93 | resource "aws_iam_role" "buildkit_instance_role" { 94 | name = "BuildKitInstanceRole" 95 | 96 | assume_role_policy = jsonencode({ 97 | Version = "2012-10-17" 98 | Statement = [ 99 | { 100 | Action = "sts:AssumeRole" 101 | Effect = "Allow" 102 | Principal = { 103 | Service = "ec2.amazonaws.com" 104 | } 105 | } 106 | ] 107 | }) 108 | } 109 | 110 | # IAM Role Policy Attachment 111 | # This is not required for our example but will be useful if you're pushing to ECR 112 | # resource "aws_iam_role_policy_attachment" "buildkit_instance_policy" { 113 | # policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess" 114 | # role = aws_iam_role.buildkit_instance_role.name 115 | # } 116 | 117 | # IAM Instance Profile 118 | resource "aws_iam_instance_profile" "buildkit_instance_profile" { 119 | name = "BuildKitInstanceProfile" 120 | role = aws_iam_role.buildkit_instance_role.name 121 | } 122 | 123 | # OIDC Provider for GitHub Actions 124 | resource "aws_iam_openid_connect_provider" "github_actions" { 125 | url = "https://token.actions.githubusercontent.com" 126 | client_id_list = ["sts.amazonaws.com"] 127 | thumbprint_list = ["6938fd4d98bab03faadb97b34396831e3780aea1", "1c58a3a8518e8759bf075b76b750d4f2df264fcd"] 128 | } 129 | 130 | # IAM Role for GitHub Actions 131 | resource "aws_iam_role" "github_actions_role" { 132 | name = "GithubActionsBuildKitRole" 133 | 134 | assume_role_policy = jsonencode({ 135 | Version = "2012-10-17" 136 | Statement = [ 137 | { 138 | Effect = "Allow" 139 | Principal = { 140 | Federated = aws_iam_openid_connect_provider.github_actions.arn 141 | } 142 | Action = "sts:AssumeRoleWithWebIdentity" 143 | Condition = { 144 | StringEquals = { 145 | "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" 146 | } 147 | StringLike = { 148 | "token.actions.githubusercontent.com:sub" = "repo:${var.github_org}/${var.github_repo}:*" 149 | } 150 | } 151 | } 152 | ] 153 | }) 154 | } 155 | 156 | # IAM Policy for GitHub Actions Role 157 | resource "aws_iam_role_policy" "github_actions_policy" { 158 | name = "GithubActionsBuildKitPolicy" 159 | role = aws_iam_role.github_actions_role.id 160 | 161 | policy = jsonencode({ 162 | Version = "2012-10-17" 163 | Statement = [ 164 | { 165 | Effect = "Allow" 166 | Action = [ 167 | "ec2:DescribeInstances", 168 | "ecr:GetAuthorizationToken", 169 | "ecr:BatchCheckLayerAvailability", 170 | "ecr:GetDownloadUrlForLayer", 171 | "ecr:BatchGetImage", 172 | "ecr:PutImage", 173 | "ecr:InitiateLayerUpload", 174 | "ecr:UploadLayerPart", 175 | "ecr:CompleteLayerUpload" 176 | ] 177 | Resource = "*" 178 | } 179 | ] 180 | }) 181 | } 182 | 183 | # Output 184 | output "buildkit_instance_public_ip" { 185 | value = aws_instance.buildkit.public_ip 186 | description = "The public IP address of the BuildKit instance" 187 | } 188 | 189 | output "github_actions_role_arn" { 190 | value = aws_iam_role.github_actions_role.arn 191 | description = "The ARN of the IAM role for GitHub Actions" 192 | } --------------------------------------------------------------------------------