├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE-APACHE ├── LICENSE-MIT ├── README.md ├── executor ├── README.md └── run.py └── images └── ubuntu ├── Makefile ├── README.md ├── cloud-init ├── meta-data └── user-data ├── files ├── gha-runner-version ├── gha-runner.service ├── regenerate-ssh-host-keys.service ├── regenerate-ssh-host-keys.sh ├── resize-disk.service ├── resize-disk.sh └── start-gha-runner.py ├── image.pkr.hcl └── scripts ├── disable-timers.sh ├── finalize.sh ├── install-awscli.sh ├── install-gha-runner.sh ├── install-packages.sh ├── setup-disk-resize.sh ├── setup-grub.sh └── setup-ssh.sh /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: {} 4 | merge_group: {} 5 | 6 | jobs: 7 | build-vm: 8 | name: Build image ${{ matrix.image }}-${{ matrix.arch.name }} 9 | runs-on: ubuntu-24.04 10 | permissions: 11 | contents: read 12 | 13 | strategy: 14 | matrix: 15 | image: 16 | - ubuntu 17 | arch: 18 | - name: x86_64 19 | mode: host 20 | # We have to build AArch64 in emulated mode: while there is support in GitHUb Actions for 21 | # Arm runners, as of May 2025 they don't have nested virtualization enabled, preventing 22 | # the use of KVM (required by the "host" mode). 23 | - name: aarch64 24 | mode: emul 25 | 26 | env: 27 | IMAGE_NAME: ${{ matrix.image }} 28 | IMAGE_ARCH: ${{ matrix.arch.name }} 29 | IMAGE_MODE: ${{ matrix.arch.mode }} 30 | steps: 31 | - name: Checkout the source code 32 | uses: actions/checkout@v4 33 | 34 | - name: Install Packer 35 | uses: hashicorp/setup-packer@76e3039aa951aa4e6efe7e6ee06bc9ceb072142d 36 | 37 | - name: Install QEMU 38 | run: | 39 | sudo apt-get update 40 | sudo apt-get install -y qemu-system 41 | 42 | # https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/ 43 | # Snippet authored by gsauthof. 44 | - name: Enable KVM usage for the runner user 45 | run: | 46 | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules 47 | sudo udevadm control --reload-rules 48 | sudo udevadm trigger --name-match=kvm 49 | 50 | - name: Build the image 51 | run: | 52 | cd "images/${IMAGE_NAME}" 53 | make "${IMAGE_ARCH}-${IMAGE_MODE}" 54 | env: 55 | # This token is only needed to bypass the rate limit when downloading Packer plugins. The 56 | # token only has read-only access to repository contents anyway, so there is no risk 57 | # passing it to Packer. 58 | PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} 59 | 60 | # Compression level 9 seems to be the one where we stop getting improvements, after trying to 61 | # compress some images locally in May 2025. 62 | - name: Compress the image 63 | run: zstd -9 images/${IMAGE_NAME}/build/${IMAGE_NAME}-${IMAGE_ARCH}.qcow2 64 | 65 | - name: Upload the image as an artifact 66 | uses: actions/upload-artifact@v4 67 | with: 68 | name: ${{ matrix.image }}-${{ matrix.arch.name }}.qcow2.zst 69 | path: images/${{ matrix.image }}/build/${{ matrix.image }}-${{ matrix.arch.name }}.qcow2.zst 70 | if-no-files-found: error 71 | retention-days: 1 72 | compression-level: 0 73 | 74 | upload: 75 | name: Upload images 76 | runs-on: ubuntu-latest 77 | if: github.event_name == 'merge_group' 78 | needs: 79 | - build-vm 80 | 81 | environment: upload 82 | permissions: 83 | id-token: write 84 | 85 | steps: 86 | - name: Download built images 87 | uses: actions/download-artifact@v4 88 | with: 89 | path: images/ 90 | pattern: "*.qcow2.zst" 91 | merge-multiple: true 92 | 93 | - name: Authenticate with AWS 94 | uses: aws-actions/configure-aws-credentials@v4 95 | with: 96 | aws-region: us-west-1 97 | role-to-assume: arn:aws:iam::890664054962:role/gha-self-hosted-images-upload 98 | 99 | - name: Upload images to S3 100 | run: | 101 | # We cannot use `aws s3 cp` since as far as I can tell it has no way to set the 102 | # `if-none-match: *` header. The header is required by the IAM policy to prevent the CI 103 | # credentials from overriding existing files. 104 | for file in images/*; do 105 | echo "uploading ${file}..." 106 | aws s3api put-object \ 107 | --bucket rust-gha-self-hosted-images \ 108 | --key "images/${GITHUB_SHA}/$(basename "${file}")" \ 109 | --body "${file}" \ 110 | --if-none-match "*" 111 | done 112 | 113 | - name: Mark the current commit as the last one 114 | run: | 115 | echo "${GITHUB_SHA}" > latest 116 | aws s3api put-object \ 117 | --bucket rust-gha-self-hosted-images \ 118 | --key latest \ 119 | --body latest \ 120 | --content-type text/plain 121 | 122 | finish: 123 | name: CI finished 124 | runs-on: ubuntu-latest 125 | permissions: {} 126 | if: "${{ !cancelled() }}" 127 | needs: 128 | - build-vm 129 | - upload 130 | steps: 131 | - name: Check if all jobs were successful or skipped 132 | run: echo "${NEEDS}" | jq --exit-status 'all(.result == "success" or .result == "skipped")' 133 | env: 134 | NEEDS: "${{ toJson(needs) }}" 135 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /images/*/build 2 | /executor/instances.json 3 | -------------------------------------------------------------------------------- /LICENSE-APACHE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any 2 | person obtaining a copy of this software and associated 3 | documentation files (the "Software"), to deal in the 4 | Software without restriction, including without 5 | limitation the rights to use, copy, modify, merge, 6 | publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software 8 | is furnished to do so, subject to the following 9 | conditions: 10 | 11 | The above copyright notice and this permission notice 12 | shall be included in all copies or substantial portions 13 | of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 16 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 17 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 18 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 19 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 22 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 23 | DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitHub Actions self-hosted runners infrastructure 2 | 3 | This repository contains the infrastructure to run self-hosted GitHub Actions 4 | runners for the Rust project. The tools and scripts here are meant to be used 5 | only by the Rust Infrastructure Team: we do not intend to support running them 6 | outside our infra, and there might be breaking changes in the future. 7 | 8 | The contents of this repository are released under either the MIT or the Apache 9 | 2.0 license, at your option. 10 | 11 | ## Deployment and operations 12 | 13 | The production servers will pull this repository every 15 minutes, and if a 14 | change in the `images/` directory was done images will also be rebuilt. Check 15 | out [the documentation][forge] on the forge for instructions on how to operate 16 | the production deployment. 17 | 18 | [forge]: https://forge.rust-lang.org/infra/docs/gha-self-hosted.html 19 | -------------------------------------------------------------------------------- /executor/README.md: -------------------------------------------------------------------------------- 1 | # Ephemeral VMs executor 2 | 3 | This directory contains the Python script used to spawn ephemeral VMs for the 4 | Rust CI. The script starts VMs using QEMU, and is designed to work with VM 5 | images produced by this repository. 6 | 7 | This README only documents how an user should use the script: technical 8 | documentation on how the script works is present as comments inside the script 9 | itself. 10 | 11 | ## Configuring the instances 12 | 13 | The script reads the `instances.json` file in the current working directory at 14 | startup, loading the definition of each supported virtual machine. The file 15 | contains a list of instance objects, each with the following keypairs: 16 | 17 | * **name**: name of the instance. 18 | * **image**: path to the QCOW2 file of the base image. 19 | * **arch**: the architecture of the virtual machine (`x86_64` or `aarch64`). 20 | * **cpu-cores**: amount of CPU cores to allocate to the VM. 21 | * **ram**: amount of RAM to allocate to the VM. 22 | * **root-disk**: amount of disk space to allocate to the VM. 23 | * **timeout-seconds**: number of seconds after the VM is shut down. 24 | * **ssh-port**: port number to assign to the VM's SSH server. Documentation on 25 | how to log into the VM is available below. 26 | * **config**: arbitrary object containing instance-specific configuration. This 27 | data will be available inside the VM, and can be used by the base image to 28 | configure itself. A configuration pre-processor is available, allowing to 29 | fetch some configuration values at startup time. Documentation on how to 30 | access the configuration inside the VM is available below. 31 | 32 | An example of such file is: 33 | 34 | ```json 35 | [ 36 | { 37 | "name": "vm-1", 38 | "image": "../images/ubuntu/build/x86_64/rootfs.qcow2", 39 | "arch": "x86_64", 40 | "cpu-cores": 4, 41 | "ram": "4G", 42 | "root-disk": "80G", 43 | "timeout-seconds": 14400, 44 | "ssh-port": 2201, 45 | "config": { 46 | "repo": "rust-lang-ci/rust", 47 | "token": "${{ gha-install-token:rust-lang-ci/rust }}", 48 | "whitelisted-event": "push" 49 | } 50 | }, 51 | { 52 | "name": "vm-2", 53 | "image": "../images/ubuntu/build/aarch64/rootfs.qcow2", 54 | "arch": "aarch64", 55 | "cpu-cores": 2, 56 | "ram": "2G", 57 | "root-disk": "20G", 58 | "timeout-seconds": 14400, 59 | "ssh-port": 2202, 60 | "config": { 61 | "repo": "rust-lang-ci/rust", 62 | "token": "${{ gha-install-token:rust-lang-ci/rust }}", 63 | "whitelisted-event": "push" 64 | } 65 | } 66 | ] 67 | ``` 68 | 69 | ## Configuration pre-processor 70 | 71 | The script allows to fetch some configuration values dynamically right before 72 | the virtual machine is started. A limited number of functions is available. 73 | 74 | ### Fetching the GHA installation token 75 | 76 | To fetch the GitHub Actions runner installation token for a repository, you can 77 | set the following value in the config: 78 | 79 | ``` 80 | ${{ gha-install-token:ORGANIZATION/REPOSITORY }} 81 | ``` 82 | 83 | When this parameter is present, the script will call the GitHub API to fetch a 84 | new installation token. For that to work the `GITHUB_TOKEN` environment 85 | variable needs to be set, containing a valid GitHub API token. 86 | 87 | ## Starting an instance 88 | 89 | To start an instance, run the following command inside the directory containing 90 | the `instances.json` file: 91 | 92 | ``` 93 | ./run.py IMAGE-NAME 94 | ``` 95 | 96 | The command will start an ephemeral copy of the VM and then delete it once the 97 | VM shuts down. 98 | 99 | ## Logging into the instance 100 | 101 | The script allows you to connect to the spawned VM through SSH, through the 102 | `ssh-port` defined in the instance configuration. You can connect by running: 103 | 104 | ``` 105 | ssh -p manage@localhost 106 | ``` 107 | 108 | The password for the `manage` user in our VMs is `password`. The SSH server 109 | takes a while to start up, so you might have to wait before being able to log 110 | in. Keep in mind that our VM images regenerate the SSH host keys every time 111 | they boot, so you'll likely get host key mismatch errors when you try to 112 | connect. 113 | 114 | ## Reading the instance configuration inside the VM 115 | 116 | The script will mount a virtual CD-ROM inside each virtual machine, containing 117 | a file called `instance.json`. The file will contain a JSON document with a 118 | copy of the `name` and `config` keys from the host's `instances.json`. 119 | 120 | It's possible for the VM to eject the virtual CD-ROM once it's done reading its 121 | contents, for example to prevent future untrusted processes inside the VM from 122 | reading it. Once the CD-ROM tray is opened, the script will automatically 123 | remove the CD-ROM. 124 | -------------------------------------------------------------------------------- /executor/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import os 5 | import pathlib 6 | import random 7 | import re 8 | import shutil 9 | import signal 10 | import subprocess 11 | import sys 12 | import telnetlib 13 | import tempfile 14 | import threading 15 | import time 16 | import urllib.request 17 | 18 | 19 | # Range of ports where QMP could be bound. 20 | QMP_PORT_RANGE = (50000, 55000) 21 | 22 | # How many seconds to wait after a graceful shutdown signal before killing the 23 | # virtual machine. 24 | GRACEFUL_SHUTDOWN_TIMEOUT = 60 25 | 26 | # How many seconds should pass between each call to the GitHub API. 27 | GITHUB_API_POLL_INTERVAL = 15 28 | 29 | # Architecture-specific QEMU flags and BIOS blob URL. 30 | QEMU_ARCH = { 31 | "x86_64": { 32 | "flags": [ 33 | # Standard x86_64 machine with hardware acceleration. 34 | "-machine", "pc,accel=kvm", 35 | ], 36 | }, 37 | "aarch64": { 38 | "flags": [ 39 | # Virtual AArch64 machine with hardware acceleration. 40 | "-machine", "virt,gic_version=3,accel=kvm", 41 | # Use the host's CPU variant. 42 | "-cpu", "host", 43 | ], 44 | # Installed with `sudo apt-get install qemu-efi-aarch64` 45 | "bios": "/usr/share/qemu-efi-aarch64/QEMU_EFI.fd", 46 | }, 47 | } 48 | 49 | 50 | class VM: 51 | def __init__(self, instance, env): 52 | self._base = instance["image"] 53 | self._vm_timeout = instance["timeout-seconds"] 54 | self._ssh_port = instance["ssh-port"] 55 | self._cpu = instance["cpu-cores"] 56 | self._ram = instance["ram"] 57 | self._disk = instance["root-disk"] 58 | self._env = env 59 | 60 | # Once the GitHub Actions build start, the VM won't reloaad anymore 61 | # when a SIGUSR1 is received. 62 | self._prevent_reloads = False 63 | 64 | self._arch = instance["arch"] 65 | if self._arch not in QEMU_ARCH: 66 | raise RuntimeError(f"unsupported architecture: {self._arch}") 67 | 68 | self._path = pathlib.Path(tempfile.mkdtemp()) 69 | self._path_root = self._path / "root.qcow2" 70 | self._path_cdrom = self._path / "env.iso" 71 | 72 | self._process = None 73 | self._qmp_shutdown_port = random.randint(*QMP_PORT_RANGE) 74 | self._qmp_tray_ejector_port = random.randint(*QMP_PORT_RANGE) 75 | 76 | self._copy_base_image() 77 | self._create_config_cdrom() 78 | 79 | def _copy_base_image(self): 80 | if self._path.exists(): 81 | shutil.rmtree(self._path) 82 | 83 | self._path.mkdir(exist_ok=True) 84 | 85 | log("creating the disk image") 86 | subprocess.run([ 87 | "qemu-img", "create", 88 | # Path of the base image. 89 | "-b", str(pathlib.Path(self._base).resolve()), 90 | # Use a Copy on Write filesystem, to avoid having to copy the whole 91 | # base image every time we start a VM. 92 | "-f", "qcow2", 93 | # Explicitly set format of backing file 94 | "-F", "qcow2", 95 | # Path of the destination image. 96 | str(self._path_root.resolve()), 97 | # New size of the disk. 98 | self._disk, 99 | ], check=True) 100 | 101 | def _create_config_cdrom(self): 102 | tempdir = pathlib.Path(tempfile.mkdtemp()) 103 | envjson = tempdir / "instance.json" 104 | 105 | with envjson.open("w") as f: 106 | json.dump(self._env, f) 107 | 108 | log("creating the virtual CD-ROM with the instance configuration") 109 | subprocess.run([ 110 | "genisoimage", 111 | "-output", str(self._path_cdrom.resolve()), 112 | "-input-charset", "utf-8", 113 | # Call the ISO "instance-configuration" 114 | "-volid", "instance-configuration", 115 | # Generate a Joliet filesystem, which is preferred by Windows. 116 | "-joliet", 117 | # Generate a Rock Ridge filesystem, which is preferred by Linux. 118 | "-rock", 119 | # Include the `instance.json` file in the ISO. 120 | str(envjson.resolve()), 121 | ], check=True) 122 | 123 | shutil.rmtree(str(tempdir)) 124 | 125 | def run(self): 126 | if self._process is not None: 127 | raise RuntimeError("this VM was already started") 128 | 129 | def preexec_fn(): 130 | # Don't forward signals to QEMU 131 | os.setpgrp() 132 | 133 | cmd = [ 134 | f"qemu-system-{self._arch}", 135 | # Reserved RAM for the virtual machine. 136 | "-m", str(self._ram), 137 | # Allocated cores for the virtual machine. 138 | "-smp", str(self._cpu), 139 | # Prevent QEMU from showing a graphical console window. 140 | "-display", "none", 141 | # Mount the VM image as the root drive. 142 | "-drive", "file=" + str(self._path_root) + ",media=disk,if=virtio", 143 | # Enable networking inside the VM. 144 | "-net", "nic,model=virtio", 145 | # Forward the 22 port on the host, as the configured SSH port. 146 | "-net", "user,hostfwd=tcp::" + str(self._ssh_port) + "-:22", 147 | # Mount the instance configuration as a CD-ROM. The mounted ISO is 148 | # generated by the _create_config_cdrom method. 149 | "-cdrom", str(self._path_cdrom), 150 | # This QMP port is used by the shutdown() method to send the 151 | # shutdown signal to the QEMU VM instead of killing it. 152 | "-qmp", "telnet:127.0.0.1:" + str(self._qmp_shutdown_port) + ",server,nowait", 153 | # This QMP port is used by the TrayEjector thread to eject the 154 | # CD-ROM as soon as the guest VM opens the tray. 155 | "-qmp", "telnet:127.0.0.1:" + str(self._qmp_tray_ejector_port) + ",server,nowait", 156 | ] 157 | cmd += QEMU_ARCH[self._arch]["flags"] 158 | 159 | if "bios" in QEMU_ARCH[self._arch]: 160 | cmd += ["-bios", QEMU_ARCH[self._arch]["bios"]] 161 | 162 | log("starting the virtual machine") 163 | self._process = subprocess.Popen(cmd, preexec_fn=preexec_fn) 164 | 165 | TrayEjectorThread(self._qmp_tray_ejector_port).start() 166 | 167 | if "repo" in self._env["config"]: 168 | GitHubRunnerStatusWatcher( 169 | self._env["config"]["repo"], 170 | self._env["name"], 171 | GITHUB_API_POLL_INTERVAL, 172 | self._gha_build_started, 173 | ).start() 174 | else: 175 | log("didn't start polling the GitHub API: missing 'repo' in config") 176 | 177 | try: 178 | self._process.wait() 179 | except KeyboardInterrupt: 180 | self.shutdown() 181 | 182 | # Shutdown signal was successful, wait for clean shutdown 183 | try: 184 | if self._process is not None: 185 | self._process.wait() 186 | except KeyboardInterrupt: 187 | self.kill() 188 | 189 | def shutdown(self): 190 | if self._process is None: 191 | raise RuntimeError("can't shutdown a stopped VM") 192 | 193 | # QEMU allows interacting with the VM through the "monitoring port", 194 | # using Telnet as the protocol. This tries to connect to the monitoring 195 | # port to send the graceful shutdown signal. If it fails, we're forced 196 | # to hard-kill the virtual machine. 197 | try: 198 | qmp = QMPClient(self._qmp_shutdown_port) 199 | qmp.shutdown_vm() 200 | except Exception as e: 201 | print("failed to gracefully shutdown the VM:", e) 202 | self.kill() 203 | return 204 | 205 | log("sent shutdown signal to the VM") 206 | 207 | Timer("graceful-shutdown-timeout", self.kill, GRACEFUL_SHUTDOWN_TIMEOUT).start() 208 | 209 | def kill(self): 210 | if self._process is None: 211 | raise RuntimeError("can't kill a stopped VM") 212 | 213 | self._process.kill() 214 | self._process = None 215 | 216 | log("killed the virtual machine") 217 | 218 | def cleanup(self): 219 | shutil.rmtree(str(self._path)) 220 | 221 | def sigusr1_received(self): 222 | if self._prevent_reloads: 223 | log("did not reload as a build is currently running") 224 | else: 225 | log("reload signal received, shutting down the VM") 226 | self.shutdown() 227 | 228 | def _gha_build_started(self): 229 | self._prevent_reloads = True 230 | Timer("vm-timeout", self.shutdown, self._vm_timeout).start() 231 | 232 | 233 | class GitHubRunnerStatusWatcher(threading.Thread): 234 | def __init__(self, repo, runner_name, check_interval, then): 235 | super().__init__(name="github-runner-status-watcher", daemon=True) 236 | 237 | self._check_interval = check_interval 238 | self._repo = repo 239 | self._runner_name = runner_name 240 | self._then = then 241 | 242 | def run(self): 243 | log("started polling GitHub to detect when the runner started working") 244 | while True: 245 | runners = self._retrieve_runners() 246 | if self._runner_name in runners and runners[self._runner_name]["busy"]: 247 | log("the runner started processing a build!") 248 | self._then() 249 | break 250 | time.sleep(self._check_interval) 251 | 252 | def _retrieve_runners(self): 253 | result = {} 254 | url = f"https://api.github.com/repos/{self._repo}/actions/runners" 255 | for response in github_api("GET", url): 256 | for runner in response["runners"]: 257 | result[runner["name"]] = runner 258 | return result 259 | 260 | 261 | # We only want the instance configuration to be available at startup, and not 262 | # when the build is running. To achieve that, this thread monitors QMP for 263 | # DEVICE_TRY_MOVED events, and when the CD-ROM is ejected it detaches it from 264 | # the virtual machine. 265 | class TrayEjectorThread(threading.Thread): 266 | def __init__(self, qmp_port): 267 | super().__init__(name="tray-ejector", daemon=True) 268 | self._qmp_port = qmp_port 269 | 270 | def run(self): 271 | # Wait for the QMP port to come online. 272 | qmp = None 273 | while True: 274 | try: 275 | qmp = QMPClient(self._qmp_port) 276 | break 277 | except ConnectionRefusedError: 278 | time.sleep(0.01) 279 | 280 | try: 281 | while True: 282 | data = qmp.wait_for_event("DEVICE_TRAY_MOVED") 283 | if not data["tray-open"]: 284 | continue 285 | qmp.eject(data["device"]) 286 | log("ejected CD-ROM (device: %s)" % data["device"]) 287 | except EOFError: 288 | # The connection will be closed when the VM shuts down. We don't 289 | # care if it happens. 290 | pass 291 | 292 | 293 | # Simple thread that executes a function after a timeout 294 | class Timer(threading.Thread): 295 | def __init__(self, name, callback, timeout): 296 | super().__init__(name=name, daemon=True) 297 | 298 | self._name = name 299 | self._callback = callback 300 | self._timeout = timeout 301 | 302 | def run(self): 303 | log(f"started timer {self._name}, fires in {self._timeout} seconds") 304 | 305 | # The sleep is done in a loop to handle spurious wakeups 306 | started_at = time.time() 307 | while time.time() < started_at + self._timeout: 308 | time.sleep(self._timeout - (time.time() - started_at)) 309 | 310 | log(f"timer {self._name} fired") 311 | self._callback() 312 | 313 | 314 | # QMP (QEMU Machine Protocol) is a way to control VMs spawned with QEMU, and 315 | # to receive events from them. An introduction to the protocol is available at: 316 | # 317 | # https://wiki.qemu.org/Documentation/QMP 318 | # 319 | # A full list of commands and events is available at: 320 | # 321 | # https://www.qemu.org/docs/master/qemu-qmp-ref.html#Commands-and-Events-Index 322 | # 323 | class QMPClient: 324 | def __init__(self, port): 325 | self._conn = telnetlib.Telnet("127.0.0.1", port) 326 | 327 | # When starting the connection, QEMU sends a greeting message 328 | # containing the `QMP` key. To finish the handshake, the command 329 | # `qmp_capabilities` then needs to be sent. 330 | greeting = self._read_message() 331 | if "QMP" not in greeting: 332 | raise RuntimeError("didn't receive a greeting from the QMP server") 333 | self._write_message({"execute": "qmp_capabilities"}) 334 | self._read_success() 335 | 336 | def shutdown_vm(self): 337 | self._write_message({"execute": "system_powerdown"}) 338 | self._read_success() 339 | 340 | def eject(self, device, *, force=False): 341 | self._write_message({ 342 | "execute": "eject", 343 | "arguments": { 344 | "device": device, 345 | "force": force, 346 | }, 347 | }) 348 | self._read_success() 349 | 350 | def wait_for_event(self, event): 351 | while True: 352 | message = self._read_message() 353 | if "event" in message and message["event"] == event: 354 | return message["data"] 355 | 356 | def _read_success(self): 357 | result = self._read_message() 358 | if "return" not in result: 359 | raise RuntimeError("QMP returned an error: " + repr(result)) 360 | 361 | def _write_message(self, message): 362 | self._conn.write(json.dumps(message).encode("utf-8") + b"\r\n") 363 | 364 | def _read_message(self): 365 | return json.loads(self._conn.read_until(b'\n').decode("utf-8").strip()) 366 | 367 | 368 | class ConfigPreprocessor: 369 | # This regex matches: ${{ FUNCTION:ARGS }} 370 | _VARIABLE_RE = re.compile(r"^\${{ *(?P[a-zA-Z0-9_-]+):(?P[^}]+)}}$") 371 | 372 | def __init__(self, config): 373 | self._config = config 374 | 375 | def process(self): 376 | for key, value in self._config.items(): 377 | matches = self._VARIABLE_RE.match(value) 378 | if matches is None: 379 | continue 380 | 381 | function = matches.group("function").strip() 382 | args = matches.group("args").strip() 383 | 384 | if function == "gha-install-token": 385 | self._config[key] = self._fetch_gha_install_token(args) 386 | else: 387 | raise ValueError(f"unknown preprocessor function: {function}") 388 | 389 | return self._config 390 | 391 | def _fetch_gha_install_token(self, repo): 392 | log(f"fetching the GHA installation token for {repo}") 393 | 394 | res = next(github_api( 395 | "POST", 396 | f"https://api.github.com/repos/{repo}/actions/runners/registration-token", 397 | )) 398 | return res["token"] 399 | 400 | 401 | NEXT_LINK_RE = re.compile(r"<([^>]+)>; rel=\"next\"") 402 | 403 | def github_api(method, url): 404 | try: 405 | github_token = os.environ["GITHUB_TOKEN"] 406 | except KeyError: 407 | raise RuntimeError("missing environment variable GITHUB_TOKEN") from None 408 | 409 | while url is not None: 410 | request = urllib.request.Request(url) 411 | request.add_header("User-Agent", "https://github.com/rust-lang/gha-self-hosted (infra@rust-lang.org)") 412 | request.add_header("Authorization", f"token {github_token}") 413 | request.method = method 414 | 415 | response = urllib.request.urlopen(request) 416 | 417 | # Handle pagination of the GitHub API 418 | url = None 419 | if "Link" in response.headers: 420 | captures = NEXT_LINK_RE.search(response.headers["Link"]) 421 | if captures is not None: 422 | url = captures.group(1) 423 | 424 | yield json.load(response) 425 | 426 | 427 | signal_vms = [] 428 | def sigusr1_received(sig, frame): 429 | for vm in signal_vms: 430 | vm.sigusr1_received() 431 | 432 | 433 | def run(instance_name): 434 | signal.signal(signal.SIGUSR1, sigusr1_received) 435 | 436 | with open("instances.json") as f: 437 | instances = json.load(f) 438 | 439 | instance = None 440 | for candidate in instances: 441 | if candidate["name"] == instance_name: 442 | instance = candidate 443 | break 444 | else: 445 | print(f"error: instance not found: {instance_name}", file=sys.stderr) 446 | exit(1) 447 | 448 | config = ConfigPreprocessor(instance["config"]) 449 | env = { 450 | "name": instance["name"], 451 | "config": config.process(), 452 | } 453 | 454 | vm = VM(instance, env) 455 | signal_vms.append(vm) 456 | 457 | vm.run() 458 | vm.cleanup() 459 | 460 | 461 | def log(*args, **kwargs): 462 | print("==>", *args, **kwargs) 463 | sys.stdout.flush() 464 | 465 | if __name__ == "__main__": 466 | if len(sys.argv) == 2: 467 | run(sys.argv[1]) 468 | else: 469 | print(f"usage: {sys.argv[0]} ", file=sys.stderr) 470 | exit(1) 471 | -------------------------------------------------------------------------------- /images/ubuntu/Makefile: -------------------------------------------------------------------------------- 1 | ENV = PACKER_CACHE_DIR=build/cache 2 | PACKER_ARGS = -var "git_sha=$(shell git rev-parse HEAD)" 3 | 4 | .PHONY: host clean init x86_64-host x86_64-emul aarch64-host aarch64-emul 5 | 6 | host: 7 | make $(shell uname -m)-host 8 | 9 | clean: 10 | rm -rf build 11 | 12 | init: 13 | $(ENV) packer init image.pkr.hcl 14 | 15 | x86_64-host: init 16 | rm -rf build/packer-tmp 17 | $(ENV) packer build $(PACKER_ARGS) -var emulated=false -var arch=x86_64 image.pkr.hcl 18 | mv build/packer-tmp/rootfs.qcow2 build/ubuntu-x86_64.qcow2 19 | 20 | x86_64-emul: init 21 | rm -rf build/packer-tmp 22 | $(ENV) packer build $(PACKER_ARGS) -var emulated=true -var arch=x86_64 image.pkr.hcl 23 | mv build/packer-tmp/rootfs.qcow2 build/ubuntu-x86_64.qcow2 24 | 25 | aarch64-host: init build/qemu-efi-aarch64.fd 26 | rm -rf build/packer-tmp 27 | $(ENV) packer build $(PACKER_ARGS) -var emulated=false -var arch=aarch64 -var firmware=build/qemu-efi-aarch64.fd image.pkr.hcl 28 | mv build/packer-tmp/rootfs.qcow2 build/ubuntu-aarch64.qcow2 29 | 30 | aarch64-emul: init build/qemu-efi-aarch64.fd 31 | rm -rf build/packer-tmp 32 | $(ENV) packer build $(PACKER_ARGS) -var emulated=true -var arch=aarch64 -var firmware=build/qemu-efi-aarch64.fd image.pkr.hcl 33 | mv build/packer-tmp/rootfs.qcow2 build/ubuntu-aarch64.qcow2 34 | 35 | # Running QEMU for AArch64 requires the QEMU_EFI.fd file to be provided. Some distributions like 36 | # Debian and Ubuntu package it, but the path they use is not consistent across distros. To avoid 37 | # problems, this step downloads the (mirrored) Debian package and extracts the file from it. 38 | # 39 | # Debian packages are `ar` archives containing the `data.tar.xz` tarball (and other metadata). That 40 | # tarball then contains the files that will actually be installed on the system. So, to extract a 41 | # file out of it we first need to extract the .deb with `ar`, and then the file with `tar`. 42 | EFI_TMP = build/qemu-efi-aarch64-tmp 43 | build/qemu-efi-aarch64.fd: 44 | rm -rf $(EFI_TMP) 45 | mkdir $(EFI_TMP) 46 | curl -Lo $(EFI_TMP)/package.deb https://ci-mirrors.rust-lang.org/gha-self-hosted/qemu-efi-aarch64_2022.11-6.deb 47 | cd $(EFI_TMP) && ar x package.deb 48 | tar -C $(EFI_TMP) -xf $(EFI_TMP)/data.tar.xz 49 | cp $(EFI_TMP)/usr/share/qemu-efi-aarch64/QEMU_EFI.fd $@ 50 | -------------------------------------------------------------------------------- /images/ubuntu/README.md: -------------------------------------------------------------------------------- 1 | # Ubuntu VM images 2 | 3 | > [!CAUTION] 4 | > 5 | > These images are strictly meant to be used in Rust's self-hosted CI. The Rust 6 | > infrastructure team provides no support for third parties attempting to use 7 | > the images, nor any stability guarantee. If you want to use these images, we 8 | > recommend forking this repository. 9 | 10 | This directory contains the source code used to build the Ubuntu images for 11 | Rust's self-hosted CI. The images are built with [Packer]. 12 | 13 | The images are based on Ubuntu 20.04, and are prepared for x86_64 and AArch64. 14 | 15 | ## Accessing the images built by CI 16 | 17 | Our production infrastructure relies on VM images built by CI. These images are 18 | uploaded to [gha-self-hosted-images.infra.rust-lang.org] when a PR is being 19 | merged to `main` with the merge queue. 20 | 21 | In order to download any image, you must first retrieve the latest commit hash, 22 | which is available at the [`/latest`][cdn-latest] URL. Then, access the relevant 23 | URL depending on the image you want, replacing `${commit}` with the commit hash 24 | you previously retrieved: 25 | 26 | | Architecture | Compression | URL template | 27 | | ------------ | ----------- | -------------------------------------------- | 28 | | x86_64 | zstandard | `/images/${commit}/ubuntu-x86_64.qcow2.zst` | 29 | | AArch64 | zstandard | `/images/${commit}/ubuntu-aarch64.qcow2.zst` | 30 | 31 | ### Rolling back to a previously built image 32 | 33 | Merging changes that break our self-hosted runners might happen, and in those 34 | cases the easiest way to roll back is to create a new PR reverting the 35 | problematic changes. 36 | 37 | If that is not quick enough, you can ask a member of infra-admins to manually 38 | override the `latest` object inside of the `rust-gha-self-hosted-images` S3 39 | bucket to point to the known good commit. No CDN invalidation is needed, as that 40 | file intentionally has a short TTL. 41 | 42 | ## Building the image locally 43 | 44 | To build the images you should have the latest version of [Packer] and QEMU 45 | installed on your system. Running `make` will build the image for your host 46 | architecture. If you want to build specific images, the following commands are 47 | available: 48 | 49 | | Architecture | Native build | Emulated build | Output path | 50 | | ------------ | ------------------- | ------------------- | ---------------------------- | 51 | | x86_64 | `make x86_64-host ` | `make x86_64-emul` | `build/ubuntu-x86_64.qcow2` | 52 | | AArch64 | `make aarch64-host` | `make aarch64-emul` | `build/ubuntu-aarch64.qcow2` | 53 | 54 | ## Build process overview 55 | 56 | The build process for the image is fully driven by Packer, configured in 57 | `image.pkr.hcl`. The `Makefile` entry point is only responsible to download some 58 | pre-requisites, pass the correct variables to Packer, and move files to their 59 | correct location. Once Packer downloads the base Ubuntu image, it boots it with 60 | QEMU (either natively or emulated depending on the architecture) and configures 61 | it in two stages. 62 | 63 | The first stage is performed by [cloud-init]: Packer spins up a local HTTP 64 | server with the content of the `cloud-init/` directory, and points cloud-init to 65 | it. cloud-init is responsible to create the user Packer will SSH into, allowing 66 | the second stage to begin. 67 | 68 | The second stage is performed by Packer SSH'ing into the VM. It copies the 69 | `files/` directory (containing support files needed by our scripts) into the VM 70 | (at `/tmp/packer-files`), and then calls each of the scripts defined in the 71 | `scripts/` directory. These scripts are actually responsible for most of the VM 72 | configuration. 73 | 74 | > [!NOTE] 75 | > 76 | > Adding a script to the `scripts/` directory is **not** enough for it to be 77 | > executed. You will also need to explicitly list it in `image.pkr.hcl`. 78 | 79 | ## Image runtime requirements 80 | 81 | The image is configured through a virtual CD-ROM that must be mounted in the 82 | virtual machine with the `instance-configuration` disk label. The CD-ROM must 83 | contain an `instance.json` file with the following schema: 84 | 85 | * `name`: name of the runner. 86 | * `config`: 87 | * `repo`: GitHub repository to register the runner into. 88 | * `token`: GitHub Actions registration token. 89 | * `whitelisted-event` *(optional)*: value that will be set in the 90 | `RUST_WHITELISTED_EVENT_NAME` environment variable. 91 | 92 | ## Image runtime behavior 93 | 94 | Each time it boots, the VM will: 95 | 96 | * Resize the disk image to use all allocated space (implemented in 97 | `files/regenerate-ssh-host-keys.sh`). 98 | * Regenerate the SSH host keys, to avoid reusing the keys baked into the image 99 | (implemented in `files/regenerate-ssh-host-keys.sh`). 100 | * Mount the virtual CD-ROM (see "Image runtime requirements"), load the runner 101 | configuration from it, eject the CD-ROM, and start the runner (implemented in 102 | `files/start-gha-runner.py`). 103 | 104 | The GitHub Actions runner will then listen for jobs, and execute a single job, 105 | once the job finishes, the runner will shut down the VM. 106 | 107 | The VM provides passwordless sudo access via SSH through the `manage` user 108 | (password: `password`). 109 | 110 | [Packer]: https://developer.hashicorp.com/packer 111 | [cloud-init]: https://cloud-init.io/ 112 | [gha-self-hosted-images.infra.rust-lang.org]: https://gha-self-hosted-images.infra.rust-lang.org 113 | [cdn-latest]: https://gha-self-hosted-images.infra.rust-lang.org/latest 114 | -------------------------------------------------------------------------------- /images/ubuntu/cloud-init/meta-data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rust-lang/gha-self-hosted/88bab7bde768c2ce324df929f8895ef65f33119d/images/ubuntu/cloud-init/meta-data -------------------------------------------------------------------------------- /images/ubuntu/cloud-init/user-data: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | package_update: false 3 | ssh_pwauth: true 4 | 5 | users: 6 | - name: manage 7 | sudo: ALL=(ALL) NOPASSWD:ALL 8 | groups: users, admin 9 | lock_passwd: false 10 | shell: /bin/bash 11 | 12 | # Password: password 13 | # The hash is generated with the following command: 14 | # 15 | # mkpasswd --method=SHA-512 --rounds=4096 16 | # 17 | passwd: $6$rounds=4096$0PrPeCPXlp.crF$P6SA0aV1jUwFxByJu0702fMs9SwXMub0LbGfCdYv8suao96jnlXcSZ3oPewMs0fgMui9.8O.GRKFh80xUosJU0 18 | 19 | apt: 20 | preserve_sources_list: true 21 | -------------------------------------------------------------------------------- /images/ubuntu/files/gha-runner-version: -------------------------------------------------------------------------------- 1 | 2.316.0-rust4 2 | -------------------------------------------------------------------------------- /images/ubuntu/files/gha-runner.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=GitHub Actions Runner 3 | After=network.target 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/start-gha-runner 7 | User=gha 8 | Group=gha 9 | WorkingDirectory=/gha 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /images/ubuntu/files/regenerate-ssh-host-keys.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Regenerate SSH host keys 3 | Before=ssh.service 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/regenerate-ssh-host-keys 7 | Type=oneshot 8 | 9 | [Install] 10 | WantedBy=multi-user.target 11 | -------------------------------------------------------------------------------- /images/ubuntu/files/regenerate-ssh-host-keys.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | export DEBIAN_FRONTEND=noninteractive 7 | 8 | # Regenerate the SSH host keys, which were deleted during VM creation. 9 | dpkg-reconfigure openssh-server 10 | -------------------------------------------------------------------------------- /images/ubuntu/files/resize-disk.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Regenerate SSH host keys 3 | 4 | [Service] 5 | ExecStart=/usr/local/bin/resize-disk 6 | Type=oneshot 7 | 8 | [Install] 9 | WantedBy=multi-user.target 10 | -------------------------------------------------------------------------------- /images/ubuntu/files/resize-disk.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | # Detect the current partition 7 | full_device="$(mount | grep ' on / ' | awk '{print($1)}')" 8 | device="$(echo "${full_device}" | sed 's/\(.\+\)[0-9]\+$/\1/g')" 9 | partition="$(echo "${full_device}" | sed 's/.\+\([0-9]\+\)$/\1/g')" 10 | 11 | # Resize the partition table - https://superuser.com/a/1156509 12 | sudo sgdisk -e "${device}" 13 | sudo sgdisk -d "${partition}" "${device}" 14 | sudo sgdisk -N "${partition}" "${device}" 15 | sudo partprobe "${device}" 16 | 17 | # Resize the root file system 18 | sudo resize2fs "${full_device}" 19 | -------------------------------------------------------------------------------- /images/ubuntu/files/start-gha-runner.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import os 5 | import tempfile 6 | import subprocess 7 | import uuid 8 | 9 | CDROM = "/dev/disk/by-label/instance-configuration" 10 | 11 | # Mount the virtual CD containing the environment 12 | temp = tempfile.mkdtemp() 13 | subprocess.run(["sudo", "mount", "-o", "ro", CDROM, temp], check=True) 14 | 15 | # Load the environment from it 16 | with open(temp + "/instance.json") as f: 17 | instance = json.load(f) 18 | 19 | # Eject the CD containing the environment 20 | subprocess.run(["sudo", "umount", temp], check=True) 21 | try: 22 | subprocess.run(["sudo", "eject", CDROM], check=True) 23 | except subprocess.CalledProcessError: 24 | print("warning: failed to eject the CD-ROM") 25 | os.rmdir(temp) 26 | 27 | # Configure the GitHub Actions runner 28 | subprocess.run([ 29 | "./config.sh", "--unattended", "--replace", 30 | "--url", "https://github.com/" + instance["config"]["repo"], 31 | "--token", instance["config"]["token"], 32 | "--name", instance["name"] + uuid.uuid4().hex, 33 | "--ephemeral", 34 | ], check=True) 35 | 36 | # Start the runner 37 | env = dict(os.environ) 38 | if "whitelisted-event" in instance["config"]: 39 | env["RUST_WHITELISTED_EVENT_NAME"] = instance["config"]["whitelisted-event"] 40 | subprocess.run(["./run.sh"], env=env, check=True) 41 | 42 | # Stop the machine 43 | subprocess.run(["sudo", "poweroff"], check=True) 44 | -------------------------------------------------------------------------------- /images/ubuntu/image.pkr.hcl: -------------------------------------------------------------------------------- 1 | packer { 2 | required_plugins { 3 | qemu = { 4 | source = "github.com/hashicorp/qemu" 5 | version = "~> 1" 6 | } 7 | } 8 | } 9 | 10 | build { 11 | sources = ["source.qemu.ubuntu"] 12 | 13 | // Copy the support files the scripts rely on. 14 | provisioner "shell" { 15 | inline = [ 16 | "mkdir /tmp/packer-files", 17 | ] 18 | } 19 | provisioner "file" { 20 | destination = "/tmp/packer-files/" 21 | source = "./files/" 22 | } 23 | 24 | // Run all the scripts needed to configure the machine. 25 | provisioner "shell" { 26 | env = { 27 | GIT_SHA = var.git_sha 28 | } 29 | scripts = [ 30 | "./scripts/install-packages.sh", 31 | "./scripts/install-gha-runner.sh", 32 | "./scripts/install-awscli.sh", 33 | "./scripts/setup-ssh.sh", 34 | "./scripts/setup-disk-resize.sh", 35 | "./scripts/setup-grub.sh", 36 | "./scripts/disable-timers.sh", 37 | "./scripts/finalize.sh", 38 | ] 39 | } 40 | } 41 | 42 | source "qemu" "ubuntu" { 43 | vm_name = "rootfs.qcow2" 44 | output_directory = "build/packer-tmp" 45 | 46 | accelerator = var.emulated ? "tcg" : "kvm" 47 | machine_type = var.arch == "aarch64" ? (var.emulated ? "virt" : "virt,gic_version=3") : "pc" 48 | 49 | # The values of cortex-a57 and Haswell are mostly arbitrary (recent enough CPUs). 50 | cpu_model = var.emulated ? (var.arch == "aarch64" ? "cortex-a57" : "Haswell") : "host" 51 | 52 | disk_discard = "unmap" 53 | disk_image = true 54 | disk_interface = "virtio-scsi" 55 | disk_size = "5G" 56 | 57 | # Serve the cloud-init/ directory with the QEMU provisioner's HTTP server. 58 | # This allows us to do the initial configuration (adding the SSH user) with cloud-init. 59 | http_directory = "cloud-init/" 60 | 61 | # Download the latest cloud image for the specified Ubuntu version. Note that cloud images are 62 | # periodically rebuilt to include new security updates in them, so we cannot hardcode a checksum. 63 | iso_checksum = "file:${local.ubuntu_url}/SHA256SUMS" 64 | iso_url = "${local.ubuntu_url}/ubuntu-${local.ubuntu_version}-server-cloudimg-${var.arch == "aarch64" ? "arm64" : "amd64"}.img" 65 | 66 | # On AArch64 the machine won't boot unless we provide the QEMU_EFI.fd file as the firmware. 67 | firmware = var.firmware 68 | 69 | # Do not show any GUI when building the machine. 70 | use_default_display = true 71 | 72 | qemu_binary = "qemu-system-${var.arch}" 73 | qemuargs = [ 74 | # Show the VM output in the Packer logs. 75 | ["-nographic", ""], 76 | ["-serial", "pty"], 77 | # Set the kernel parameters, 78 | ["-smbios", "type=1,serial=ds=nocloud-net;instance-id=gha-self-hosted;seedfrom=http://{{ .HTTPIP }}:{{ .HTTPPort }}/"], 79 | ] 80 | 81 | # Username and password of the VM are configured through cloud-init. 82 | ssh_username = "manage" 83 | ssh_password = "password" 84 | 85 | # For emulated builds we increase the timeouts, as bringing up an emulated VM can be slow. 86 | ssh_handshake_attempts = var.emulated ? 100 : 10 87 | ssh_timeout = var.emulated ? "1h" : "5m" 88 | } 89 | 90 | locals { 91 | ubuntu_version = "20.04" 92 | ubuntu_url = "https://cloud-images.ubuntu.com/releases/${local.ubuntu_version}/release" 93 | } 94 | 95 | variable "arch" { 96 | type = string 97 | validation { 98 | condition = var.arch == "x86_64" || var.arch == "aarch64" 99 | error_message = "Unsupported architecture." 100 | } 101 | } 102 | 103 | variable "emulated" { 104 | type = bool 105 | } 106 | 107 | variable "git_sha" { 108 | type = string 109 | } 110 | 111 | variable "firmware" { 112 | type = string 113 | default = null 114 | } 115 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/disable-timers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | # Disable timers updating stuff in the background while the VM is running. We 7 | # had failures on CI due to the VM updating software in the background, and 8 | # that should not happen. The VM is ephemeral and periodically rebuilt anyway. 9 | sudo systemctl disable \ 10 | motd-news.timer \ 11 | apt-daily.timer \ 12 | fwupd-refresh.timer \ 13 | apt-daily-upgrade.timer \ 14 | systemd-tmpfiles-clean.timer \ 15 | logrotate.timer \ 16 | man-db.timer \ 17 | e2scrub_all.timer \ 18 | fstrim.timer 19 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/finalize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | # Commit changes to disk, to prevent losing them when Packer kills the VM at 7 | # the end of the build. 8 | # 9 | # MUST BE THE LAST COMMAND EXECUTED IN THIS SCRIPT! 10 | # 11 | echo "synchronizing changes to disk..." 12 | sudo sync 13 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/install-awscli.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | cd /tmp 7 | 8 | ARCH="$(uname -m)" 9 | curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" 10 | unzip awscliv2.zip 11 | sudo ./aws/install 12 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/install-gha-runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Install and configure the GitHub Actions runner on the image. 3 | 4 | set -euo pipefail 5 | IFS=$'\n\t' 6 | 7 | AGENT_VERSION="$(cat /tmp/packer-files/gha-runner-version)" 8 | AGENT_REPO="rust-lang/gha-runner" 9 | 10 | case "$(uname -m)" in 11 | x86_64) 12 | AGENT_PLATFORM="linux-x64" 13 | ;; 14 | aarch64) 15 | AGENT_PLATFORM="linux-arm64" 16 | ;; 17 | *) 18 | echo "error: unsupported platform: $(uname -m)" 19 | exit 1 20 | ;; 21 | esac 22 | 23 | echo "adding the gha user..." 24 | sudo adduser gha --home /gha --disabled-password 25 | sudo adduser gha docker 26 | echo "gha ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/gha-nopasswd 27 | 28 | echo "downloading and installing the runner..." 29 | cd /gha 30 | sudo -u gha -- curl -Lo runner.tar.gz https://github.com/${AGENT_REPO}/releases/download/v${AGENT_VERSION}/actions-runner-${AGENT_PLATFORM}-${AGENT_VERSION}.tar.gz 31 | sudo -u gha -- tar -xzf ./runner.tar.gz 32 | sudo -u gha -- rm ./runner.tar.gz 33 | 34 | echo "configuring startup of the runner..." 35 | sudo cp /tmp/packer-files/gha-runner.service /etc/systemd/system/gha-runner.service 36 | sudo cp /tmp/packer-files/start-gha-runner.py /usr/local/bin/start-gha-runner 37 | sudo chmod +x /usr/local/bin/start-gha-runner 38 | sudo systemctl daemon-reload 39 | sudo systemctl enable gha-runner.service # Will start at the next boot. 40 | 41 | echo "adding runner information..." 42 | cat > /tmp/setup_info << EOF 43 | [ 44 | { 45 | "group": "Image details", 46 | "detail": "rust-lang/gha-self-hosted commit: ${GIT_SHA}\nImage build time: $(date --iso-8601=seconds --utc)" 47 | } 48 | ] 49 | EOF 50 | sudo -u gha cp /tmp/setup_info /gha/.setup_info 51 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/install-packages.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | PACKAGES=( 7 | # Needed by QEMU to be able to send a graceful shutdown signal 8 | acpid 9 | 10 | # Needed by rustc's CI 11 | docker.io 12 | docker-buildx 13 | jq 14 | python-is-python2 15 | python3-pip 16 | 17 | # Needed by install-awscli 18 | unzip 19 | ) 20 | 21 | export DEBIAN_FRONTEND=noninteractive 22 | 23 | sudo apt-get update 24 | sudo apt-get upgrade -y 25 | sudo apt-get install -y ${PACKAGES[@]} 26 | 27 | # Enable Docker at startup 28 | sudo systemctl enable docker 29 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/setup-disk-resize.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | # Install a small service that resizes the disk on boot 7 | sudo cp /tmp/packer-files/resize-disk.service /etc/systemd/system/resize-disk.service 8 | sudo cp /tmp/packer-files/resize-disk.sh /usr/local/bin/resize-disk 9 | sudo chmod +x /usr/local/bin/resize-disk 10 | sudo systemctl daemon-reload 11 | sudo systemctl enable resize-disk.service # Will start at the next boot. 12 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/setup-grub.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | # Reduce the GRUB timeout to 1, otherwise some VMs might wait 30 seconds before 7 | # booting into the OS. 8 | sudo sed -i 's/^GRUB_TIMEOUT=[0-9]*$/GRUB_TIMEOUT=1/g' /etc/default/grub 9 | sudo bash -c "echo 'GRUB_RECORDFAIL_TIMEOUT=0' >> /etc/default/grub" 10 | sudo update-grub 11 | -------------------------------------------------------------------------------- /images/ubuntu/scripts/setup-ssh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | IFS=$'\n\t' 5 | 6 | # Install a small service that regenerates SSH host keys on boot. 7 | sudo cp /tmp/packer-files/regenerate-ssh-host-keys.service /etc/systemd/system/regenerate-ssh-host-keys.service 8 | sudo cp /tmp/packer-files/regenerate-ssh-host-keys.sh /usr/local/bin/regenerate-ssh-host-keys 9 | sudo chmod +x /usr/local/bin/regenerate-ssh-host-keys 10 | sudo systemctl daemon-reload 11 | sudo systemctl enable regenerate-ssh-host-keys.service # Will start at the next boot. 12 | 13 | # Remove the current host keys 14 | sudo rm /etc/ssh/ssh_host_* 15 | --------------------------------------------------------------------------------