├── config.sh ├── cleanup.py ├── requirements.txt ├── start.sh ├── env.py ├── .github └── workflows │ └── deploy.yaml ├── .pre-commit-config.yaml ├── LICENSE ├── Containerfile ├── run.py ├── README.md └── prepare.py /config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | cat << EOS 4 | { 5 | "driver": { 6 | "name": "Openstack", 7 | "version": "2022.03.28.0" 8 | } 9 | } 10 | EOS 11 | -------------------------------------------------------------------------------- /cleanup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import openstack 3 | 4 | import env 5 | 6 | 7 | def main() -> None: 8 | conn = openstack.connect() 9 | for server in conn.compute.servers(name=env.VM_NAME): 10 | conn.compute.delete_server(server) 11 | 12 | 13 | if __name__ == "__main__": 14 | main() 15 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.4 2 | bcrypt==3.2.0 3 | certifi==2021.5.30 4 | cffi==1.14.5 5 | chardet==4.0.0 6 | cryptography==3.4.7 7 | decorator==5.0.9 8 | dogpile.cache==1.1.3 9 | idna==2.10 10 | iso8601==0.1.14 11 | jmespath==0.10.0 12 | jsonpatch==1.32 13 | jsonpointer==2.1 14 | keystoneauth1==4.3.1 15 | munch==2.5.0 16 | netifaces==0.11.0 17 | openstacksdk==0.57.0 18 | os-service-types==1.7.0 19 | paramiko==2.10.3 20 | pbr==5.6.0 21 | pycparser==2.20 22 | PyNaCl==1.4.0 23 | PyYAML==5.4.1 24 | requests==2.25.1 25 | requestsexceptions==1.4.0 26 | six==1.16.0 27 | stevedore==3.3.0 28 | tenacity==8.0.1 29 | urllib3==1.26.5 30 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | trap cleanup 1 2 3 6 4 | 5 | cleanup() { 6 | gitlab-runner unregister --all-runners 7 | sleep 5 8 | } 9 | 10 | if [[ "$TLS_CA_CERT" ]]; then 11 | mkdir -p "$HOME"/.gitlab-runner/certs/ 12 | echo "$TLS_CA_CERT" > "$HOME"/.gitlab-runner/certs/$(echo "$CI_SERVER_URL" | cut -d'/' -f3 | cut -d':' -f1).crt 13 | fi 14 | 15 | echo "$PRIVATE_KEY" > "$HOME"/priv_key 16 | 17 | gitlab-runner register --non-interactive \ 18 | --executor=custom \ 19 | --custom-config-exec="$HOME"/config.sh \ 20 | --custom-prepare-exec="$HOME"/prepare.py \ 21 | --custom-run-exec="$HOME"/run.py \ 22 | --custom-cleanup-exec="$HOME"/cleanup.py 23 | 24 | if [[ "$CONCURRENT" ]]; then 25 | sed -i "s/concurrent = .*/concurrent = $CONCURRENT/g" "$HOME"/.gitlab-runner/config.toml 26 | fi 27 | 28 | gitlab-runner run 29 | -------------------------------------------------------------------------------- /env.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | VM_NAME = f"gitlab-builder-{os.getenv('CUSTOM_ENV_CI_RUNNER_ID')}-project-{os.getenv('CUSTOM_ENV_CI_PROJECT_ID')}-concurrent-{os.getenv('CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID')}-job-{os.getenv('CUSTOM_ENV_CI_JOB_ID')}" # noqa 4 | 5 | FLAVOR = os.getenv("CUSTOM_ENV_FLAVOR") or os.getenv("FLAVOR") 6 | BUILDER_IMAGE = os.getenv("CUSTOM_ENV_BUILDER_IMAGE") or os.getenv("BUILDER_IMAGE") 7 | NETWORK = os.getenv("CUSTOM_ENV_NETWORK") or os.getenv("NETWORK") 8 | KEY_PAIR_NAME = os.getenv("CUSTOM_ENV_KEY_PAIR_NAME") or os.getenv("KEY_PAIR_NAME") 9 | SECURITY_GROUP = os.getenv("CUSTOM_ENV_SECURITY_GROUP") or os.getenv("SECURITY_GROUP") 10 | USERNAME = os.getenv("CUSTOM_ENV_USERNAME") or os.getenv("USERNAME") 11 | PRIVATE_KEY_PATH = f"{os.getenv('HOME')}/priv_key" 12 | SSH_TIMEOUT = os.getenv("CUSTOM_ENV_SSH_TIMEOUT") or os.getenv("SSH_TIMEOUT") or "30" 13 | 14 | BUILD_FAILURE_EXIT_CODE = os.getenv("BUILD_FAILURE_EXIT_CODE") 15 | SYSTEM_FAILURE_EXIT_CODE = os.getenv("SYSTEM_FAILURE_EXIT_CODE") 16 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yaml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: 8 | - "*" 9 | 10 | jobs: 11 | deploy: 12 | runs-on: ubuntu-latest 13 | if: startsWith(github.ref, 'refs/tags/') 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set env 17 | run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV 18 | - name: Build container image 19 | uses: redhat-actions/buildah-build@v2.5 20 | with: 21 | image: quay.io/redhatqe/openstack-gitlab-runner 22 | tags: ${{ env.RELEASE_VERSION }} latest 23 | dockerfiles: Containerfile 24 | oci: true 25 | - name: Push image to registry 26 | uses: redhat-actions/push-to-registry@v2.2 27 | with: 28 | image: openstack-gitlab-runner 29 | tags: ${{ env.RELEASE_VERSION }} latest 30 | registry: quay.io/redhatqe 31 | username: ${{ secrets.QUAY_IO_USERNAME }} 32 | password: ${{ secrets.QUAY_IO_TOKEN }} 33 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/asottile/reorder_python_imports 3 | rev: v3.0.1 4 | hooks: 5 | - id: reorder-python-imports 6 | language_version: python3 7 | - repo: https://github.com/psf/black 8 | rev: 22.1.0 9 | hooks: 10 | - id: black 11 | args: [--safe, --quiet, --line-length, "100"] 12 | language_version: python3 13 | require_serial: true 14 | - repo: https://gitlab.com/pycqa/flake8.git 15 | rev: 3.9.2 16 | hooks: 17 | - id: flake8 18 | language_version: python3 19 | args: 20 | - --max-line-length=100 21 | - --ignore=W503,E203 22 | - repo: https://github.com/pre-commit/pre-commit-hooks 23 | rev: v4.1.0 24 | hooks: 25 | - id: trailing-whitespace 26 | language_version: python3 27 | - id: end-of-file-fixer 28 | language_version: python3 29 | - id: debug-statements 30 | language_version: python3 31 | - repo: https://github.com/asottile/pyupgrade 32 | rev: v2.31.1 33 | hooks: 34 | - id: pyupgrade 35 | language_version: python3 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Red Hat Quality Engineering 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Containerfile: -------------------------------------------------------------------------------- 1 | ARG GITLAB_RUNNER_VERSION=v13.12.0 2 | 3 | FROM registry.access.redhat.com/ubi8:8.5 AS builder 4 | 5 | ARG GITLAB_RUNNER_VERSION 6 | 7 | ENV GITLAB_REPO=https://gitlab.com/gitlab-org/gitlab-runner.git \ 8 | PATH=$PATH:/root/go/bin/ 9 | 10 | RUN dnf install -y git-core make go ncurses && \ 11 | git clone --depth=1 --branch=${GITLAB_RUNNER_VERSION} ${GITLAB_REPO} && \ 12 | cd gitlab-runner && \ 13 | make runner-bin-host && \ 14 | chmod a+x out/binaries/gitlab-runner && \ 15 | out/binaries/gitlab-runner --version 16 | 17 | FROM registry.access.redhat.com/ubi8:8.5 18 | 19 | ARG GITLAB_RUNNER_VERSION 20 | 21 | COPY --from=builder /gitlab-runner/out/binaries/gitlab-runner /usr/bin 22 | 23 | ENV HOME=/home/gitlab-runner \ 24 | VENV=/openstack_driver_venv 25 | 26 | ENV PATH="$VENV/bin:$PATH" 27 | 28 | LABEL maintainer="Dmitry Misharov " \ 29 | io.openshift.tags="gitlab,ci,runner" \ 30 | name="openstack-gitlab-runner" \ 31 | io.k8s.display-name="GitLab runner" \ 32 | summary="GitLab runner" \ 33 | description="A GitLab runner image with openstack custom executor." \ 34 | io.k8s.description="A GitLab runner image with openstack custom executor." 35 | 36 | WORKDIR $HOME 37 | 38 | COPY cleanup.py env.py config.sh prepare.py run.py requirements.txt start.sh . 39 | 40 | RUN dnf install -y --nodocs python38-pip git-core && \ 41 | python3.8 -m venv $VENV && \ 42 | pip install wheel && \ 43 | pip install -r requirements.txt && \ 44 | dnf remove -y git-core && \ 45 | dnf clean all -y 46 | 47 | RUN chgrp -R 0 $HOME && \ 48 | chmod +x cleanup.py config.sh prepare.py run.py start.sh && \ 49 | chmod -R g=u $HOME 50 | 51 | USER 1001 52 | 53 | CMD ["./start.sh"] 54 | -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | import openstack 5 | import paramiko 6 | 7 | import env 8 | 9 | 10 | def get_server_ip(conn: openstack.connection.Connection) -> str: 11 | server = list(conn.compute.servers(name=env.VM_NAME, status="ACTIVE"))[0] 12 | return list(conn.compute.server_ips(server))[0].address 13 | 14 | 15 | def execute_script_on_server(ssh: paramiko.client.SSHClient, script_path: str) -> int: 16 | stdin, stdout, stderr = ssh.exec_command("/bin/bash") 17 | with open(script_path) as f: 18 | stdin.channel.send(f.read()) 19 | stdin.channel.shutdown_write() 20 | for line in iter(lambda: stdout.readline(2048), ""): 21 | print(line, sep="", end="", flush=True) 22 | exit_status = stdout.channel.recv_exit_status() 23 | if exit_status != 0: 24 | for line in iter(lambda: stderr.readline(2048), ""): 25 | print(line, sep="", end="", flush=True) 26 | return exit_status 27 | 28 | 29 | def get_ssh_client(ip: str) -> paramiko.client.SSHClient: 30 | ssh_client = paramiko.client.SSHClient() 31 | pkey = paramiko.rsakey.RSAKey.from_private_key_file(env.PRIVATE_KEY_PATH) 32 | ssh_client.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) 33 | ssh_client.connect( 34 | hostname=ip, 35 | username=env.USERNAME, 36 | pkey=pkey, 37 | look_for_keys=False, 38 | allow_agent=False, 39 | timeout=int(env.SSH_TIMEOUT), 40 | ) 41 | return ssh_client 42 | 43 | 44 | def main() -> None: 45 | conn = openstack.connect() 46 | ip = get_server_ip(conn) 47 | ssh_client = get_ssh_client(ip) 48 | exit_status = execute_script_on_server(ssh_client, sys.argv[1]) 49 | ssh_client.close() 50 | if exit_status != 0: 51 | sys.exit(int(env.BUILD_FAILURE_EXIT_CODE)) 52 | 53 | 54 | if __name__ == "__main__": 55 | main() 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitLab CI Openstack executor 2 | 3 | GitLab CI doesn't support Openstack as an executor but provides the ability to implement your own 4 | executor by using scripts to provision, run, and clean up CI environment. This repository contains 5 | such scripts as well as a Containerfile to build and configure a container image with Gitlab Runner 6 | that uses custom Openstack executor. 7 | 8 | ## Building 9 | 10 | ```sh 11 | git clone https://github.com/RedHatQE/openstack-gitlab-executor.git 12 | cd openstack-gitlab-executor 13 | podman build --build-arg GITLAB_RUNNER_VERSION= -f Containerfile -t openstack-gitlab-runner 14 | ``` 15 | 16 | ## Configuration 17 | 18 | The container expects the following environment variables: 19 | 20 | ### Instance variables 21 | 22 | `FLAVOR` - Default instance flavor reference 23 | 24 | `BUILDER_IMAGE` - Default image to use for instance provisioning 25 | 26 | `NETWORK` - Default network name 27 | 28 | `KEY_PAIR_NAME` - Default SSH key pair name 29 | 30 | `SECURITY_GROUP` - Default security group 31 | 32 | `USERNAME` - Default username for SSH connection to instances 33 | 34 | `PRIVATE_KEY` - Private key content 35 | 36 | `SSH_TIMEOUT` - Timeout for establishing SSH connection 37 | 38 | ### GitLab Runner variables 39 | 40 | `RUNNER_TAG_LIST` - Tag list 41 | 42 | `REGISTRATION_TOKEN` - Runner's registration token 43 | 44 | `RUNNER_NAME` - Runner name 45 | 46 | `CI_SERVER_URL` - Runner URL 47 | 48 | `RUNNER_BUILDS_DIR` - Path to `builds` directory on the Openstack instance 49 | 50 | `RUNNER_CACHE_DIR` - Path to `cache` directory on the Openstack instance 51 | 52 | `CONCURRENT` - Limits how many jobs can run concurrently (default 1) 53 | 54 | ### [Openstack variables](https://docs.openstack.org/python-openstackclient/latest/cli/man/openstack.html#environment-variables) 55 | 56 | `OS_AUTH_URL` - Openstack authentication URL 57 | 58 | `OS_PROJECT_NAME` - Project-level authentication scope (name or ID) 59 | 60 | `OS_USERNAME` - Authentication username 61 | 62 | `OS_PASSWORD` - Authentication password 63 | 64 | `OS_PROJECT_DOMAIN_NAME` - Domain name or ID containing project 65 | 66 | `OS_USER_DOMAIN_NAME` - Domain name or ID containing user 67 | 68 | `OS_REGION_NAME` - Authentication region name 69 | 70 | `OS_IDENTITY_API_VERSION` - Identity API version 71 | 72 | `OS_INTERFACE` - Interface type 73 | 74 | ## Usage 75 | 76 | Create an env file with all variables: 77 | 78 | ```sh 79 | cat env.txt 80 | 81 | RUNNER_TAG_LIST= 82 | REGISTRATION_TOKEN= 83 | RUNNER_NAME= 84 | CI_SERVER_URL= 85 | RUNNER_BUILDS_DIR= 86 | RUNNER_CACHE_DIR= 87 | CONCURRENT= 88 | 89 | FLAVOR= 90 | BUILDER_IMAGE= 91 | NETWORK= 92 | KEY_PAIR_NAME= 93 | SECURITY_GROUP= 94 | USERNAME= 95 | 96 | OS_AUTH_URL= 97 | OS_PROJECT_NAME= 98 | OS_USERNAME= 99 | OS_PASSWORD= 100 | OS_PROJECT_DOMAIN_NAME= 101 | OS_USER_DOMAIN_NAME= 102 | OS_REGION_NAME= 103 | OS_IDENTITY_API_VERSION= 104 | OS_INTERFACE= 105 | ``` 106 | 107 | Run a container: 108 | 109 | ```sh 110 | podman run -it \ 111 | -e PRIVATE_KEY="$(cat )" 112 | --env-file=env.txt \ 113 | quay.io/redhatqe/openstack-gitlab-runner:latest 114 | ``` 115 | 116 | You can override instance configuration defaults by providing environment variables in a GitLab CI 117 | job config. For example, if you want to use another Openstack image to provision builder instance 118 | you should provide the following: 119 | 120 | ```yaml 121 | stages: 122 | - build 123 | 124 | build: 125 | stage: build 126 | variables: 127 | BUILDER_IMAGE: my-custom-image 128 | tags: 129 | - some-tag 130 | script: 131 | - some command 132 | ``` 133 | -------------------------------------------------------------------------------- /prepare.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import traceback 4 | from operator import itemgetter 5 | 6 | import openstack 7 | import paramiko 8 | from tenacity import retry 9 | from tenacity import RetryCallState 10 | from tenacity import stop_after_attempt 11 | from tenacity import wait_fixed 12 | 13 | import env 14 | 15 | 16 | def _duplicated_image_message_detected(image_name: str, message: str) -> bool: 17 | return f"More than one Image exists with the name '{image_name}'" in message 18 | 19 | 20 | def _latest_updated_image_by_name( 21 | connection: openstack.connection.Connection, image_name: str 22 | ) -> openstack.compute.v2.image.Image: 23 | filtered_images = [x for x in connection.image.images() if x.name == image_name] 24 | return max(filtered_images, key=itemgetter("updated_at")) 25 | 26 | 27 | def _try_get_image( 28 | connection: openstack.connection.Connection, image_name: str 29 | ) -> openstack.compute.v2.image.Image: 30 | 31 | try: 32 | image = connection.compute.find_image(image_name) 33 | except openstack.exceptions.DuplicateResource as e: 34 | if _duplicated_image_message_detected(image_name, e.message): 35 | image = _latest_updated_image_by_name(connection, image_name) 36 | print(f"Multiple images with the same name, using latest: {image.id}") 37 | else: 38 | raise e 39 | return image 40 | 41 | 42 | def provision_server( 43 | conn: openstack.connection.Connection, 44 | ) -> openstack.compute.v2.server.Server: 45 | image = _try_get_image(conn, env.BUILDER_IMAGE) 46 | flavor = conn.compute.find_flavor(env.FLAVOR) 47 | network = conn.network.find_network(env.NETWORK) 48 | server = conn.compute.create_server( 49 | name=env.VM_NAME, 50 | flavor_id=flavor.id, 51 | image_id=image.id, 52 | key_name=env.KEY_PAIR_NAME, 53 | security_groups=[{"name": env.SECURITY_GROUP}], 54 | networks=[{"uuid": network.id}], 55 | ) 56 | return conn.compute.wait_for_server(server, wait=600) 57 | 58 | 59 | def get_server_ip( 60 | conn: openstack.connection.Connection, server: openstack.compute.v2.server.Server 61 | ) -> str: 62 | return list(conn.compute.server_ips(server))[0].address 63 | 64 | 65 | def check_ssh(ip: str) -> None: 66 | ssh_client = paramiko.client.SSHClient() 67 | pkey = paramiko.rsakey.RSAKey.from_private_key_file(env.PRIVATE_KEY_PATH) 68 | ssh_client.set_missing_host_key_policy(paramiko.client.AutoAddPolicy()) 69 | 70 | def before_callback(retry_state: RetryCallState): 71 | print( 72 | f"Attempt: {retry_state.attempt_number}; timeout: {env.SSH_TIMEOUT} seconds", 73 | flush=True, 74 | ) 75 | 76 | @retry( 77 | reraise=True, 78 | stop=stop_after_attempt(10), 79 | wait=wait_fixed(int(env.SSH_TIMEOUT)), 80 | before=before_callback, 81 | ) 82 | def connect(): 83 | ssh_client.connect( 84 | hostname=ip, 85 | username=env.USERNAME, 86 | pkey=pkey, 87 | look_for_keys=False, 88 | allow_agent=False, 89 | timeout=int(env.SSH_TIMEOUT), 90 | ) 91 | 92 | connect() 93 | ssh_client.close() 94 | 95 | 96 | def main() -> None: 97 | print( 98 | "Source code of this driver https://github.com/RedHatQE/openstack-gitlab-executor", 99 | flush=True, 100 | ) 101 | print("Connecting to Openstack", flush=True) 102 | try: 103 | conn = openstack.connect() 104 | print(f"Provisioning an instance {env.VM_NAME}", flush=True) 105 | server = provision_server(conn) 106 | ip = get_server_ip(conn, server) 107 | print(f"Instance {env.VM_NAME} is running on address {ip}", flush=True) 108 | conn.close() 109 | print("Waiting for SSH connection", flush=True) 110 | check_ssh(ip) 111 | print("SSH connection has been established", flush=True) 112 | except Exception: 113 | traceback.print_exc() 114 | sys.exit(int(env.SYSTEM_FAILURE_EXIT_CODE)) 115 | 116 | 117 | if __name__ == "__main__": 118 | main() 119 | --------------------------------------------------------------------------------