├── img ├── scallops.png └── scallops-infra.png ├── scripts ├── Powershell │ └── disabledefender.ps1 └── bash │ ├── gitlab_backup.sh │ ├── gcloud_logger.sh │ ├── gitlab_startup.sh │ └── gitlab_helpers.sh ├── gitlab-runner ├── win-values.yaml ├── linux-values.yaml ├── kaniko-values.yaml └── dockerhub-values.yaml ├── outputs.tf ├── versions.tf ├── .gitignore ├── provider.tf ├── LICENSE.md ├── debug.tf ├── network.tf ├── locals.tf ├── storage.tf ├── CHANGELOG.md ├── config.tfvars ├── secrets.tf ├── load-balancer.tf ├── iam.tf ├── vars.tf ├── gke.tf ├── main.tf └── README.MD /img/scallops.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SygniaLabs/ScallOps/HEAD/img/scallops.png -------------------------------------------------------------------------------- /img/scallops-infra.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SygniaLabs/ScallOps/HEAD/img/scallops-infra.png -------------------------------------------------------------------------------- /scripts/Powershell/disabledefender.ps1: -------------------------------------------------------------------------------- 1 | Set-MpPreference -DisableRealtimeMonitoring $true -SubmitSamplesConsent NeverSend 2 | Get-MpPreference -Verbose -------------------------------------------------------------------------------- /gitlab-runner/win-values.yaml: -------------------------------------------------------------------------------- 1 | rbac: 2 | create: true 3 | rules: 4 | - apiGroups: [""] 5 | resources: ["pods/attach","pods/exec"] 6 | verbs: ["create", "patch", "delete"] 7 | - apiGroups: [""] 8 | resources: ["pods","services"] 9 | verbs: ["get","watch","create", "delete"] 10 | - apiGroups: [""] 11 | resources: ["configmaps","secrets"] 12 | verbs: ["get","create","update","delete"] 13 | 14 | runners: 15 | tags: "windows,kubernetes" -------------------------------------------------------------------------------- /gitlab-runner/linux-values.yaml: -------------------------------------------------------------------------------- 1 | rbac: 2 | create: true 3 | rules: 4 | - apiGroups: [""] 5 | resources: ["pods/attach","pods/exec"] 6 | verbs: ["create", "patch", "delete", "get"] 7 | - apiGroups: [""] 8 | resources: ["pods","services"] 9 | verbs: ["get","watch","create", "delete"] 10 | - apiGroups: [""] 11 | resources: ["configmaps","secrets"] 12 | verbs: ["get","create","update","delete"] 13 | 14 | runners: 15 | tags: "linux,kubernetes" -------------------------------------------------------------------------------- /gitlab-runner/kaniko-values.yaml: -------------------------------------------------------------------------------- 1 | rbac: 2 | create: true 3 | rules: 4 | - apiGroups: [""] 5 | resources: ["pods/attach","pods/exec"] 6 | verbs: ["create", "patch", "delete"] 7 | - apiGroups: [""] 8 | resources: ["pods","services"] 9 | verbs: ["get","watch","create", "delete"] 10 | - apiGroups: [""] 11 | resources: ["configmaps","secrets"] 12 | verbs: ["get","create","update","delete"] 13 | 14 | runners: 15 | protected: true 16 | tags: "kaniko,kubernetes" -------------------------------------------------------------------------------- /gitlab-runner/dockerhub-values.yaml: -------------------------------------------------------------------------------- 1 | rbac: 2 | create: true 3 | rules: 4 | - apiGroups: [""] 5 | resources: ["pods/attach","pods/exec"] 6 | verbs: ["create", "patch", "delete"] 7 | - apiGroups: [""] 8 | resources: ["pods","services"] 9 | verbs: ["get","watch","create", "delete"] 10 | - apiGroups: [""] 11 | resources: ["configmaps","secrets"] 12 | verbs: ["get","create","update","delete"] 13 | 14 | runners: 15 | imagePullPolicy: always 16 | tags: "dockerhub,kubernetes" -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | #Deployment outputs 2 | 3 | output "gitlab_ext_ip" { 4 | description = "Gitlab's external IP address." 5 | value = google_compute_instance.gitlab.network_interface.0.access_config.0.nat_ip 6 | } 7 | 8 | output "lb_ext_ip" { 9 | description = "Load balancer external IP address." 10 | value = google_compute_global_forwarding_rule.lb_forward_rule.ip_address 11 | } 12 | 13 | output "gitlab_root_password_secret" { 14 | description = "Secret name where Gitlab (Web UI) root account's password is stored" 15 | value = google_secret_manager_secret.gitlab_initial_root_pwd.secret_id 16 | } -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">=1.5.0" 3 | 4 | required_providers { 5 | google = { 6 | source = "hashicorp/google" 7 | version = ">=4.77.0" 8 | } 9 | kubernetes = { 10 | source = "hashicorp/kubernetes" 11 | version = ">= 2.10.0" 12 | } 13 | helm = { 14 | source = "hashicorp/helm" 15 | version = ">= 2.5.1" 16 | } 17 | tls = { 18 | source = "hashicorp/tls" 19 | version = ">= 3.3.0" 20 | } 21 | google-beta = { 22 | source = "hashicorp/google-beta" 23 | version = ">= 4.44.0" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # Crash log files 9 | crash.log 10 | 11 | # Ignore override files as they are usually used to override resources locally and so 12 | # are not checked in 13 | override.tf 14 | override.tf.json 15 | *_override.tf 16 | *_override.tf.json 17 | 18 | 19 | # Exclude Terraform lock file 20 | .terraform.lock.hcl 21 | 22 | #Exclude state and config files 23 | 24 | backend.tf 25 | current*.tfvars 26 | 27 | # Ignore downloaded backups 28 | gitlab_backups/** 29 | 30 | # Certificates 31 | *.crt 32 | *.key 33 | 34 | # k8s 35 | kubeconfig -------------------------------------------------------------------------------- /provider.tf: -------------------------------------------------------------------------------- 1 | provider "google" { 2 | alias = "offensive-pipeline" 3 | project = var.project_id 4 | } 5 | 6 | provider "google" { 7 | alias = "dns_infra" 8 | project = var.dns_project_id 9 | } 10 | 11 | data "google_client_config" "default" {} 12 | 13 | provider "kubernetes" { 14 | host = "https://${module.gke.endpoint}" 15 | token = data.google_client_config.default.access_token 16 | cluster_ca_certificate = base64decode(module.gke.ca_certificate) 17 | } 18 | 19 | provider "helm" { 20 | kubernetes { 21 | host = "https://${module.gke.endpoint}" 22 | token = data.google_client_config.default.access_token 23 | cluster_ca_certificate = base64decode(module.gke.ca_certificate) 24 | } 25 | } -------------------------------------------------------------------------------- /scripts/bash/gitlab_backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" 4 | 5 | #Imports 6 | DEPLOYMENT_GCS_PREFIX=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gcs-prefix` 7 | GCLOUD_LOG_NAME="gitlab-backup-exec" 8 | 9 | gsutil cp $DEPLOYMENT_GCS_PREFIX/scripts/bash/gcloud_logger.sh ./ 10 | source ./gcloud_logger.sh 11 | gsutil cp $DEPLOYMENT_GCS_PREFIX/scripts/bash/gitlab_helpers.sh ./ 12 | source ./gitlab_helpers.sh 13 | 14 | 15 | 16 | check_installation $GCLOUD_LOG_NAME 17 | 18 | if [ $GITLAB_INSTALLED == 'true' ]; then 19 | logger $GCLOUD_LOG_NAME "INFO" "Performing gitlab backup" 20 | get_backup_archive_password $GCLOUD_LOG_NAME 21 | execute_backup $GCLOUD_LOG_NAME $GITLAB_EE_VERSION 22 | 23 | else 24 | logger $GCLOUD_LOG_NAME "ERROR" "Gitlab installation was not found!" 25 | fi 26 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 SYGNIA 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /debug.tf: -------------------------------------------------------------------------------- 1 | ### Debugging resources #### 2 | 3 | # Identity Aware Proxy (IAP) 4 | # IAP is GCP feature used to connect to GCE over SSH/RDP with browser or IAP Desktop 5 | # Adding FW rule for connecting to linux nodes, windows nodes and Gitlab instance via IAP 6 | resource "google_compute_firewall" "iap_pipeline" { 7 | count = var.debug_flag ? 1 : 0 8 | name = "${var.infra_name}-allow-iap" 9 | network = module.gcp-network.network_name 10 | provider = google.offensive-pipeline 11 | allow { 12 | protocol = "tcp" 13 | ports = ["22","3389"] 14 | } 15 | 16 | source_ranges = ["35.235.240.0/20"] 17 | target_tags = [ 18 | local.gitlab_instance_name, 19 | local.gke_linux_pool_tag, 20 | local.gke_win_pool_tag 21 | ] 22 | } 23 | 24 | # DEBUG: Save Gitlab Server certificate. 25 | resource "local_file" "tls_key_pem_file" { 26 | count = var.debug_flag ? 1 : 0 27 | content = tls_private_key.gitlab-self-signed-cert-key.private_key_pem 28 | filename = "${path.module}/gitlab.local.key" 29 | } 30 | resource "local_file" "tls_cert_pem_file" { 31 | count = var.debug_flag ? 1 : 0 32 | content = tls_self_signed_cert.gitlab-self-signed-cert.cert_pem 33 | filename = "${path.module}/gitlab.local.crt" 34 | } 35 | 36 | # DEBUG: Save kube config file 37 | 38 | module "gke_auth" { 39 | count = var.debug_flag ? 1 : 0 40 | source = "terraform-google-modules/kubernetes-engine/google//modules/auth" 41 | project_id = var.project_id 42 | cluster_name = module.gke.name 43 | location = module.gke.location 44 | } 45 | 46 | resource "local_file" "kubeconfig" { 47 | depends_on = [module.gke_auth] 48 | count = var.debug_flag ? 1 : 0 49 | content = module.gke_auth[0].kubeconfig_raw 50 | filename = "${path.module}/kubeconfig" 51 | } 52 | -------------------------------------------------------------------------------- /network.tf: -------------------------------------------------------------------------------- 1 | # VPC and subnets 2 | 3 | module "gcp-network" { 4 | source = "terraform-google-modules/network/google" 5 | project_id = var.project_id 6 | network_name = "${var.infra_name}-offensive-pipeline-vpc" 7 | 8 | subnets = [ 9 | { 10 | subnet_name = "${var.infra_name}-offensive-pipeline-subnet" 11 | subnet_ip = local.vpc_main_subnet 12 | subnet_region = var.region 13 | } 14 | ] 15 | 16 | secondary_ranges = { 17 | ("${var.infra_name}-offensive-pipeline-subnet") = [ 18 | { 19 | range_name = "${var.infra_name}-gke-pods-subnet" 20 | ip_cidr_range = local.gke_pod_subnet 21 | }, 22 | { 23 | range_name = "${var.infra_name}-gke-service-subnet" 24 | ip_cidr_range = local.gke_svc_subnet 25 | } 26 | ] 27 | } 28 | } 29 | 30 | 31 | # Operators access rule 32 | resource "google_compute_firewall" "operators" { 33 | name = "${var.infra_name}-operators-access" 34 | network = module.gcp-network.network_name 35 | provider = google.offensive-pipeline 36 | source_ranges = var.operator_ips 37 | target_tags = [local.gitlab_instance_name] 38 | 39 | allow { 40 | protocol = "tcp" 41 | ports = var.operator_ports 42 | } 43 | } 44 | 45 | 46 | # Rule allowing internal access from K8s' Pods to Gitlab 47 | resource "google_compute_firewall" "pods-to-gitlab-access" { 48 | name = "${var.infra_name}-gke-pods-gitlab-access" 49 | network = module.gcp-network.network_name 50 | provider = google.offensive-pipeline 51 | source_ranges = [local.gke_pod_subnet] 52 | target_tags = [local.gitlab_instance_name] 53 | 54 | allow { 55 | protocol = "tcp" 56 | ports = var.operator_ports 57 | } 58 | } 59 | 60 | 61 | 62 | # DNS Record 63 | 64 | resource "google_dns_record_set" "ext-dns" { 65 | provider = google.dns_infra 66 | count = var.external_hostname != "" ? 1 : 0 67 | name = "${var.external_hostname}." 68 | type = "A" 69 | ttl = var.dns_record_ttl 70 | managed_zone = var.dns_managed_zone_name 71 | rrdatas = [google_compute_global_forwarding_rule.lb_forward_rule.ip_address] 72 | } 73 | 74 | -------------------------------------------------------------------------------- /scripts/bash/gcloud_logger.sh: -------------------------------------------------------------------------------- 1 | logger() { 2 | # Param 1: Log name 3 | # Param 2: Module name 4 | # Param 3: Level (INFO, DEBUG, ERROR) 5 | # Param 4: Message 6 | local log=$1 7 | local module=`hostname` 8 | local level=$2 9 | local message=$3 10 | error_levels=("INFO" "DEBUG" "ERROR") 11 | # Validate 12 | if [ $# -ne 3 ]; then 13 | echo "Bad params" 14 | fi 15 | # Check if error level is valid 16 | if ! [[ " ${error_levels[@]} " =~ " ${level} " ]]; then 17 | gcloud logging write $log "$module: Invalid log error level." --severity="ERROR" 18 | level="DEBUG" 19 | fi 20 | echo "$level: $message" 21 | gcloud logging write $log "$module: $message" --severity=$level 22 | } 23 | 24 | 25 | 26 | ### Try / Except execution wrapper with GCloud logging ### 27 | # Usage 1: exec_wrapper $ERR_ACTION_EXIT $LOGNAME "My Bash Command" # <- This will log an error (To LOGNAME) and exit the script in case of command error 28 | # Usage 2: exec_wrapper $ERR_ACTION_CONT $LOGNAME "My Bash Command" # <- This will only log an error (To LOGNAME) in case of command error 29 | # Successful command outputs will be printed out 30 | 31 | readonly ERR_ACTION_EXIT="Exit" 32 | readonly ERR_ACTION_CONT="Continue" 33 | readonly STD_OUT_PATH="/tmp/stdoutput" 34 | readonly STD_ERR_PATH="/tmp/stderr" 35 | 36 | exec_wrapper () { 37 | 38 | local errAction=$1 39 | local logName=$2 40 | local cmdExec=$3 41 | echo "wrapper exec: $cmdExec" 42 | $cmdExec 1>$STD_OUT_PATH 2>$STD_ERR_PATH 43 | 44 | local errCode=$? 45 | if [ $errCode -ne 0 ]; then 46 | errMsg=$(cat $STD_ERR_PATH) 47 | logger $logName "ERROR" "ErrCode: $errCode, ErrAction: $errAction, Message: $errMsg" 48 | if [[ $errAction == $ERR_ACTION_EXIT ]]; then 49 | logger $logName "DEBUG" "Stopping execution due to error action" 50 | exit 1 51 | fi 52 | fi 53 | cat $STD_OUT_PATH 54 | } 55 | 56 | exec_wrapper_no_print () { 57 | local errAction=$1 58 | local logName=$2 59 | local cmdExec=$3 60 | echo "wrapper exec: no print." 61 | $cmdExec 2> $STD_ERR_PATH 62 | 63 | local errCode=$? 64 | if [ $errCode -ne 0 ]; then 65 | errMsg=$(cat $STD_ERR_PATH) 66 | logger $logName "ERROR" "ErrCode: $errCode, ErrAction: $errAction, Message: $errMsg" 67 | if [[ $errAction == $ERR_ACTION_EXIT ]]; then 68 | logger $logName "DEBUG" "Stopping execution due to error action" 69 | exit 1 70 | fi 71 | fi 72 | } -------------------------------------------------------------------------------- /locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | gitlab_instance_name = "${var.infra_name}-gitlab" 3 | instance_internal_domain = "${local.gitlab_instance_name}.${var.region}-${var.zone}.c.${var.project_id}.internal" 4 | instance_internal_url = "${var.gitlab_instance_protocol}://${local.instance_internal_domain}" 5 | gitlab_package_dl_link = join("/", [ 6 | "https://packages.gitlab.com/gitlab/gitlab-ee/packages", 7 | "${var.os_name}", 8 | "${var.os_release}", 9 | "gitlab-ee_${var.gitlab_version}.0_amd64.deb", 10 | "download.deb" 11 | ]) 12 | vpc_main_subnet = "10.0.0.0/22" # 10.0.0.0 - 10.0.3.255 , 1024 IPs 13 | gke_pod_subnet = "10.2.0.0/17" # 10.2.0.0 - 10.2.127.255 - 32,768 IPs. 14 | gke_svc_subnet = "10.2.128.0/20" # 10.2.128.0 - 10.2.143.255 - 4096 IPs. 15 | artifact_registry_host = "${var.region}-docker.pkg.dev" 16 | artifact_registry_id = "${var.infra_name}-containers" 17 | artifact_registry_namespace = "${var.project_id}/${local.artifact_registry_id}" 18 | gke_linux_pool_tag = "gke-${var.infra_name}-offensive-pipeline-gke-linux-pool" 19 | gke_win_pool_tag = "gke-${var.infra_name}-offensive-pipeline-windows-pool" 20 | gke_win_pool_start_script = join("/", [ 21 | "gs:/", 22 | google_storage_bucket.deployment_utils.name, 23 | google_storage_bucket_object.disable_windows_defender_ps.name 24 | ]) 25 | 26 | gitlab_startup_script = join("/", [ 27 | "gs:/", 28 | google_storage_bucket.deployment_utils.name, 29 | google_storage_bucket_object.gitlab_startup_script.name 30 | ]) 31 | 32 | gitlab_migrate_backup = var.migrate_gitlab ? join("/", [ 33 | "gs:/", 34 | google_storage_bucket.deployment_utils.name, 35 | var.migrate_gitlab_backup_path 36 | ]) : "" 37 | } -------------------------------------------------------------------------------- /scripts/bash/gitlab_startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Gitlab starup script 4 | # This script will either install, migrate or reconfigure Gitlab installation according to the values set in the instance metadata. 5 | # Installation / Migration will be executed once in a life of an instance. 6 | # Reconfiguration will be executed on every boot. 7 | # Reconfiguration updates the Gitlab's External URL and certificates if they changed through metadata. 8 | 9 | 10 | #Vars 11 | DEPLOYMENT_GCS_PREFIX=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gcs-prefix` 12 | GITLAB_TARGET_INSTALL_VERSION=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/target-gitlab-version` 13 | GITLAB_PKG_LINK=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-package-dl-link` 14 | GCS_PATH_TO_BACKUP=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gcs-path-to-backup` 15 | GCLOUD_LOG_NAME="gitlab-startup" 16 | BACKUP_ARCHIVE_PATH="/tmp/backup_archived.zip" 17 | 18 | #Imports 19 | gsutil cp $DEPLOYMENT_GCS_PREFIX/scripts/bash/gitlab_helpers.sh ./ 20 | gsutil cp $DEPLOYMENT_GCS_PREFIX/scripts/bash/gcloud_logger.sh ./ 21 | source ./gcloud_logger.sh 22 | source ./gitlab_helpers.sh 23 | 24 | 25 | # Start 26 | logger $GCLOUD_LOG_NAME "INFO" "Starting Gitlab instance setup" 27 | 28 | check_installation $GCLOUD_LOG_NAME 29 | set_gitlab_vars $GCLOUD_LOG_NAME 30 | 31 | if [ $GITLAB_INSTALLED == 'false' ]; then 32 | logger $GCLOUD_LOG_NAME "INFO" "Starting Gitlab installation" 33 | gitlab_deps_install $GCLOUD_LOG_NAME 34 | gitlab_install $GCLOUD_LOG_NAME $GITLAB_PKG_LINK 35 | setup_cicd_vars $GCLOUD_LOG_NAME 36 | 37 | if [ $GCS_PATH_TO_BACKUP == 'NONE' ]; then 38 | create_groups $GCLOUD_LOG_NAME 39 | import_scallops_recipes $GCLOUD_LOG_NAME 40 | create_cicd_vars $GCLOUD_LOG_NAME 41 | 42 | else 43 | logger $GCLOUD_LOG_NAME "INFO" "Migrating Gitlab from provided backup $GCS_PATH_TO_BACKUP" 44 | get_backup_archive $GCLOUD_LOG_NAME $GCS_PATH_TO_BACKUP $BACKUP_ARCHIVE_PATH 45 | get_backup_archive_password $GCLOUD_LOG_NAME 46 | restore_backup $GCLOUD_LOG_NAME $BACKUP_ARCHIVE_PATH 47 | update_cicd_vars $GCLOUD_LOG_NAME 48 | fi 49 | 50 | seed_instance_reg_token $GCLOUD_LOG_NAME 51 | seed_gitlab_root_pwd $GCLOUD_LOG_NAME 52 | seed_scallops_recipes_runner_token $GCLOUD_LOG_NAME 53 | setup_gitlab_backup $GCLOUD_LOG_NAME $DEPLOYMENT_GCS_PREFIX 54 | 55 | else 56 | logger $GCLOUD_LOG_NAME "INFO" "Skipping Gitlab installation" 57 | check_upgrade $GCLOUD_LOG_NAME $GITLAB_TARGET_INSTALL_VERSION 58 | fi 59 | 60 | 61 | reconfigure_gitlab $GCLOUD_LOG_NAME 62 | logger $GCLOUD_LOG_NAME "INFO" "Gitlab instance setup completed" -------------------------------------------------------------------------------- /storage.tf: -------------------------------------------------------------------------------- 1 | # CICD utilities Storage 2 | 3 | resource "random_string" "deployment_utils_bucket_suffix" { 4 | length = 4 5 | special = false 6 | lower = true 7 | upper = false 8 | } 9 | 10 | resource "google_storage_bucket" "deployment_utils" { 11 | name = "${var.infra_name}-utils-${random_string.deployment_utils_bucket_suffix.result}" 12 | location = "US" 13 | storage_class = "STANDARD" 14 | uniform_bucket_level_access = true 15 | provider = google.offensive-pipeline 16 | force_destroy = true 17 | } 18 | 19 | 20 | resource "google_storage_bucket_object" "disable_windows_defender_ps" { 21 | name = "scripts/Powershell/disabledefender.ps1" 22 | bucket = google_storage_bucket.deployment_utils.name 23 | source = "${path.module}/scripts/Powershell/disabledefender.ps1" 24 | } 25 | 26 | 27 | resource "google_storage_bucket_object" "gitlab_startup_script" { 28 | depends_on = [ 29 | google_storage_bucket_object.gitlab_helpers_script, 30 | google_storage_bucket_object.gcloud_logger_script, 31 | google_storage_bucket_object.gitlab_backup_script] 32 | 33 | name = "scripts/bash/gitlab_startup.sh" 34 | bucket = google_storage_bucket.deployment_utils.name 35 | source = "${path.module}/scripts/bash/gitlab_startup.sh" 36 | } 37 | 38 | resource "google_storage_bucket_object" "gitlab_helpers_script" { 39 | name = "scripts/bash/gitlab_helpers.sh" 40 | bucket = google_storage_bucket.deployment_utils.name 41 | source = "${path.module}/scripts/bash/gitlab_helpers.sh" 42 | } 43 | 44 | resource "google_storage_bucket_object" "gcloud_logger_script" { 45 | name = "scripts/bash/gcloud_logger.sh" 46 | bucket = google_storage_bucket.deployment_utils.name 47 | source = "${path.module}/scripts/bash/gcloud_logger.sh" 48 | } 49 | 50 | resource "google_storage_bucket_object" "gitlab_backup_script" { 51 | depends_on = [google_storage_bucket_object.gitlab_helpers_script, 52 | google_storage_bucket_object.gcloud_logger_script] 53 | 54 | name = "scripts/bash/gitlab_backup.sh" 55 | bucket = google_storage_bucket.deployment_utils.name 56 | source = "${path.module}/scripts/bash/gitlab_backup.sh" 57 | } 58 | 59 | # Migration resource 60 | 61 | # Transfer is done in this way since state file won't support big files in resource attributes. 62 | resource "null_resource" "transfer_gitlab_backup" { 63 | count = var.migrate_gitlab ? 1 : 0 64 | triggers = { 65 | gcs_backup_path = "gs://${var.migrate_gitlab_backup_bucket}/${var.migrate_gitlab_backup_path}" 66 | } 67 | provisioner "local-exec" { 68 | when = create 69 | command = "gsutil cp gs://${var.migrate_gitlab_backup_bucket}/${var.migrate_gitlab_backup_path} ${local.gitlab_migrate_backup}" 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | 4 | ## [0.6] - 2024-08-19 5 | ### Added 6 | - Added LB resource and related resources 7 | - Added application external Certificate service 8 | - Modify DNS record set to LB IP instead of Gitlab instance 9 | - Assigned Gitlab to instance group 10 | - Added option to specify external integration IP addresses. 11 | - Upgraded google terraform provider 12 | 13 | 14 | ## [0.5] - 2023-07-19 15 | ### Added 16 | - Enabled runner session server on linux-runner to allow the use of Interactive Web Terminal (Job Debug button) 17 | 18 | ### Changed 19 | - Shortened runner poll intervals 20 | - Optimize K8s performance 21 | - Upgraded GKE to version 1.26.5 22 | - Containers are now stored in Artifact Registry instead of deprecating Container Registry (GCR). 23 | - Modified node pools disk type to SSD 24 | - Modified Linux node image type to COS_CONTAINERD 25 | - Enabled image streaming 26 | - Upgraded related providers and modules 27 | - Modified k8s pdb resources according to provider upgrades 28 | - Gitlab 16.1.2 upgrade 29 | - Default to Gitlab 16.1.2-ee and runner 0.54.0 30 | - Option to set target Gitlab version to upgrade the application from tf vars 31 | - Added relevant locals 32 | - Added multiple ubuntu release options for the instance 33 | - Changed runner TOML config to be set from terraform instead of Helm chart Yaml. 34 | - Modified Gitlab instance disk to 160G SSD. 35 | - Added Gitlab DEB package link metadata variable. 36 | 37 | ### Fixed 38 | - Backup creation bug. The created backup snapshot was not deleted after upload, causing overload to the local disk. 39 | - Modified IAM binding to add memeber instead of overwrite all members on user resources. 40 | - Fixed Gitlab restore bug via new exec-wrapper function. 41 | 42 | ## [0.4] - 2022-10-22 43 | ### Added 44 | - Option to specify Scallop-Recipes repo URL (For auto import) 45 | - Error handling for Gitlab installation and backup Bash scripts 46 | - Deployment TF debug option - IAP firewall rules, export configuration files 47 | - Integration with GCP logger for Gitlab installation and backup 48 | 49 | ### Changed 50 | - README + architecture image 51 | - Updated K8s RBAC permissions for runners 52 | - Removed generated backup password. Password secret is supplied by the user. 53 | - Updated tfvars template 54 | - Upgraded Gitlab, K8s, Runners versions 55 | - Adjusted instance level CI/CD variables names 56 | - Upgraded Windows node pool machine type to 4 vCPU & 16GB RAM 57 | - Now limiting concurrent jobs for 30 in Windows and 50 in Linux 58 | - K8s nodes are now preemptible 59 | 60 | ### Fixed 61 | - Fixed TF resources dependencies 62 | - Fixed Defender disarm detection issue 63 | - Fixed K8s Scale up issues 64 | 65 | ## [0.3] - 2022-01-07 66 | ### Added 67 | - Added support to migrate from an older Gitlab instance 68 | - Added Gitlab instance automatic backup capability 69 | - K8s Pod disruption budget resources to fix scale down issue 70 | - New runner that supports Dockerhub credentials to pull private images 71 | 72 | ### Changed 73 | - Modified max pods per node in K8s, meanning more CI jobs simoultaneously 74 | - Scopred bucket admin permissions to conainer registry bucket only 75 | - Reorganized resources into seperate files 76 | - Increased CPUs on linux pool 77 | - Removed Gitlab API token seeding. Gitlab modifications performed using gitlab-rails console 78 | 79 | ### Fixed 80 | - Windows Runner configuration 81 | - Minor fixes, duplicate code removal 82 | - Fixed Linux won't scale down problem 83 | 84 | 85 | ## [0.2] - 2021-12-21 86 | ### Fixed 87 | - Windows Runner configuration 88 | - Relativ paths 89 | - Runner version update 90 | 91 | 92 | ## [0.1] - 2021-08-09 93 | ### Initial 94 | - Initial working deployment 95 | -------------------------------------------------------------------------------- /config.tfvars: -------------------------------------------------------------------------------- 1 | ##### Scallops IAC variables ##### 2 | #### Note that some variables are required (#Required), and some variables modifications will take effect also after deployment (#PostDeploymentModifiable). 3 | 4 | 5 | ## GCP Project ID 6 | project_id = "" #Required 7 | 8 | ## The name you wish to have as a prefix for the deployment's resources. Must comply with ^[a-z]([a-z0-9]*[a-z0-9])$ 9 | infra_name = "scallops" #Required 10 | 11 | ## The name of an existing bucket you wish to receive backups to. Terraform will create the required permission to upload the backup archive. 12 | backups_bucket_name = "" #Required #PostDeploymentModifiable 13 | 14 | ## An existing secret ID in the same GCP project (project_id) storing a password for the backup process (Allowed symbols for secret value: -_ ) 15 | ## Creating a secret through GCP secret manager https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create 16 | gitlab_backup_key_secret_id = "" #Required #PostDeploymentModifiable 17 | 18 | 19 | 20 | ## Gitlab version to install 21 | ## Ruuner chart version must be compaitibe with the Gitlab version -> https://docs.gitlab.com/runner/#gitlab-runner-versions 22 | ## Note the Gitlab application version from the selected Chart version -> https://artifacthub.io/packages/helm/gitlab/gitlab-runner 23 | ## You can make upgrades to your Gitlab instance from here. Just reset the instance once the `apply` completes. 24 | # gitlab_version = "16.1.2-ee" 25 | # runner_chart_url = "https://gitlab-charts.s3.amazonaws.com/gitlab-runner-0.54.0.tgz" 26 | 27 | 28 | ## IP addresses that can interact with the Gitlab instance via HTTP/S (Office IP / Home IPs) 29 | # operator_ips = [] #Optional #PostDeploymentModifiable 30 | 31 | ## Enable debugging resources such as IAP Firewall rules, and export of config files 32 | # debug_flag = false #Optional #PostDeploymentModifiable 33 | 34 | ## The Gitlab instance Web server protocol, http or https. 35 | # gitlab_instance_protocol = "https" #Optional 36 | 37 | ## Region for the k8s cluster, Gitlab instance and network. 38 | # region = us-central-1 #Optional 39 | 40 | ## Zone for the k8s cluster, Gitlab instance and network. 41 | # zone = "a" #Optional 42 | 43 | 44 | 45 | ## External DNS ## #Optional #PostDeploymentModifiable #GitlabRestartRequired 46 | ## Uncomment the 3 lines below if wishing to supply external DNS name for accessing Gitalb instance 47 | 48 | # dns_project_id = "" # The project ID where the managed DNS zone is located 49 | # dns_managed_zone_name = "mydomain-com" # The configured managed DNS zone name 50 | # external_hostname = "scallops.mydomain.com" # The hostname you wish to set of the instance (Must be subdomain of the managed zone) 51 | 52 | 53 | 54 | ## Docker hub credentials (https://docs.docker.com/docker-hub/access-tokens/) #Optional #PostDeploymentModifiable 55 | ## An existing secret name in secret-manager storing Dockerhub credentials to fetch private container images (format is username:password or username:access-token). 56 | # dockerhub-creds-secret = "" 57 | 58 | 59 | ## Scallops-Recipes repository. Use the default repository or specify alternative fork in a Git path HTTPS format. 60 | ## The specified repository will be imported to Gitlab as the Scallops-Recipes repository. 61 | ## *Ignored if performing a migration 62 | # scallops_recipes_git_url = "https://github.com/SygniaLabs/ScallOps-Recipes.git" #Optional 63 | # scallops_recipes_git_creds_secret = "my-github-creds-secret" #Optional 64 | 65 | 66 | 67 | ## Migration variables #Optional 68 | ## If you plan on migrating from a different gitlab instance, uncomment all migration variables below, and follow requirements. 69 | ## 1. 'gitlab_backup_key_secret_id' secret must store the password value decrypting the archived backup zip. 70 | ## 2. 'gitlab_version' must be equal to the version you are migrating from. 71 | ## 3. Operation requires Gsutil on the terraform deployer system as backup will be downloaded locally 72 | 73 | # migrate_gitlab = true ## If performing migration from another Gitlab instance and got a backup file from previous instance. true/false. 74 | # migrate_gitlab_backup_bucket = "" ## The Google Storage Bucket to your Gitlab backup e.g. 'mybucket1-abcd' 75 | # migrate_gitlab_backup_path = "" ## The path to the archived backup zip e.g 'backups/gitlab-xxx-backup.zip' -------------------------------------------------------------------------------- /secrets.tf: -------------------------------------------------------------------------------- 1 | #### Secret creation and version manager #### 2 | 3 | 4 | # Self signed TLS certificate generation 5 | 6 | resource "tls_private_key" "gitlab-self-signed-cert-key" { 7 | algorithm = "ECDSA" 8 | ecdsa_curve = "P384" 9 | } 10 | 11 | resource "tls_self_signed_cert" "gitlab-self-signed-cert" { 12 | private_key_pem = tls_private_key.gitlab-self-signed-cert-key.private_key_pem 13 | 14 | subject { 15 | common_name = "gitlab.local" 16 | organization = "Company" 17 | } 18 | 19 | dns_names = flatten([ 20 | "${local.gitlab_instance_name}.local", 21 | local.instance_internal_domain, 22 | var.external_hostname != "" ? [var.external_hostname] : [] 23 | ]) 24 | 25 | ip_addresses = ["10.0.0.2"] 26 | validity_period_hours = 87600 //Certificate will be valid for 10 years 27 | 28 | allowed_uses = [ 29 | "key_encipherment", 30 | "digital_signature" 31 | ] 32 | } 33 | 34 | 35 | # Gitlab server certificate 36 | 37 | resource "google_secret_manager_secret" "gitlab-self-signed-cert-key" { 38 | provider = google.offensive-pipeline 39 | secret_id = "${local.gitlab_instance_name}-cert-key" 40 | labels = { 41 | label = "gitlab-cert" 42 | } 43 | replication { 44 | user_managed { 45 | replicas { 46 | location = var.region 47 | } 48 | } 49 | } 50 | } 51 | 52 | 53 | resource "google_secret_manager_secret_version" "gitlab-self-signed-cert-key-version" { 54 | secret = google_secret_manager_secret.gitlab-self-signed-cert-key.id 55 | secret_data = tls_private_key.gitlab-self-signed-cert-key.private_key_pem 56 | } 57 | 58 | 59 | resource "google_secret_manager_secret" "gitlab-self-signed-cert-crt" { 60 | provider = google.offensive-pipeline 61 | secret_id = "${local.gitlab_instance_name}-cert-crt" 62 | labels = { 63 | label = "gitlab-cert" 64 | } 65 | replication { 66 | user_managed { 67 | replicas { 68 | location = var.region 69 | } 70 | } 71 | } 72 | } 73 | 74 | 75 | resource "google_secret_manager_secret_version" "gitlab-self-signed-cert-crt-version" { 76 | secret = google_secret_manager_secret.gitlab-self-signed-cert-crt.id 77 | secret_data = tls_self_signed_cert.gitlab-self-signed-cert.cert_pem 78 | } 79 | 80 | 81 | 82 | # Gitlab installation & deployment 83 | 84 | resource "random_password" "gitlab_runner_registration_token" { 85 | length = 20 86 | special = true 87 | override_special = "-_" 88 | } 89 | resource "random_password" "gitlab_initial_root_pwd" { 90 | length = 16 91 | special = true 92 | override_special = "-_" 93 | } 94 | 95 | 96 | # Gitlab runner registration token 97 | 98 | resource "google_secret_manager_secret" "gitlab_runner_registration_token" { 99 | provider = google.offensive-pipeline 100 | secret_id = "${local.gitlab_instance_name}-runner-reg" 101 | labels = { 102 | label = "gitlab" 103 | } 104 | replication { 105 | user_managed { 106 | replicas { 107 | location = var.region 108 | } 109 | } 110 | } 111 | } 112 | 113 | 114 | resource "google_secret_manager_secret_version" "gitlab_runner_registration_token" { 115 | secret = google_secret_manager_secret.gitlab_runner_registration_token.id 116 | secret_data = random_password.gitlab_runner_registration_token.result 117 | } 118 | 119 | 120 | # Gitlab initial root password 121 | resource "google_secret_manager_secret" "gitlab_initial_root_pwd" { 122 | provider = google.offensive-pipeline 123 | secret_id = "${local.gitlab_instance_name}-root-password" 124 | labels = { 125 | label = "gitlab" 126 | } 127 | replication { 128 | user_managed { 129 | replicas { 130 | location = var.region 131 | } 132 | } 133 | } 134 | } 135 | 136 | 137 | resource "google_secret_manager_secret_version" "gitlab_initial_root_pwd" { 138 | secret = google_secret_manager_secret.gitlab_initial_root_pwd.id 139 | secret_data = random_password.gitlab_initial_root_pwd.result 140 | } 141 | 142 | 143 | # Docker hub credentials secret 144 | 145 | data "google_secret_manager_secret_version" "dockerhub-secret" { 146 | provider = google.offensive-pipeline 147 | count = var.dockerhub-creds-secret != "" ? 1 : 0 148 | secret = var.dockerhub-creds-secret 149 | } -------------------------------------------------------------------------------- /load-balancer.tf: -------------------------------------------------------------------------------- 1 | resource "google_compute_managed_ssl_certificate" "lb_proxy_ssl_cert" { 2 | provider = google.offensive-pipeline 3 | count = var.external_hostname != "" ? 1 : 0 4 | name = "${local.gitlab_instance_name}-ssl-cert" 5 | managed { 6 | domains = [var.external_hostname] 7 | } 8 | } 9 | 10 | resource "google_compute_health_check" "gitlab_service_hc" { 11 | provider = google.offensive-pipeline 12 | name = "${local.gitlab_instance_name}-backend-hc" 13 | timeout_sec = 5 14 | check_interval_sec = 30 15 | healthy_threshold = 1 16 | unhealthy_threshold = 3 17 | 18 | dynamic "http_health_check" { 19 | for_each = var.gitlab_instance_protocol == "http" ? [1] : [] 20 | content { 21 | port_name = "http" 22 | port_specification = "USE_NAMED_PORT" 23 | request_path = "/robots.txt" 24 | proxy_header = "NONE" 25 | } 26 | } 27 | 28 | dynamic "https_health_check" { 29 | for_each = var.gitlab_instance_protocol == "https" ? [1] : [] 30 | content { 31 | port_name = "https" 32 | port_specification = "USE_NAMED_PORT" 33 | request_path = "/robots.txt" 34 | proxy_header = "NONE" 35 | } 36 | } 37 | 38 | } 39 | 40 | resource "google_compute_security_policy" "lb_allow_operators_policy" { 41 | provider = google.offensive-pipeline 42 | name = "${local.gitlab_instance_name}-backend-allowed-ips" 43 | 44 | 45 | 46 | rule { 47 | action = "allow" 48 | priority = "1000" 49 | description = "Allow specific IPs" 50 | match { 51 | versioned_expr = "SRC_IPS_V1" 52 | config { 53 | src_ip_ranges = var.operator_ips 54 | } 55 | } 56 | 57 | } 58 | 59 | # Conditionally create rule for external integration address ranges 60 | dynamic "rule" { 61 | for_each = var.external_integration_ranges != null ? [1] : [] 62 | content { 63 | action = "allow" 64 | priority = 1001 65 | description = "External Integration IP addresses" 66 | 67 | match { 68 | versioned_expr = "SRC_IPS_V1" 69 | config { 70 | src_ip_ranges = var.external_integration_ranges 71 | } 72 | } 73 | } 74 | } 75 | 76 | 77 | rule { 78 | action = "deny(502)" 79 | priority = "2147483647" 80 | description = "Default rule deny all addresses" 81 | match { 82 | versioned_expr = "SRC_IPS_V1" 83 | config { 84 | src_ip_ranges = ["*"] 85 | } 86 | } 87 | 88 | } 89 | } 90 | 91 | resource "google_compute_backend_service" "gitlab_instance_service" { 92 | provider = google.offensive-pipeline 93 | name = "${local.gitlab_instance_name}-backend" 94 | port_name = var.gitlab_instance_protocol 95 | protocol = upper(var.gitlab_instance_protocol) 96 | timeout_sec = 10 97 | security_policy = google_compute_security_policy.lb_allow_operators_policy.self_link 98 | backend { 99 | group = google_compute_instance_group.gitlab_instance.self_link 100 | } 101 | health_checks = [ 102 | google_compute_health_check.gitlab_service_hc.self_link, 103 | ] 104 | } 105 | 106 | resource "google_compute_url_map" "service_url_map" { 107 | provider = google.offensive-pipeline 108 | name = "${local.gitlab_instance_name}-urlmap" 109 | default_service = google_compute_backend_service.gitlab_instance_service.self_link 110 | } 111 | 112 | resource "google_compute_target_https_proxy" "lb_https_proxy" { 113 | provider = google.offensive-pipeline 114 | name = "${local.gitlab_instance_name}-https-proxy" 115 | url_map = google_compute_url_map.service_url_map.self_link 116 | ssl_certificates = ( 117 | var.external_hostname != "" ? 118 | [google_compute_managed_ssl_certificate.lb_proxy_ssl_cert.0.self_link] : [] 119 | ) 120 | } 121 | 122 | resource "google_compute_global_forwarding_rule" "lb_forward_rule" { 123 | provider = google.offensive-pipeline 124 | name = "${local.gitlab_instance_name}-fwd-rule" 125 | target = google_compute_target_https_proxy.lb_https_proxy.self_link 126 | port_range = "443" 127 | } 128 | 129 | # https://cloud.google.com/load-balancing/docs/firewall-rules 130 | resource "google_compute_firewall" "allow_lb_gitlab_access" { 131 | provider = google.offensive-pipeline 132 | name = "${local.gitlab_instance_name}-allow-lb" 133 | network = module.gcp-network.network_name 134 | source_ranges = ["130.211.0.0/22", "35.191.0.0/16"] 135 | target_tags = google_compute_instance.gitlab.tags 136 | allow { 137 | protocol = "tcp" 138 | ports = var.operator_ports 139 | } 140 | } -------------------------------------------------------------------------------- /iam.tf: -------------------------------------------------------------------------------- 1 | #### IAM svc accounts, roles and bindings #### 2 | 3 | ## Service Accounts ## 4 | 5 | # Gitlab compute instance service account 6 | resource "google_service_account" "gitlab_service_account" { 7 | account_id = "${local.gitlab_instance_name}-svc" 8 | display_name = "Gitlab Service Account" 9 | provider = google.offensive-pipeline 10 | } 11 | 12 | # Service account with capability to push container images to sepcific repository in artifact registry 13 | resource "google_service_account" "kaniko" { 14 | account_id = "${var.infra_name}-gke-bucket" 15 | display_name = "Service Account for Pods to access artifact registry repository" 16 | provider = google.offensive-pipeline 17 | } 18 | 19 | ## Roles ## 20 | 21 | resource "random_string" "custom_roles_suffix" { 22 | length = 4 23 | special = false 24 | lower = true 25 | upper = false 26 | } 27 | 28 | # Create role with permissions to backup only without read/overwrite/delete 29 | resource "google_project_iam_custom_role" "backup_archive_role" { 30 | role_id = "gitlab_backupArchive_${var.infra_name}_${random_string.custom_roles_suffix.result}" 31 | title = "Gitlab Backup Role for ${var.infra_name}" 32 | description = "A role attached to the gitlab compute service account allowing it to update new backup archives to the specified bucket (var.backups_bucket_name)." 33 | permissions = [ 34 | "storage.objects.list", 35 | "storage.objects.create", 36 | "storage.multipartUploads.create", 37 | "storage.multipartUploads.listParts", 38 | "storage.multipartUploads.abort" 39 | ] 40 | provider = google.offensive-pipeline 41 | } 42 | 43 | 44 | # Role that allows startup script write logs 45 | resource "google_project_iam_custom_role" "compute_log_role" { 46 | role_id = "gitlab_customRole_${var.infra_name}_${random_string.custom_roles_suffix.result}" 47 | title = "Write logs role for ${var.infra_name}" 48 | description = "A role attached to the gitlab compute instance allowing it to write logs." 49 | permissions = ["logging.logEntries.create"] 50 | provider = google.offensive-pipeline 51 | } 52 | 53 | 54 | ## Bindings ## 55 | 56 | # Service account user role binding 57 | resource "google_project_iam_member" "sa_binding" { 58 | project = var.project_id 59 | provider = google.offensive-pipeline 60 | role = "roles/iam.serviceAccountUser" 61 | member = "serviceAccount:${google_service_account.gitlab_service_account.email}" 62 | } 63 | 64 | # Gitlab instance IAM Binding to storage 65 | resource "google_storage_bucket_iam_binding" "binding" { 66 | bucket = google_storage_bucket.deployment_utils.name 67 | role = "roles/storage.objectViewer" # To Read startup scripts on deployment utils bucket. 68 | members = [ 69 | "serviceAccount:${google_service_account.gitlab_service_account.email}" 70 | ] 71 | } 72 | 73 | 74 | # Attach gitlab compute service account to the bucket with the backup role 75 | resource "google_storage_bucket_iam_binding" "backup_bucket_binding" { 76 | bucket = var.backups_bucket_name 77 | role = google_project_iam_custom_role.backup_archive_role.name 78 | members = [ 79 | "serviceAccount:${google_service_account.gitlab_service_account.email}" 80 | ] 81 | } 82 | 83 | # Bind the compute custom role the the service account 84 | resource "google_project_iam_binding" "compute_binding" { 85 | project = var.project_id 86 | provider = google.offensive-pipeline 87 | role = google_project_iam_custom_role.compute_log_role.name 88 | members = [ 89 | "serviceAccount:${google_service_account.gitlab_service_account.email}", 90 | ] 91 | } 92 | 93 | 94 | # Bind artifact registry write access to specific repository 95 | resource "google_artifact_registry_repository_iam_member" "writer" { 96 | project = google_artifact_registry_repository.containers.project 97 | location = google_artifact_registry_repository.containers.location 98 | repository = google_artifact_registry_repository.containers.name 99 | role = "roles/artifactregistry.writer" 100 | member = "serviceAccount:${google_service_account.kaniko.email}" 101 | } 102 | 103 | resource "google_service_account_key" "artifact_registry_writer" { 104 | service_account_id = google_service_account.kaniko.name 105 | } 106 | 107 | 108 | # IAM Bindings for Gitlab instance to secrets 109 | resource "google_secret_manager_secret_iam_binding" "gitlab-self-key-binding" { 110 | project = google_secret_manager_secret.gitlab-self-signed-cert-key.project 111 | secret_id = google_secret_manager_secret.gitlab-self-signed-cert-key.secret_id 112 | role = "roles/secretmanager.secretAccessor" 113 | members = [ 114 | "serviceAccount:${google_service_account.gitlab_service_account.email}", 115 | ] 116 | } 117 | 118 | resource "google_secret_manager_secret_iam_binding" "gitlab-self-crt-binding" { 119 | project = google_secret_manager_secret.gitlab-self-signed-cert-crt.project 120 | secret_id = google_secret_manager_secret.gitlab-self-signed-cert-crt.secret_id 121 | role = "roles/secretmanager.secretAccessor" 122 | members = [ 123 | "serviceAccount:${google_service_account.gitlab_service_account.email}", 124 | ] 125 | } 126 | 127 | 128 | resource "google_secret_manager_secret_iam_binding" "gitlab_runner_registration_token" { 129 | project = google_secret_manager_secret.gitlab_runner_registration_token.project 130 | secret_id = google_secret_manager_secret.gitlab_runner_registration_token.secret_id 131 | role = "roles/secretmanager.secretAccessor" 132 | members = [ 133 | "serviceAccount:${google_service_account.gitlab_service_account.email}", 134 | ] 135 | } 136 | 137 | 138 | resource "google_secret_manager_secret_iam_binding" "gitlab_initial_root_pwd" { 139 | project = google_secret_manager_secret.gitlab_initial_root_pwd.project 140 | secret_id = google_secret_manager_secret.gitlab_initial_root_pwd.secret_id 141 | role = "roles/secretmanager.secretAccessor" 142 | members = [ 143 | "serviceAccount:${google_service_account.gitlab_service_account.email}", 144 | ] 145 | } 146 | 147 | 148 | resource "google_secret_manager_secret_iam_member" "gitlab_backup_key" { 149 | project = var.project_id 150 | secret_id = var.gitlab_backup_key_secret_id 151 | role = "roles/secretmanager.secretAccessor" 152 | member = "serviceAccount:${google_service_account.gitlab_service_account.email}" 153 | } 154 | 155 | resource "google_secret_manager_secret_iam_member" "git_creds" { 156 | count = var.scallops_recipes_git_creds_secret != "" ? 1 : 0 157 | project = var.project_id 158 | secret_id = var.scallops_recipes_git_creds_secret 159 | role = "roles/secretmanager.secretAccessor" 160 | member = "serviceAccount:${google_service_account.gitlab_service_account.email}" 161 | } -------------------------------------------------------------------------------- /vars.tf: -------------------------------------------------------------------------------- 1 | # Deployment wide variables 2 | variable "project_id" { 3 | type = string 4 | description = "(required) GCP Project ID to deploy to" 5 | } 6 | 7 | variable "dns_project_id" { 8 | type = string 9 | description = "If provided external_hostname, specify GCP Project ID where managed zone is located" 10 | default = "" 11 | } 12 | 13 | variable "infra_name" { 14 | type = string 15 | description = "(required) Infrastructure name or Team name" 16 | validation { 17 | condition = can(regex("^[a-z]([a-z0-9]*[a-z0-9])$", var.infra_name)) // Due to Certificate SAN, and service account name translation to email, and 1 another. 18 | error_message = "The infrastructure name must comply with the following regex ^[a-z]([a-z0-9]*[a-z0-9])$ )." 19 | } 20 | } 21 | 22 | # Backup variables 23 | variable "backups_bucket_name" { 24 | type = string 25 | description = "The name of the bucket backups are stored. Bucket must exist before apply. Terrafrom will add objectCreator permission to the gitlab svc account." 26 | } 27 | 28 | variable "gitlab_backup_key_secret_id" { 29 | description = "An existing secret ID in the same GCP project (project_id) storing a password for the backup process (Allowed symbols: -_ )" 30 | type = string 31 | default = "" 32 | } 33 | 34 | # Migration variables 35 | variable "migrate_gitlab" { 36 | type = bool 37 | description = "If performing migration from another Gitlab instance and got a backup file from previous instance" 38 | default = false 39 | } 40 | 41 | 42 | variable "migrate_gitlab_backup_bucket" { 43 | type = string 44 | description = "The Google Storage Bucket to your Gitlab backup e.g. 'mybucket1-abcd'" 45 | default = "" 46 | } 47 | 48 | variable "migrate_gitlab_backup_path" { 49 | type = string 50 | description = "The path to the archived backup zip 'backups/gitlab-xxx-backup.zip'" 51 | default = "" 52 | } 53 | 54 | 55 | 56 | # Gitlab instance related variables 57 | 58 | variable "gitlab_instance_protocol" { 59 | type = string 60 | description = "(optional) Protocol to use for Gitlab instance http / https" 61 | default = "https" 62 | validation { 63 | condition = can(regex("^https?$", var.gitlab_instance_protocol)) 64 | error_message = "The gitlab_instance_protocol can be either http/https." 65 | } 66 | } 67 | 68 | variable "gitlab_version" { 69 | type = string 70 | description = "Gitlab version to install (e.g. 15.2.1-ee). If performing migration, you must specify the Gitlab backup version from the previous instance" 71 | default = "16.1.2-ee" 72 | validation { 73 | condition = can(regex("^[0-9]+.[0-9]+.[0-9]+-ee$", var.gitlab_version)) 74 | error_message = "Invalid Gitlab version" 75 | } 76 | } 77 | 78 | variable "plans" { 79 | type = map 80 | default = { 81 | "2x8" = "n1-standard-2" #(56.72$, 0.097118$) 82 | } 83 | } 84 | 85 | variable "size" { 86 | type = string 87 | default = "2x8" 88 | } 89 | 90 | variable "os_images" { 91 | type = map 92 | default = { 93 | "ubuntu" = { 94 | "jammy" = "ubuntu-2204-lts" 95 | "focal" = "ubuntu-2004-lts" 96 | "bionic" = "ubuntu-1804-lts" 97 | } 98 | } 99 | } 100 | 101 | variable "os_name" { 102 | type = string 103 | description = "(optional) OS Image" 104 | default = "ubuntu" 105 | } 106 | 107 | variable "os_release" { 108 | type = string 109 | description = "(optional) OS Image" 110 | default = "focal" 111 | } 112 | 113 | variable "scallops_recipes_git_url" { 114 | type = string 115 | description = "Scallops-Recipes repository. Git URL must be provided in the following format: https:////.git" 116 | default = "https://github.com/SygniaLabs/ScallOps-Recipes.git" 117 | } 118 | 119 | variable "scallops_recipes_git_creds_secret" { 120 | type = string 121 | description = "A secret in the same project (project_id) storing Git credentials to access the provided scallops-recipes repository. Format is : or ." 122 | default = "" 123 | } 124 | 125 | # Networking and region related variables 126 | 127 | variable "operator_ips" { 128 | type = list(string) 129 | description = "(required) IP addresses used to operate and access Gitlab" 130 | } 131 | 132 | variable "operator_ports" { 133 | type = list(string) 134 | description = "(optional) Ports used to operate Gitlab" 135 | default = ["443","80"] 136 | } 137 | 138 | variable "external_integration_ranges" { 139 | type = list(string) 140 | description = "(optional) IP addresses used to for external integration with Gitlab" 141 | default = null 142 | } 143 | 144 | variable "region" { 145 | type = string 146 | description = "(optional) Region in which Gitlab and K8s will be deployed" 147 | default = "us-central1" 148 | } 149 | 150 | variable "zone" { 151 | type = string 152 | description = "(optional) Zone in which Gitlab GCE and K8s cluster will be deployed, K8s cluster will be Zonal and not Regional." 153 | default = "a" 154 | } 155 | 156 | 157 | 158 | # GKE related variables 159 | variable "gke_version" { 160 | description = "Kubernetes engine version" 161 | type = string 162 | default = "1.26.5-gke.1200" # Available version -> https://cloud.google.com/kubernetes-engine/docs/release-notes 163 | } 164 | 165 | variable "gke_linux_pool_version" { 166 | description = "GKE Linux node pool version" 167 | type = string 168 | default = "1.26.5-gke.1200" 169 | } 170 | 171 | variable "gke_windows_pool_version" { 172 | description = "GKE Windows node pool version" 173 | type = string 174 | default = "1.26.5-gke.1200" 175 | } 176 | 177 | variable "runner_chart_url" { 178 | description = "Gitlab runner Helm chart archive URL" # https://artifacthub.io/packages/helm/gitlab/gitlab-runner 179 | type = string 180 | default = "https://gitlab-charts.s3.amazonaws.com/gitlab-runner-0.54.0.tgz" # Correspond to Gitlab 16.1.0 181 | } 182 | 183 | 184 | # DNS and managed zone variables 185 | variable "external_hostname" { 186 | description = "The external hostname to be configured for the instance. e.g. scallops.example.com" 187 | type = string 188 | default = "" 189 | } 190 | 191 | variable "dns_managed_zone_name" { 192 | description = "The name of the Cloud DNS Managed Zone in which to create the DNS A Records specified in external_hostname. Only use if provided external_hostname. e.g. example-com" 193 | type = string 194 | default = "" 195 | } 196 | 197 | variable "dns_record_ttl" { 198 | description = "The time-to-live for the site A records (seconds)" 199 | type = number 200 | default = 300 201 | } 202 | 203 | variable "dockerhub-creds-secret" { 204 | description = "An existing secret name in the same GCP project storing the Dockerhub credentials (username:password)" 205 | type = string 206 | default = "" 207 | } 208 | 209 | variable "debug_flag" { 210 | type = bool 211 | description = "Enable debugging resources such as IAP Firewall rules, and export of config files" 212 | default = false 213 | } 214 | -------------------------------------------------------------------------------- /gke.tf: -------------------------------------------------------------------------------- 1 | ########################### GKE Cluster ################################################# 2 | 3 | 4 | ## K8s secrets and namespaces 5 | resource "kubernetes_secret" "google-application-credentials" { 6 | data = { 7 | "kaniko-token-secret.json" = base64decode(google_service_account_key.artifact_registry_writer.private_key) 8 | } 9 | metadata { 10 | name = "kaniko-secret" 11 | namespace = kubernetes_namespace.sensitive-namespace.id 12 | } 13 | } 14 | 15 | resource "kubernetes_secret" "k8s_gitlab_cert_secret-sensitive" { 16 | data = { 17 | "${local.instance_internal_domain}.crt" = tls_self_signed_cert.gitlab-self-signed-cert.cert_pem 18 | } 19 | metadata { 20 | name = "${local.instance_internal_domain}-cert" 21 | namespace = kubernetes_namespace.sensitive-namespace.id 22 | } 23 | } 24 | 25 | resource "kubernetes_secret" "dockerhub-creds-config" { 26 | count = var.dockerhub-creds-secret != "" ? 1 : 0 27 | data = { 28 | ".dockerconfigjson" = jsonencode({ 29 | auths = { 30 | "https://index.docker.io/v1/" = { 31 | auth = "${base64encode("${data.google_secret_manager_secret_version.dockerhub-secret[0].secret_data}")}" 32 | } 33 | } 34 | }) 35 | } 36 | type = "kubernetes.io/dockerconfigjson" 37 | metadata { 38 | name = "dockerhub-creds-jsonconfig" 39 | namespace = kubernetes_namespace.sensitive-namespace.id 40 | } 41 | } 42 | 43 | 44 | resource "kubernetes_namespace" "sensitive-namespace" { 45 | depends_on = [module.gke.google_container_node_pool] 46 | metadata { 47 | annotations = {name = "Store Kaniko and Dockerhub creds secrets and their related pod runners"} 48 | name = "sensitive" 49 | } 50 | } 51 | 52 | resource "kubernetes_secret" "k8s_gitlab_cert_secret" { 53 | depends_on = [module.gke.google_container_node_pool] 54 | data = { 55 | "${local.instance_internal_domain}.crt" = tls_self_signed_cert.gitlab-self-signed-cert.cert_pem 56 | } 57 | metadata { 58 | name = "${local.instance_internal_domain}-cert" 59 | namespace = "default" 60 | } 61 | } 62 | 63 | 64 | # Pod disruption budget 65 | 66 | resource "kubernetes_pod_disruption_budget_v1" "kube-dns" { 67 | depends_on = [module.gke.google_container_node_pool] 68 | metadata { 69 | name = "k8s-pdb-kube-dns" 70 | namespace = "kube-system" 71 | } 72 | spec { 73 | max_unavailable = 1 74 | selector { 75 | match_labels = { 76 | k8s-app = "kube-dns" 77 | } 78 | } 79 | } 80 | } 81 | 82 | 83 | resource "kubernetes_pod_disruption_budget_v1" "konnectivity-agent" { 84 | depends_on = [module.gke.google_container_node_pool] 85 | metadata { 86 | name = "k8s-pdb-konnectivity-agent" 87 | namespace = "kube-system" 88 | } 89 | spec { 90 | max_unavailable = "50%" 91 | selector { 92 | match_labels = { 93 | k8s-app = "konnectivity-agent" 94 | } 95 | } 96 | } 97 | } 98 | 99 | 100 | ## K8s cluster 101 | 102 | module "gke" { 103 | source = "terraform-google-modules/kubernetes-engine/google" 104 | version = "26.1.1" # https://github.com/terraform-google-modules/terraform-google-kubernetes-engine 105 | kubernetes_version = var.gke_version 106 | project_id = var.project_id 107 | name = "${var.infra_name}-offensive-pipeline" 108 | regional = false 109 | region = var.region #Required if Regional true 110 | zones = ["${var.region}-${var.zone}"] 111 | network = module.gcp-network.network_name 112 | subnetwork = module.gcp-network.subnets_names[0] 113 | default_max_pods_per_node = 30 114 | ip_range_pods = "${var.infra_name}-gke-pods-subnet" 115 | ip_range_services = "${var.infra_name}-gke-service-subnet" 116 | http_load_balancing = false 117 | horizontal_pod_autoscaling = true 118 | network_policy = false 119 | # remove_default_node_pool = true 120 | initial_node_count = 1 121 | create_service_account = true 122 | grant_registry_access = true 123 | 124 | node_pools = [ 125 | { 126 | name = "linux-pool" 127 | version = var.gke_linux_pool_version 128 | machine_type = "e2-highcpu-2" 129 | min_count = 1 130 | max_count = 8 131 | local_ssd_count = 0 132 | disk_size_gb = 100 133 | disk_type = "pd-ssd" 134 | image_type = "COS_CONTAINERD" 135 | enable_gcfs = true 136 | auto_repair = true 137 | auto_upgrade = true 138 | spot = true 139 | initial_node_count = 1 140 | } 141 | ] 142 | 143 | node_pools_oauth_scopes = { 144 | linux-pool = ["https://www.googleapis.com/auth/cloud-platform"] 145 | } 146 | 147 | node_pools_labels = { 148 | all = {} 149 | } 150 | node_pools_metadata = { 151 | all = { 152 | disable-legacy-endpoints = true 153 | } 154 | } 155 | node_pools_taints = { 156 | all = [] 157 | } 158 | node_pools_tags = { 159 | linux-pool = [local.gke_linux_pool_tag] 160 | } 161 | } 162 | 163 | 164 | 165 | ## K8s windows node pool 166 | 167 | resource "google_container_node_pool" "windows-pool" { 168 | cluster = module.gke.cluster_id 169 | initial_node_count = 1 170 | location = "${var.region}-${var.zone}" 171 | max_pods_per_node = 8 172 | name = "windows-pool" 173 | #node_count = 0 174 | node_locations = ["${var.region}-${var.zone}"] 175 | provider = google.offensive-pipeline 176 | version = var.gke_windows_pool_version 177 | autoscaling { 178 | max_node_count = 8 179 | min_node_count = 1 # at least 1 required since Gitlab K8s runner for windows has issue with scaling-up from 0->1 nodes as node-pool label cant contain the windows build version 180 | } 181 | 182 | management { 183 | auto_repair = true 184 | auto_upgrade = false 185 | } 186 | 187 | node_config { 188 | disk_size_gb = 200 189 | disk_type = "pd-ssd" 190 | guest_accelerator = [] 191 | image_type = "windows_ltsc_containerd" 192 | labels = { 193 | "cluster_name" = module.gke.name 194 | "node_pool" = "windows-pool" 195 | } 196 | local_ssd_count = 0 197 | machine_type = "t2d-standard-4" 198 | metadata = { 199 | "cluster_name" = module.gke.name 200 | "disable-legacy-endpoints" = "true" 201 | "node_pool" = "windows-pool" 202 | "windows" = "true" 203 | "windows-startup-script-url" = local.gke_win_pool_start_script 204 | } 205 | oauth_scopes = ["https://www.googleapis.com/auth/cloud-platform"] 206 | preemptible = true 207 | service_account = module.gke.service_account 208 | tags = [local.gke_win_pool_tag] 209 | taint = [ 210 | { 211 | effect = "PREFER_NO_SCHEDULE" 212 | key = "node.kubernetes.io/os" 213 | value = "windows" 214 | }, 215 | { 216 | effect = "NO_SCHEDULE" 217 | key = "node.kubernetes.io/os" 218 | value = "windows" 219 | } 220 | ] 221 | 222 | shielded_instance_config { 223 | enable_integrity_monitoring = true 224 | enable_secure_boot = false 225 | } 226 | 227 | workload_metadata_config { 228 | mode = "GKE_METADATA" 229 | } 230 | } 231 | 232 | timeouts { 233 | create = "45m" 234 | delete = "45m" 235 | update = "45m" 236 | } 237 | 238 | upgrade_settings { 239 | max_surge = 2 240 | max_unavailable = 0 241 | } 242 | } 243 | 244 | 245 | ### Artifact Registry Repository ### 246 | 247 | resource "google_artifact_registry_repository" "containers" { 248 | location = var.region 249 | repository_id = local.artifact_registry_id 250 | description = "Container repository created by terraform for ${var.infra_name}" 251 | format = "DOCKER" 252 | provider = google.offensive-pipeline 253 | } -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | ########################### Gitlab Instance ################################################# 2 | 3 | resource "google_compute_instance" "gitlab" { 4 | depends_on = [ 5 | # If migrating we have to wait for the backup to upload 6 | null_resource.transfer_gitlab_backup[0], 7 | # Need to wait for the secrets' values to take place in the secrets objects 8 | google_secret_manager_secret_version.gitlab-self-signed-cert-crt-version, 9 | google_secret_manager_secret_version.gitlab-self-signed-cert-key-version, 10 | google_secret_manager_secret_version.gitlab_initial_root_pwd, 11 | google_secret_manager_secret_version.gitlab_runner_registration_token, 12 | # Compute instance doesn't wait for any binding related to the service account to complete. These are required for the startup scripts 13 | google_storage_bucket_iam_binding.binding, 14 | google_project_iam_binding.compute_binding, 15 | google_storage_bucket_iam_binding.backup_bucket_binding, 16 | google_secret_manager_secret_iam_binding.gitlab-self-key-binding, 17 | google_secret_manager_secret_iam_binding.gitlab-self-crt-binding, 18 | google_secret_manager_secret_iam_binding.gitlab_runner_registration_token, 19 | google_secret_manager_secret_iam_binding.gitlab_initial_root_pwd, 20 | google_secret_manager_secret_iam_member.gitlab_backup_key, 21 | google_secret_manager_secret_iam_member.git_creds 22 | ] 23 | provider = google.offensive-pipeline 24 | name = local.gitlab_instance_name 25 | machine_type = var.plans[var.size] 26 | zone = "${var.region}-${var.zone}" 27 | tags = [local.gitlab_instance_name] 28 | 29 | 30 | 31 | service_account { 32 | email = google_service_account.gitlab_service_account.email 33 | scopes = ["cloud-platform"] 34 | } 35 | 36 | boot_disk { 37 | initialize_params { 38 | image = var.os_images[var.os_name][var.os_release] 39 | size = "160" 40 | type = "pd-ssd" 41 | } 42 | } 43 | 44 | network_interface { 45 | subnetwork = module.gcp-network.subnets_self_links[0] 46 | access_config {} 47 | } 48 | 49 | 50 | metadata = { 51 | gcs-prefix = "gs://${google_storage_bucket.deployment_utils.name}" 52 | gcs-path-to-backup = var.migrate_gitlab ? local.gitlab_migrate_backup : "NONE" # Migration var 53 | startup-script-url = local.gitlab_startup_script 54 | instance-external-domain = var.external_hostname != "" ? var.external_hostname : local.instance_internal_domain 55 | instance-protocol = var.gitlab_instance_protocol 56 | gitlab-initial-root-pwd-secret = google_secret_manager_secret.gitlab_initial_root_pwd.secret_id 57 | gitlab-cert-key-secret = google_secret_manager_secret.gitlab-self-signed-cert-key.secret_id 58 | gitlab-cert-public-secret = google_secret_manager_secret.gitlab-self-signed-cert-crt.secret_id 59 | gitlab-ci-runner-registration-token-secret = google_secret_manager_secret.gitlab_runner_registration_token.secret_id 60 | gitlab-backup-key-secret = var.gitlab_backup_key_secret_id 61 | gitlab-backup-bucket-name = var.backups_bucket_name 62 | target-gitlab-version = var.gitlab_version 63 | gitlab-package-dl-link = local.gitlab_package_dl_link 64 | container-registry-host = local.artifact_registry_host 65 | container-registry-namespace = local.artifact_registry_namespace 66 | scallops-recipes-git-url = var.scallops_recipes_git_url 67 | scallops-recipes-git-creds-secret = var.scallops_recipes_git_creds_secret != "" ? var.scallops_recipes_git_creds_secret : "NONE" 68 | } 69 | } 70 | 71 | 72 | resource "google_compute_instance_group" "gitlab_instance" { 73 | provider = google.offensive-pipeline 74 | name = "${local.gitlab_instance_name}-grp" 75 | description = "Instance group for Gitlab" 76 | zone = google_compute_instance.gitlab.zone 77 | instances = [google_compute_instance.gitlab.self_link] 78 | 79 | named_port { 80 | name = "https" 81 | port = 443 82 | } 83 | named_port { 84 | name = "http" 85 | port = 80 86 | } 87 | } 88 | 89 | 90 | 91 | ################################ Helm chart deployments ############################## 92 | 93 | 94 | resource "helm_release" "gitlab-runner-linux" { 95 | depends_on = [module.gke.google_container_node_pool] 96 | name = "linux" 97 | wait = false 98 | chart = var.runner_chart_url 99 | 100 | values = [ 101 | file("${path.module}/gitlab-runner/linux-values.yaml") 102 | ] 103 | 104 | set { 105 | name = "gitlabUrl" 106 | value = local.instance_internal_url 107 | } 108 | set { 109 | name = "certsSecretName" 110 | value = "${local.instance_internal_domain}-cert" 111 | } 112 | set_sensitive { 113 | name = "runnerRegistrationToken" 114 | value = random_password.gitlab_runner_registration_token.result 115 | } 116 | set { 117 | name = "sessionServer.enabled" 118 | value = "true" 119 | } 120 | set { 121 | name = "sessionServer.timeout" 122 | value = 1800 123 | } 124 | set { 125 | name = "sessionServer.loadBalancerSourceRanges[0]" 126 | value = "${google_compute_instance.gitlab.network_interface.0.access_config.0.nat_ip}/32" 127 | } 128 | set { 129 | name = "runners.config" 130 | value = < 21 | ScallOps logo 22 |

23 | 24 | ## Infrastructure Features 25 | * All Gitlab's CI/CD features for designning the Recipes. 26 | * Linux & Windows based operating systems for running the Recipes weaponization jobs. 27 | * On-demand automated node scalability (Google Kubernets Engine). 28 | * Access to work with private container images from your Google Artifact Registry. 29 | * Ability to supply Docker Hub credentials and work with private container images from your Docker Hub account. 30 | * Capability to automatically build Dockerfiles and push the images to your Artifact Registry. 31 | * Automated, weekly Gitlab backup to a desginated bucket. 32 | * Attach external hostname to the framework using Google DNS managed zones. 33 | * Ability to migrate from an older Gitlab instance or upgrade a deployed instance. 34 | 35 | ## Deployment 36 | 37 | The infrastructure can be deployed to GCP using the provided Terraform scripts. 38 | It is mainly built from a Gitlab instance that provides the CI features and a Kubernetes cluster that execute CI jobs on the relevant operating systems. 39 | There is also a use of Cloud Container Registry to store customized container images that we may use during operating the framework. 40 | Google Cloud Storage is in use to store maintenance scripts and encrypted Gitlab backups. 41 | 42 | Automated deployment Pre-requisites: 43 | * Google Cloud subscription with **OWNER** permissions on a project (It is reccomended to use a **clean** GCP project). 44 | * Access to GCP cloud shell or locally using Terraform with GCP credentials. 45 | * Google Storage Bucket to store the backups. This Bucket is not required to share the same GCP project with the rest of the deployment. 46 | * A secret storing a password which will be used to encrypt/decrypt backup archives. 47 | 48 | 49 | 50 | Clone the repository 51 | ```bash 52 | git clone https://github.com/SygniaLabs/ScallOps.git 53 | cd ScallOps 54 | nano config.tfvars 55 | ``` 56 | 57 | Carefully read the instructions related to each variable and act accordingly. 58 | 59 | **project_id**, **infra_name**, **gitlab_backup_key_secret_id** and **backups_bucket_name** variables are required. 60 | 61 | The code below is an example for *config.tfvars* file using all available optional features: 62 | ```bash 63 | ##### Scallops IAC variables ##### 64 | #### Note that some variables are required (#Required), and some variables modifications will take effect also after deployment (#PostDeploymentModifiable). 65 | 66 | 67 | ## GCP Project ID 68 | project_id = "" #Required 69 | 70 | ## The name you wish to have as a prefix for the deployment's resources. Must comply with ^[a-z]([a-z0-9]*[a-z0-9])$ 71 | infra_name = "scallops" #Required 72 | 73 | ## The name of an existing bucket you wish to receive backups to. Terraform will create the required permission to upload the backup archive. 74 | backups_bucket_name = "" #Required #PostDeploymentModifiable 75 | 76 | ## An existing secret ID in the same GCP project (project_id) storing a password for the backup process (Allowed symbols for secret value: -_ ) 77 | ## Creating a secret through GCP secret manager https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create 78 | gitlab_backup_key_secret_id = "" #Required #PostDeploymentModifiable 79 | 80 | 81 | 82 | ## Gitlab version to install 83 | ## Ruuner chart version must be compaitibe with the Gitlab version -> https://docs.gitlab.com/runner/#gitlab-runner-versions 84 | ## Note the Gitlab application version from the selected Chart version -> https://artifacthub.io/packages/helm/gitlab/gitlab-runner 85 | ## You can make upgrades to your Gitlab instance from here. Just reset the instance once the `apply` completes. 86 | # gitlab_version = "16.1.2-ee" 87 | # runner_chart_url = "https://gitlab-charts.s3.amazonaws.com/gitlab-runner-0.54.0.tgz" 88 | 89 | 90 | ## IP addresses that can interact with the Gitlab instance via HTTP/S (Office IP / Home IPs) 91 | # operator_ips = [] #Optional #PostDeploymentModifiable 92 | 93 | ## Enable debugging resources such as IAP Firewall rules, and export of config files 94 | # debug_flag = false #Optional #PostDeploymentModifiable 95 | 96 | ## The Gitlab instance Web server protocol, http or https. 97 | # gitlab_instance_protocol = "https" #Optional 98 | 99 | ## Region for the k8s cluster, Gitlab instance and network. 100 | # region = us-central-1 #Optional 101 | 102 | ## Zone for the k8s cluster, Gitlab instance and network. 103 | # zone = "a" #Optional 104 | 105 | 106 | 107 | ## External DNS ## #Optional #PostDeploymentModifiable #GitlabRestartRequired 108 | ## Uncomment the 3 lines below if wishing to supply external DNS name for accessing Gitalb instance 109 | 110 | # dns_project_id = "" # The project ID where the managed DNS zone is located 111 | # dns_managed_zone_name = "mydomain-com" # The configured managed DNS zone name 112 | # external_hostname = "scallops.mydomain.com" # The hostname you wish to set of the instance (Must be subdomain of the managed zone) 113 | 114 | 115 | 116 | ## Docker hub credentials (https://docs.docker.com/docker-hub/access-tokens/) #Optional #PostDeploymentModifiable 117 | ## An existing secret name in secret-manager storing Dockerhub credentials to fetch private container images (format is username:password or username:access-token). 118 | # dockerhub-creds-secret = "" 119 | 120 | 121 | ## Scallops-Recipes repository. Use the default repository or specify alternative fork in a Git path HTTPS format. 122 | ## The specified repository will be imported to Gitlab as the Scallops-Recipes repository. 123 | ## *Ignored if performing a migration 124 | # scallops_recipes_git_url = "https://github.com/SygniaLabs/ScallOps-Recipes.git" #Optional 125 | # scallops_recipes_git_creds_secret = "my-github-creds-secret" #Optional 126 | 127 | 128 | 129 | ## Migration variables #Optional 130 | ## If you plan on migrating from a different gitlab instance, uncomment all migration variables below, and follow requirements. 131 | ## 1. 'gitlab_backup_key_secret_id' secret must store the password value decrypting the archived backup zip. 132 | ## 2. 'gitlab_version' must be equal to the version you are migrating from. 133 | ## 3. Operation requires Gsutil on the terraform deployer system as backup will be downloaded locally 134 | 135 | # migrate_gitlab = true ## If performing migration from another Gitlab instance and got a backup file from previous instance. true/false. 136 | # migrate_gitlab_backup_bucket = "" ## The Google Storage Bucket to your Gitlab backup e.g. 'mybucket1-abcd' 137 | # migrate_gitlab_backup_path = "" ## The path to the archived backup zip e.g 'backups/gitlab-xxx-backup.zip' 138 | 139 | ##### 140 | ``` 141 | 142 | 143 | If using locally, make sure that you are authenticated to your GCP 144 | ```bash 145 | gcloud auth list # List and check currently authenticated account 146 | gcloud auth login # Authneticate 147 | ``` 148 | 149 | 150 | Deploy using terraform while pointing to the created configuration file 151 | ```bash 152 | terraform init 153 | terraform apply --var-file=./config.tfvars 154 | ``` 155 | 156 | Once deployed, you should receive the Gitlab instance IP address and the secret name where the password of the Gitlab's root account is stored. 157 | Before accessing the Gitlab instance, you should look for 'Gitlab instance setup completed' in the Gcloud logging 'gitlab-startup' log name. 158 | Errors within Gitlab setup will also be logged, but not all of them will stop the setup. Checkout the entry above the error to understand which process errored. 159 | 160 | ```console 161 | .... 162 | .... 163 | module.gke.google_container_node_pool.pools["linux-pool"]: Still creating... [7m31s elapsed] 164 | module.gke.google_container_node_pool.pools["linux-pool"]: Creation complete after 7m32s [id=projects/my-scallops/locations/us-central1-a/clusters/scallops-offensive-pipeline/nodePools/linux-pool] 165 | module.gke_auth.data.google_container_cluster.gke_cluster: Reading... 166 | google_container_node_pool.windows-pool: Creating... 167 | module.gke_auth.data.google_container_cluster.gke_cluster: Read complete after 0s [id=projects/my-scallops/locations/us-central1-a/clusters/scallops-offensive-pipeline] 168 | kubernetes_namespace.sensitive-namespace: Creating... 169 | kubernetes_pod_disruption_budget.kube-dns: Creating... 170 | kubernetes_pod_disruption_budget.konnectivity-agent: Creating... 171 | kubernetes_secret.k8s_gitlab_cert_secret: Creating... 172 | helm_release.gitlab-runner-linux: Creating... 173 | kubernetes_pod_disruption_budget.konnectivity-agent: Creation complete after 1s [id=kube-system/k8s-pdb-konnectivity-agent] 174 | kubernetes_secret.k8s_gitlab_cert_secret: Creation complete after 1s [id=default/scallops-gitlab.us-central1-a.c.my-scallops.internal-cert] 175 | kubernetes_pod_disruption_budget.kube-dns: Creation complete after 1s [id=kube-system/k8s-pdb-kube-dns] 176 | kubernetes_namespace.sensitive-namespace: Creation complete after 1s [id=sensitive] 177 | kubernetes_secret.google-application-credentials: Creating... 178 | kubernetes_secret.dockerhub-creds-config[0]: Creating... 179 | kubernetes_secret.k8s_gitlab_cert_secret-sensitive: Creating... 180 | helm_release.gitlab-runner-win: Creating... 181 | kubernetes_secret.dockerhub-creds-config[0]: Creation complete after 0s [id=sensitive/dockerhub-creds-jsonconfig] 182 | kubernetes_secret.google-application-credentials: Creation complete after 0s [id=sensitive/kaniko-secret] 183 | kubernetes_secret.k8s_gitlab_cert_secret-sensitive: Creation complete after 0s [id=sensitive/scallops-gitlab.us-central1-a.c.my-scallops.internal-cert] 184 | helm_release.gitlab-runner-kaniko: Creating... 185 | helm_release.gitlab-runner-dockerhub[0]: Creating... 186 | google_container_node_pool.windows-pool: Still creating... [10s elapsed] 187 | helm_release.gitlab-runner-linux: Still creating... [10s elapsed] 188 | helm_release.gitlab-runner-win: Still creating... [10s elapsed] 189 | helm_release.gitlab-runner-kaniko: Still creating... [10s elapsed] 190 | helm_release.gitlab-runner-linux: Creation complete after 13s [id=linux] 191 | helm_release.gitlab-runner-win: Creation complete after 13s [id=windows] 192 | helm_release.gitlab-runner-dockerhub[0]: Still creating... [10s elapsed] 193 | helm_release.gitlab-runner-dockerhub[0]: Creation complete after 11s [id=dockerhub-privates] 194 | helm_release.gitlab-runner-kaniko: Creation complete after 12s [id=kaniko] 195 | google_container_node_pool.windows-pool: Still creating... [20s elapsed] 196 | google_container_node_pool.windows-pool: Still creating... [10m50s elapsed] 197 | google_container_node_pool.windows-pool: Creation complete after 10m55s [id=projects/my-scallops/locations/us-central1-a/clusters/scallops-offensive-pipeline/nodePools/windows-pool] 198 | 199 | Apply complete! Resources: 65 added, 0 changed, 0 destroyed. 200 | 201 | Outputs: 202 | 203 | gitlab_ext_ip = "X.X.X.X" 204 | gitlab_root_password_secret = "scallops-gitlab-root-password" 205 | ``` 206 | 207 | [ScallOps-Recipes](https://github.com/SygniaLabs/ScallOps-Recipes) repository should be pre-imported into the Gitlab insatnce together with few instance level CI/CD variables configured. All you have left to do is executing the recipes :) 208 | If you don't see the Recipes repository, you can import it [manually](https://docs.gitlab.com/ee/user/project/import/repo_by_url.html). 209 | > Important! If it is your initial deployment, you must run the deployment's initialization pipeline. 210 | 1. Refer to the imported Recipes repository at /ci/scallops-recipes in your Gitlab instance. 211 | 2. Navigate to CI/CD -> Pipelines -> Run Pipeline. 212 | 3. Delete all variables and add 'DEPLOYMENT_INIT' with value 1. 213 | 4. Hit 'Run Pipeline' and wait for the initialization process to complete. 214 | 215 | ## Architecture 216 | 217 | The included Terraform scripts will deploy the following resources into your GCP project: 218 | - Compute Engine - Gitlab Instance, managing sources and CI/CD jobs 219 | - Google Kubernetes Engine (GKE) - K8s cluster to host our CI/CD jobs 220 | - Default node pool - Single machine with K8s system pods and runners pods 221 | - Linux node pool - For the Gitlab-runners pods, and linux related jobs 222 | - Windows node pool - For Windows related jobs 223 | - Gitlab-runner Helm packages (Linux-runner, Windows-runner, Kaniko-runner, \[dockerhub-privates-runner\]) 224 | - Google Cloud Storage (GCS) - To store maintenance scripts 225 | - Google Artifact Registry - To store cutomized container images 226 | - Service Accounts 227 | 1. Used by Gitlab Compute instance to pull maintenance scripts and store backup 228 | 2. User who is attached to the GKE cluster nodes to pull container images from the project's registry (Read only) 229 | 3. User who is attached to the Kaniko CI job runner allowing to push customized container images to Artifact Registry 230 | - VPC network and related firewall rules to allow operation 231 | 232 |

233 | Infrastructure-layout 234 |

235 | 236 | 237 | 238 | ## GCP Cloud Costs 239 | 240 | Idle / Minimal usage: 241 | * Gitlab Instance: 51.46$ / month (n1-standard-2) 242 | * 2 x Linux node: 2 x 10.83$ = 21.66$ / month (e2-highcpu-2 preemptible) 243 | * Windows node: 134.34$ / month (t2d-standard-4 preemptible) 244 | * Storage 245 | * Utilities & Migration bucket: 20GB (depends on backup size) - 0.40$ / month 246 | * Artifact Registry (depends on container images volume): 100GB - 9.95$ / month 247 | * Boot Disks 248 | * Gitlab: Zonal SSD PD: 160 GB - 27.20$ / month 249 | * GKE Linux: Zonal SSD PD: 100 GB - 17$ / month 250 | * GKE Windows: Zonal SSD PD: 200 GB - 34$ / month 251 | * Secret manager: 0.06$ /month 252 | * GKE: One Zonal cluster is free per billing account 253 | * **Total: 318$ (us-central-1)** -- [source](https://cloud.google.com/products/calculator/#id=a582ea75-5872-4616-8313-41192c043ae7) 254 | 255 | Per Job: 256 | * Linux: Same as idle since system already up. When scaled 0.015$ per hour for each running node. 257 | * Windows: Same as idle since one system is already up. When scaled additional 0.2$ per hour for each running node. 258 | 259 | 260 | ## Backup procedure 261 | 262 | The deployment requires to supply a GCS bucket name to store the [Gitlab backup](https://docs.gitlab.com/ee/raketasks/backup_restore.html#back-up-gitlab). 263 | Backup occurs every Saturday during 10AM UTC, and is done by the Gitlab compute instance using the *[gitlab_backup.sh](scripts/bash/gitlab_backup.sh)* bash script, which is stored on GCS and setup into crontab during deployment. 264 | The backup archive is encrypted with a password stored in GCP secret-manager which is supplied (gitlab_backup_key_secret_id) by the user during Terraform deployment. 265 | 266 | The encrypted archive contains the following components: 267 | * Timestamp_yyyy_mm_dd_14.5.2-ee_gitlab_backup.tar (A file containing the Gitlab DB) 268 | * gitlab.rb (The main gitlab configuration file) 269 | * gitlab-secret.json (A file containing keys to decrypt various DB data) 270 | * ssl/hostname.key (SSL certificate private key belonging to the existing hostname) 271 | * ssl/hostname.cer (SSL public certificate file belonging to the existing hostname) 272 | 273 | The backup process log is casted to a [GCP log](https://cloud.google.com/logging/docs/view/logs-explorer-interface) named "gitlab-backup-exec". 274 | 275 | *Note that this backup structure works with the deployment migration capability. 276 | 277 | 278 | ## Open issues 279 | - Windows related containers are **not** built and deployed automatically to your Container Registry. For now, you will have to do it manually. You can use the supplied [Windows Dockerfiles](https://github.com/SygniaLabs/ScallOps-Recipes/tree/main/_ci-maintain/dockerfiles/windows). 280 | - Windows related containers must include native Powershell Core (PWSH). 281 | - Concurrent Linux related jobs are limited to 50 due to the Gitlab's instance compute resources. 282 | - Concurrent Windows related jobs are limited to 30 due to the time it takes to pull a container image when vertical scaling is triggered. 283 | 284 | 285 | ## Security Consideration 286 | Altough we made efforts to secure the given permissions and layout of the deployed environment, there may always be a scenario in which someone will achieve some sort of unautohrized access to the internal GCP cloud components. Therefore, it is **highly recommended to use a clean GCP project** when deploying this environment. 287 | 288 | ## References 289 | 290 | CI/CD guides: 291 | 292 | * CI/CD concept: https://hackernoon.com/understanding-the-basic-concepts-of-cicd-fw4k32s1 293 | * Gitlab CI docs: https://docs.gitlab.com/ee/ci/ 294 | * Gitlab CI Runner K8s executor: https://docs.gitlab.com/runner/executors/kubernetes.html 295 | 296 | Infrastructure references: 297 | 298 | * Terraform & Gcloud: https://registry.terraform.io/providers/hashicorp/google/latest/docs 299 | * GKE: https://cloud.google.com/kubernetes-engine/docs/concepts/kubernetes-engine-overview 300 | * Container registry access: https://cloud.google.com/container-registry/docs/access-control 301 | * Helm Charts: https://helm.sh/docs/topics/charts/ 302 | * Kaniko: https://github.com/GoogleContainerTools/kaniko -------------------------------------------------------------------------------- /scripts/bash/gitlab_helpers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | readonly RAILS_CMD_PATH=$(pwd)/railscmd.rb 4 | 5 | check_installation() { 6 | local logName=$1 7 | 8 | # In case GitLab instance gets restarted, skip the script. 9 | logger $logName "INFO" "Checking whether GitLab is installed" 10 | GITLAB_VERSION_FILE=/opt/gitlab/version-manifest.txt 11 | GITLAB_INSTALLED="false" 12 | 13 | if [ -f "$GITLAB_VERSION_FILE" ]; then 14 | logger $logName "DEBUG" "$GITLAB_VERSION_FILE exists" 15 | GITLAB_INSTALLED="true" 16 | GITLAB_EE_VERSION=$(grep gitlab-ee $GITLAB_VERSION_FILE | cut -d " " -f2)-ee 17 | logger $logName "INFO" "Installed GitLab version is $GITLAB_EE_VERSION" 18 | else 19 | logger $logName "DEBUG" "$GITLAB_VERSION_FILE does not exist" 20 | fi 21 | } 22 | 23 | check_upgrade() { 24 | local logName=$1 25 | local gitlabTargetVersion=$2 26 | 27 | # Check if we wish to upgrade the Gitlab application 28 | # Current GITLAB_EE_VERSION is required in the env. 29 | logger $logName "INFO" "Checking whether we want to upgrade Gitlab" 30 | if [ "$GITLAB_EE_VERSION" != "$gitlabTargetVersion" ]; then 31 | logger $logName "INFO" "GitLab version $GITLAB_EE_VERSION does not match the target version $gitlabTargetVersion. Upgrading..." 32 | # Skip the auto backup when making an upgrade 33 | logger $logName "INFO" "Skipping auto backup" 34 | exec_wrapper $ERR_ACTION_CONT $logName "touch /etc/gitlab/skip-auto-backup" 35 | 36 | logger $logName "INFO" "Updating package soruces..." 37 | exec_wrapper $ERR_ACTION_EXIT $logName "apt update" 38 | exec_wrapper $ERR_ACTION_CONT $logName "apt-cache madison gitlab-ee" 39 | 40 | logger $logName "INFO" "Installing Gitlab at version $gitlabTargetVersion" 41 | exec_wrapper $ERR_ACTION_EXIT $logName "apt install gitlab-ee=$gitlabTargetVersion.0" 42 | 43 | logger $logName "INFO" "Running checks..." 44 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl status" 45 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rake gitlab:check SANITIZE=true" 46 | 47 | logger $logName "INFO" "GitLab version upgraded successfully" 48 | else 49 | logger $logName "INFO" "GitLab is aligned with the targeted version" 50 | fi 51 | } 52 | 53 | 54 | gitlab_deps_install () { 55 | local logName=$1 56 | # Install Dependencies 57 | logger $logName "INFO" "Running package updater" 58 | exec_wrapper $ERR_ACTION_EXIT $logName "apt-get update" 59 | 60 | logger $logName "INFO" "Installing dependencies" 61 | exec_wrapper $ERR_ACTION_EXIT $logName "apt-get install -y curl ca-certificates tzdata perl jq coreutils zip p7zip-full" 62 | } 63 | 64 | 65 | set_gitlab_vars () { 66 | local logName=$1 67 | logger $logName "INFO" "Fetching Gitlab network variables" 68 | ## Network variables 69 | INSTANCE_PROTOCOL=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/instance-protocol` #http/https 70 | INSTANCE_EXTERNAL_DOMAIN=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/instance-external-domain` 71 | EXTERNAL_IP=`curl -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip` 72 | CONTAINER_REGISTRY_HOST=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/container-registry-host` 73 | CONTAINER_REGISTRY_NAMESPACE=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/container-registry-namespace` 74 | 75 | #Ext URL 76 | EXTERNAL_URL="$INSTANCE_PROTOCOL://$INSTANCE_EXTERNAL_DOMAIN" 77 | logger $logName "DEBUG" "Gitlab External URL will be $EXTERNAL_URL" 78 | 79 | #Container registry prefix 80 | logger $logName "DEBUG" "Container registry prefix: $CONTAINER_REGISTRY_HOST/$CONTAINER_REGISTRY_NAMESPACE" 81 | 82 | } 83 | 84 | 85 | reconfigure_gitlab () { 86 | # Sets Gitlab's certificate and External URL from configured metadata and secrets 87 | local logName=$1 88 | if [[ $INSTANCE_PROTOCOL == "https" ]] 89 | then 90 | logger $logName "INFO" "Gitlab set to HTTPS, setting self-signed certificate on server" 91 | mkdir -p /etc/gitlab/ssl 92 | chmod 755 /etc/gitlab/ssl 93 | GITLAB_CERT_KEY_SECRET=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-cert-key-secret` 94 | gcloud secrets versions access latest --secret=$GITLAB_CERT_KEY_SECRET > /etc/gitlab/ssl/$INSTANCE_EXTERNAL_DOMAIN.key 95 | GITLAB_CERT_PUB_SECRET=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-cert-public-secret` 96 | gcloud secrets versions access latest --secret=$GITLAB_CERT_PUB_SECRET > /etc/gitlab/ssl/$INSTANCE_EXTERNAL_DOMAIN.crt 97 | fi 98 | 99 | # Use the provided external url 100 | 101 | logger $logName "INFO" "Setting Gitlab external url to $EXTERNAL_URL" 102 | echo "external_url \"$EXTERNAL_URL\"" >> /etc/gitlab/gitlab.rb 103 | 104 | # Reconfigure installation 105 | logger $logName "INFO" "Reconfiguring Gitlab" 106 | 107 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl reconfigure" 108 | 109 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl restart" 110 | 111 | logger $logName "INFO" "Updating EXTERNAL URL to CI_EXTERNAL_URL CI/CD variable" 112 | echo "Ci::InstanceVariable.where(key: 'CI_EXTERNAL_URL').update(value: '$EXTERNAL_URL')" > $RAILS_CMD_PATH 113 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 114 | } 115 | 116 | 117 | gitlab_install () { 118 | local logName=$1 119 | local gitlabPkg=$2 120 | local pkgOutputName="gitlab-ee_amd64.deb" 121 | # Install Postfix non-interactive 122 | logger $logName "INFO" "Starting postfix installation for domain: $INSTANCE_EXTERNAL_DOMAIN" 123 | debconf-set-selections <<< "postfix postfix/mailname string $INSTANCE_EXTERNAL_DOMAIN" 124 | debconf-set-selections <<< "postfix postfix/main_mailer_type string 'Internet Site'" 125 | exec_wrapper $ERR_ACTION_CONT $logName "apt-get install --assume-yes postfix" 126 | 127 | # Install Gitlab Server 128 | logger $logName "INFO" "Setting up Gitlab installation" 129 | curl https://packages.gitlab.com/install/repositories/gitlab/gitlab-ee/script.deb.sh | bash 130 | #Specific version installation: 131 | logger $logName "INFO" "Downloading Gitlab from $gitlabPkg" 132 | exec_wrapper $ERR_ACTION_EXIT $logName "wget --content-disposition -O $pkgOutputName $gitlabPkg" 133 | logger $logName "INFO" "Installing Gitlab" 134 | exec_wrapper $ERR_ACTION_EXIT $logName "dpkg -i $pkgOutputName" 135 | 136 | logger $logName "INFO" "Reconfiguring Gitlab" 137 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl reconfigure" 138 | } 139 | 140 | 141 | setup_cicd_vars () { 142 | local logName=$1 143 | # Set instance level environment variables, so pipelines can utilize them 144 | logger $logName "INFO" "Fetching values for instance level CI/CD variables" 145 | GCP_PROJECT_ID=`gcloud config list --format 'value(core.project)' 2>/dev/null` 146 | INSTANCE_INTERNAL_HOSTNAME=`curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/hostname"` 147 | INSTANCE_INTERNAL_URL=$INSTANCE_PROTOCOL://$INSTANCE_INTERNAL_HOSTNAME 148 | INSTANCE_INTERNAL_API_V4_URL=$INSTANCE_PROTOCOL://$INSTANCE_INTERNAL_HOSTNAME/api/v4 149 | 150 | logger $logName "DEBUG" " 151 | CI_EXTERNAL_URL=$EXTERNAL_URL 152 | CI_SERVER_HOST=$INSTANCE_INTERNAL_HOSTNAME 153 | CI_SERVER_URL=$INSTANCE_INTERNAL_URL 154 | CI_API_V4_URL=$INSTANCE_INTERNAL_API_V4_URL 155 | CONTAINER_REGISTRY_NAMESPACE=$CONTAINER_REGISTRY_NAMESPACE 156 | CONTAINER_REGISTRY_HOST=$CONTAINER_REGISTRY_HOST 157 | " 158 | 159 | } 160 | 161 | create_cicd_vars () { 162 | local logName=$1 163 | # New instance level variables 164 | logger $logName "INFO" "Creating instance level CI/CD variables" 165 | 166 | echo "Ci::InstanceVariable.new(key: 'CI_EXTERNAL_URL', value: '$EXTERNAL_URL').save" > $RAILS_CMD_PATH 167 | echo "Ci::InstanceVariable.new(key: 'CI_SERVER_HOST', value: '$INSTANCE_INTERNAL_HOSTNAME').save" >> $RAILS_CMD_PATH 168 | echo "Ci::InstanceVariable.new(key: 'CI_SERVER_URL', value: '$INSTANCE_INTERNAL_URL').save" >> $RAILS_CMD_PATH 169 | echo "Ci::InstanceVariable.new(key: 'CI_API_V4_URL', value: '$INSTANCE_INTERNAL_API_V4_URL').save" >> $RAILS_CMD_PATH 170 | echo "Ci::InstanceVariable.new(key: 'CONTAINER_REGISTRY_NAMESPACE', value: '$CONTAINER_REGISTRY_NAMESPACE').save" >> $RAILS_CMD_PATH 171 | echo "Ci::InstanceVariable.new(key: 'CONTAINER_REGISTRY_HOST', value: '$CONTAINER_REGISTRY_HOST').save" >> $RAILS_CMD_PATH 172 | 173 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 174 | 175 | } 176 | 177 | update_cicd_vars () { 178 | local logName=$1 179 | # Update instance level variables values accroding to the new GCP project and compute hostname 180 | logger $logName "INFO" "Updating instance level CI/CD variables" 181 | 182 | echo "Ci::InstanceVariable.where(key: 'CI_SERVER_HOST').update(value: '$INSTANCE_INTERNAL_HOSTNAME')" > $RAILS_CMD_PATH 183 | echo "Ci::InstanceVariable.where(key: 'CI_SERVER_URL').update(value: '$INSTANCE_INTERNAL_URL')" >> $RAILS_CMD_PATH 184 | echo "Ci::InstanceVariable.where(key: 'CI_API_V4_URL').update(value: '$INSTANCE_INTERNAL_API_V4_URL')" >> $RAILS_CMD_PATH 185 | echo "Ci::InstanceVariable.where(key: 'CONTAINER_REGISTRY_NAMESPACE').update(value: '$CONTAINER_REGISTRY_NAMESPACE').save" >> $RAILS_CMD_PATH 186 | echo "Ci::InstanceVariable.where(key: 'CONTAINER_REGISTRY_HOST').update(value: '$CONTAINER_REGISTRY_HOST').save" >> $RAILS_CMD_PATH 187 | 188 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 189 | } 190 | 191 | 192 | seed_instance_reg_token () { 193 | local logName=$1 194 | # Seed shared runners registartion token 195 | GITLAB_RUNNER_REG_SECRET=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-ci-runner-registration-token-secret` 196 | logger $logName "INFO" "Reading shared runners registration token from secret $GITLAB_RUNNER_REG_SECRET" 197 | GITLAB_RUNNER_REG=`gcloud secrets versions access latest --secret=$GITLAB_RUNNER_REG_SECRET` 198 | logger $logName "INFO" "Seeding shared runners registration token" 199 | 200 | echo "appset = Gitlab::CurrentSettings.current_application_settings; appset.set_runners_registration_token('$GITLAB_RUNNER_REG'); appset.save!" > $RAILS_CMD_PATH 201 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 202 | } 203 | 204 | 205 | seed_gitlab_root_pwd () { 206 | local logName=$1 207 | # Seed gitlab root password 208 | GITLAB_INITIAL_ROOT_PASSWORD_SECRET=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-initial-root-pwd-secret` 209 | logger $logName "INFO" "Reading gitlab root password from secret $GITLAB_INITIAL_ROOT_PASSWORD_SECRET" 210 | GITLAB_INITIAL_ROOT_PASSWORD=`gcloud secrets versions access latest --secret=$GITLAB_INITIAL_ROOT_PASSWORD_SECRET` 211 | 212 | logger $logName "INFO" "Seeding gitlab root password" 213 | echo "user = User.find_by_username('root'); user.password = '$GITLAB_INITIAL_ROOT_PASSWORD'; user.password_confirmation = '$GITLAB_INITIAL_ROOT_PASSWORD'; user.save!" > $RAILS_CMD_PATH 214 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 215 | } 216 | 217 | 218 | 219 | create_groups () { 220 | local logName=$1 221 | # Create repo groups (ci, community, private) 222 | logger $logName "INFO" "Creating groups: ci, community, private" 223 | 224 | echo "Groups::CreateService.new(User.find_by_id(1), params = {name: 'CI CD Tools', path: 'ci', visibility_level: 10}).execute" > $RAILS_CMD_PATH 225 | echo "Groups::CreateService.new(User.find_by_id(1), params = {name: 'Community Tools', path: 'community', visibility_level: 10}).execute" >> $RAILS_CMD_PATH 226 | echo "Groups::CreateService.new(User.find_by_id(1), params = {name: 'Private Tools', path: 'private', visibility_level: 10}).execute" >> $RAILS_CMD_PATH 227 | 228 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 229 | } 230 | 231 | 232 | import_scallops_recipes () { 233 | local logName=$1 234 | # Import SCALLOPS-RECIPES project repo 235 | readonly SCALLOPS_RECIPES_GIT_URL=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/scallops-recipes-git-url` 236 | local gitCredsSecretName=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/scallops-recipes-git-creds-secret` 237 | logger $logName "INFO" "Importing SCALLOPS-RECIPES repo from $SCALLOPS_RECIPES_GIT_URL to ci group" 238 | 239 | if [ $gitCredsSecretName != 'NONE' ]; then 240 | logger $logName "INFO" "Reading Git credentials from secret $gitCredsSecretName to import $SCALLOPS_RECIPES_GIT_URL repository" 241 | local gitCreds=`gcloud secrets versions access latest --secret=$gitCredsSecretName` 242 | SCALLOPS_RECIPES_GIT_URL="https://$gitCreds@${SCALLOPS_RECIPES_GIT_URL:8}" 243 | fi 244 | 245 | 246 | logger $logName "DEBUG" "Ignore the following error: (undefined method repository for :octokit:Symbol)" 247 | echo "cigrp = Group.find_by_path_or_name('ci'); rootuser = User.find_by_id(1); Project.new(import_url: '$SCALLOPS_RECIPES_GIT_URL', name: 'Scallops Recipes', path: 'scallops-recipes', visibility_level: 10, creator: rootuser, namespace: cigrp).save" > $RAILS_CMD_PATH 248 | echo "newprj = Project.find_by_full_path('ci/scallops-recipes'); Gitlab::GithubImport::Importer::RepositoryImporter.new(newprj, :octokit).execute" >> $RAILS_CMD_PATH 249 | 250 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" # Ignore the following error (undefined method `repository' for :octokit:Symbol) 251 | } 252 | 253 | 254 | seed_scallops_recipes_runner_token () { 255 | local logName=$1 256 | # Seed scallops-recipes sepcific runners registration token 257 | # Make sure to invoke seed_instance_reg_token function before this one, so the $GITLAB_RUNNER_REG variable will be set 258 | logger $logName "INFO" "Seeding Scallops-Recipes runners registration token" 259 | local scallopsRunnerReg=GR1348941$GITLAB_RUNNER_REG-scallops-recipes 260 | 261 | echo "scallopsprj = Project.find_by_full_path('ci/scallops-recipes'); scallopsprj.set_runners_token('$scallopsRunnerReg'); scallopsprj.save!" > $RAILS_CMD_PATH 262 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rails runner $RAILS_CMD_PATH" 263 | } 264 | 265 | 266 | setup_gitlab_backup () { 267 | local logName=$1 268 | local gcsPrefix=$2 269 | # Download backup cron executor and cron job #Backup will occur every Saturday on 10:00 UTC 270 | logger $logName "INFO" "Setting up backup procedure with crontab" 271 | exec_wrapper $ERR_ACTION_CONT $logName "gsutil cp $gcsPrefix/scripts/bash/gitlab_backup.sh /gitlab_backup.sh" 272 | chmod +x /gitlab_backup.sh 273 | echo "0 10 * * 6 /gitlab_backup.sh" > gitlab-backup-cron 274 | logger $logName "DEBUG" "Crontab content $(cat gitlab-backup-cron)" 275 | exec_wrapper $ERR_ACTION_CONT $logName "crontab gitlab-backup-cron" 276 | rm gitlab-backup-cron 277 | } 278 | 279 | 280 | 281 | get_backup_archive () { 282 | local logName=$1 283 | local gcsPathToBackup=$2 284 | local backupArchivePath=$3 285 | 286 | # Download backup 287 | logger $logName "INFO" "Downloading backup from $gcsPathToBackup" 288 | exec_wrapper $ERR_ACTION_CONT $logName "gsutil cp $gcsPathToBackup $backupArchivePath" 289 | 290 | } 291 | 292 | 293 | get_backup_archive_password () { 294 | local logName=$1 295 | 296 | # Get the backup archive password 297 | logger $logName "INFO" "Getting backup password secret from instance metadata" 298 | GITLAB_BACKUP_PASSWORD_SECRET=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-backup-key-secret` 299 | logger $logName "INFO" "Reading password for the backup archive from secret - $GITLAB_BACKUP_PASSWORD_SECRET" 300 | GITLAB_BACKUP_PASSWORD=`gcloud secrets versions access latest --secret=$GITLAB_BACKUP_PASSWORD_SECRET` 301 | logger $logName "DEBUG" "Fetched password with length of ${#GITLAB_BACKUP_PASSWORD} characters" 302 | } 303 | 304 | 305 | restore_backup () { 306 | local logName=$1 307 | local backupArchivePath=$2 308 | local backupDir="backup-extracted" 309 | 310 | # Migrate configuration files and SSL certificates 311 | 312 | mkdir -p $backupDir 313 | 314 | logger $logName "INFO" "Extracting backup from $backupArchivePath to $backupDir" 315 | exec_wrapper_no_print $ERR_ACTION_EXIT $logName "7z x -p$GITLAB_BACKUP_PASSWORD $backupArchivePath -o./$backupDir" 316 | 317 | # Stop Gitlab services 318 | logger $logName "INFO" "Stopping Gitlab services: puma, sidekiq" 319 | gitlab-ctl stop puma 320 | gitlab-ctl stop sidekiq 321 | gitlab-ctl status 322 | 323 | logger $logName "INFO" "Copying configuration files and certificates from $backupDir" 324 | cp $backupDir/gitlab* /etc/gitlab 325 | cp -R $backupDir/ssl /etc/gitlab 326 | 327 | # Restore from backup 328 | local backupFileName=$(ls $backupDir | grep _gitlab_backup.tar) 329 | logger $logName "INFO" "Copying extracted backup archive" 330 | cp $backupDir/$backupFileName /var/opt/gitlab/backups/ 331 | chown git:git /var/opt/gitlab/backups/$backupFileName 332 | 333 | logger $logName "INFO" "Restoring from backup snapshot... $backupFileName" 334 | local restoreBackupName=$(echo $backupFileName | cut -d "_" -f 1-5) 335 | exec_wrapper_no_print $ERR_ACTION_EXIT $logName "gitlab-rake gitlab:backup:restore BACKUP=$restoreBackupName force=yes" 336 | 337 | logger $logName "INFO" "Reconfiguring Gitlab" 338 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl reconfigure" 339 | 340 | logger $logName "INFO" "Restarting Gitlab services" 341 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl restart" 342 | 343 | logger $logName "INFO" "Checking Gitlab services health" 344 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rake gitlab:check SANITIZE=true" 345 | 346 | logger $logName "INFO" "Checking secrets decryptability" 347 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-rake gitlab:doctor:secrets" 348 | } 349 | 350 | 351 | 352 | execute_backup () { 353 | ### Scallops customized Gitlab backup script ### 354 | ## Run as root 355 | 356 | local logName=$1 357 | local gitlabVersion=$2 358 | local instanceName=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/name` 359 | local timestamp=`date +"%s"` 360 | local backupDir="backup-$timestamp" 361 | local backupArchiveFile="$instanceName-$gitlabVersion-$backupDir.zip" 362 | 363 | 364 | logger $logName "INFO" "Starting Gitlab backup for $instanceName" 365 | 366 | # Create backup folder 367 | mkdir -p $backupDir 368 | 369 | # Stop Gitlab services 370 | logger $logName "INFO" "Stopping Gitlab services (unicorn, sidekiq, puma)" 371 | gitlab-ctl stop unicorn 372 | gitlab-ctl stop sidekiq 373 | gitlab-ctl stop puma 374 | 375 | # Create back up TAR 376 | logger $logName "INFO" "Creaing backup tar file" 377 | exec_wrapper $ERR_ACTION_EXIT $logName "gitlab-backup create" 378 | 379 | # Restart Gitlab services back 380 | logger $logName "INFO" "Restarting gitlab services" 381 | exec_wrapper $ERR_ACTION_CONT $logName "gitlab-ctl restart" 382 | 383 | # Copy DB backup and configurations 384 | local mostRecentBackupName=`ls -t /var/opt/gitlab/backups/ | head -1` 385 | logger $logName "INFO" "Using backup: $mostRecentBackupName" 386 | 387 | logger $logName "INFO" "Copying DB backup, gitlab configurations and SSL ceritficates" 388 | mv /var/opt/gitlab/backups/$mostRecentBackupName $backupDir/ 389 | cp /etc/gitlab/gitlab.rb $backupDir/ 390 | cp /etc/gitlab/gitlab-secrets.json $backupDir/ 391 | cp -R /etc/gitlab/ssl/ $backupDir/ 392 | 393 | # Get the backup bucket name 394 | logger $logName "INFO" "Getting backup bucket name from instance metadata" 395 | GITLAB_BACKUPS_BUCKET_NAME=`curl -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/gitlab-backup-bucket-name` 396 | 397 | 398 | # Archive and encrypt backup 399 | logger $logName "INFO" "Archiving and encrypting backup" 400 | exec_wrapper_no_print $ERR_ACTION_EXIT $logName "7z a -p$GITLAB_BACKUP_PASSWORD $backupDir.zip ./$backupDir/*" 401 | 402 | # Upload archived backup 403 | logger $logName "INFO" "Uploading backup as: gs://$GITLAB_BACKUPS_BUCKET_NAME/gitlab-backups/$backupArchiveFile" 404 | exec_wrapper $ERR_ACTION_CONT $logName "gsutil cp $backupDir.zip gs://$GITLAB_BACKUPS_BUCKET_NAME/gitlab-backups/$backupArchiveFile" 405 | 406 | 407 | # Delete source directory and backup archive 408 | logger $logName "INFO" "Deleting processed files" 409 | rm -r $backupDir 410 | rm $backupDir.zip 411 | 412 | logger $logName "INFO" "Gitlab backup completed" 413 | } --------------------------------------------------------------------------------