├── backend.tf ├── .gitmodules ├── CODEOWNERS ├── get.jenkins.io.tf ├── variables.tf ├── Jenkinsfile_updatecli ├── .gitignore ├── main.tf ├── dns.tf ├── weekly.ci.jenkins.io.tf ├── versions.tf ├── updatecli ├── scripts │ ├── dateadd.sh │ └── datediff.sh ├── updatecli.d │ ├── terraform-providers │ │ ├── mysql.yaml │ │ ├── local.yaml │ │ ├── random.yaml │ │ ├── azuread.yaml │ │ ├── azurerm.yaml │ │ ├── kubernetes.yaml.disabled │ │ └── postgresql.yaml │ ├── service-ips.yaml │ ├── updatecli-end-dates.yaml.tpl │ ├── fs-sp-writer-end-dates_trusted.ci.jenkins.io.tf.tpl │ ├── packer-resources-azurevm-end-dates.yaml │ └── fs-sp-writer-end-dates_infra.ci.jenkins.io.tf.tpl └── values.yaml ├── uplink.jenkins.io.tf ├── rating.jenkins.io.tf ├── LICENSE ├── plugin-health.jenkins.io.tf ├── Jenkinsfile_k8s ├── docs.jenkins.io.tf ├── stats.jenkins.io.tf ├── contributors.jenkins.io.tf ├── updates.jenkins.io.tf ├── reports.jenkins.io.tf ├── keycloak.jenkins.io.tf ├── plugins.jenkins.io.tf ├── matomo.jenkins.io.tf ├── javadoc.jenkins.io.tf ├── ldap.jenkins.io.tf ├── archives.tf ├── data-storage-jenkins-io.tf ├── providers.tf ├── README.adoc ├── postgres-public-db.tf ├── release.ci.jenkins.io.tf ├── mysql-public-db.tf ├── packer-resources.tf ├── vnets.tf ├── agent.trusted.ci.jenkins.io.tf ├── puppet.jenkins.io.tf ├── cert.ci.jenkins.io.tf ├── dockerhub-mirror.tf ├── .terraform.lock.hcl ├── infraci.jenkins.io-agents-2.tf ├── outputs.tf ├── trusted.ci.jenkins.io.tf ├── publick8s.tf └── locals.tf /backend.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "azurerm" { 3 | } 4 | } 5 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule ".shared-tools"] 2 | path = .shared-tools 3 | url = https://github.com/jenkins-infra/shared-tools 4 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Order is important. The last matching pattern has the most precedence. 2 | 3 | * @jenkins-infra/azure 4 | * @jenkins-infra/kubernetes 5 | * @jenkins-infra/terraform 6 | -------------------------------------------------------------------------------- /get.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "get_jenkins_io" { 2 | name = "get-jenkins-io" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "location" { 2 | type = string 3 | default = "East US 2" 4 | } 5 | 6 | variable "terratest" { 7 | type = bool 8 | description = "value" 9 | default = false 10 | } 11 | -------------------------------------------------------------------------------- /Jenkinsfile_updatecli: -------------------------------------------------------------------------------- 1 | updatecli(action: 'diff') 2 | 3 | if (env.BRANCH_IS_PRIMARY) { 4 | // Only trigger a daily check on the principal branch 5 | properties([pipelineTriggers([cron('@daily')])]) 6 | updatecli(action: 'apply') 7 | } 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled files 2 | *.tfstate 3 | *.tfstate.backup 4 | *.html 5 | .ruby-* 6 | *.sw* 7 | .*.json 8 | .tf-prepare/ 9 | .tf-remote-state-enabled 10 | .terraform 11 | backend-config 12 | terraform-plan-output.txt 13 | tfplan 14 | # temporary and local test 15 | .tmp/ 16 | # sensitive files from terraform outputs 17 | .env* 18 | *.zip 19 | jenkins-infra-data-reports 20 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | # Service Principal ID used by the CI to authenticate terraform against the Azure API 2 | # Defined in the (private) repository jenkins-infra/terraform-states (in ./azure/main.tf) 3 | data "azuread_service_principal" "terraform_production" { 4 | display_name = "terraform-azure-production" 5 | } 6 | 7 | # Resource groups used to store (and lock) our public IPs 8 | resource "azurerm_resource_group" "prod_public_ips" { 9 | name = "prod-public-ips" 10 | location = var.location 11 | tags = local.default_tags 12 | } 13 | 14 | data "azurerm_client_config" "current" { 15 | } 16 | -------------------------------------------------------------------------------- /dns.tf: -------------------------------------------------------------------------------- 1 | # Jenkins.io DNS zone 2 | data "azurerm_resource_group" "proddns_jenkinsio" { 3 | name = "proddns_jenkinsio" 4 | } 5 | data "azurerm_dns_zone" "jenkinsio" { 6 | name = "jenkins.io" 7 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 8 | } 9 | 10 | # Jenkins-ci.org DNS zone 11 | data "azurerm_resource_group" "proddns_jenkinsci" { 12 | name = "proddns_jenkinsci" 13 | } 14 | data "azurerm_dns_zone" "jenkinsciorg" { 15 | name = "jenkins-ci.org" 16 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsci.name 17 | } 18 | -------------------------------------------------------------------------------- /weekly.ci.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "weekly_ci_jenkins_io" { 2 | name = "weekly-ci-jenkins-io" 3 | location = var.location 4 | } 5 | resource "azurerm_managed_disk" "weekly_ci_jenkins_io" { 6 | name = "weekly-ci-jenkins-io" 7 | location = azurerm_resource_group.weekly_ci_jenkins_io.location 8 | resource_group_name = azurerm_resource_group.weekly_ci_jenkins_io.name 9 | storage_account_type = "StandardSSD_ZRS" 10 | create_option = "Empty" 11 | disk_size_gb = 8 12 | tags = local.default_tags 13 | } 14 | -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | 2 | terraform { 3 | required_version = ">= 1.12, <1.13" 4 | required_providers { 5 | azurerm = { 6 | source = "hashicorp/azurerm" 7 | } 8 | azuread = { 9 | source = "hashicorp/azuread" 10 | } 11 | kubernetes = { 12 | source = "hashicorp/kubernetes" 13 | } 14 | local = { 15 | source = "hashicorp/local" 16 | } 17 | postgresql = { 18 | source = "cyrilgdn/postgresql" 19 | } 20 | random = { 21 | source = "hashicorp/random" 22 | } 23 | mysql = { 24 | source = "petoju/mysql" 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /updatecli/scripts/dateadd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script calculates next date for certificates 3 | ## 4 | set -eux -o pipefail 5 | DATE_BIN='date' 6 | 7 | ## non GNU operating system 8 | if command -v gdate >/dev/null 2>&1 9 | then 10 | DATE_BIN='gdate' 11 | fi 12 | command -v "${DATE_BIN}" >/dev/null 2>&1 || { echo "ERROR: ${DATE_BIN} command not found. Exiting."; exit 1; } 13 | 14 | # "${DATE_BIN}" --utc +"%Y-%m-%dT00:00:00Z" -d '+1 month' #keep for later use with allinone not alpine based 15 | "${DATE_BIN}" -d@"$(( $(date +%s)+60*60*24*30*3))" --utc '+%Y-%m-%dT00:00:00Z' # next date in around 3 month 16 | -------------------------------------------------------------------------------- /updatecli/scripts/datediff.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script calculate diff between dates for letsencrypt expiration 3 | set -eux -o pipefail 4 | 5 | currentexpirydate="${1}" 6 | DATE_BIN='date' 7 | 8 | ## non GNU operating system 9 | if command -v gdate >/dev/null 2>&1 10 | then 11 | DATE_BIN='gdate' 12 | fi 13 | command -v "${DATE_BIN}" >/dev/null 2>&1 || { echo "ERROR: ${DATE_BIN} command not found. Exiting."; exit 1; } 14 | 15 | currentdateepoch=$("${DATE_BIN}" --utc "+%s" 2>/dev/null) 16 | expirydateepoch=$("${DATE_BIN}" "+%s" -d "$currentexpirydate") 17 | 18 | datediff=$(((expirydateepoch-currentdateepoch)/(60*60*24))) # diff per days 19 | 20 | if [ "$datediff" -lt 21 ] # launch renew 21 days before expiration 21 | then 22 | echo "time for update" 23 | exit 0 24 | else 25 | echo "not yet expired" 26 | exit 1 27 | fi 28 | -------------------------------------------------------------------------------- /uplink.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "postgresql_database" "uplink" { 2 | name = "uplink" 3 | owner = postgresql_role.uplink.name 4 | } 5 | resource "random_password" "pgsql_uplink_user_password" { 6 | length = 24 7 | special = false 8 | } 9 | resource "postgresql_role" "uplink" { 10 | name = "uplinkadmin" 11 | login = true 12 | password = random_password.pgsql_uplink_user_password.result 13 | } 14 | 15 | # This (sensitive) output is meant to be encrypted into the production secret system, to be provided as a secret to the uplink.jenkins.io application 16 | output "uplink_dbconfig" { 17 | sensitive = true 18 | description = "YAML (secret) values for the Helm chart jenkins-infra/uplink" 19 | value = <<-EOT 20 | postgresql: 21 | url: postgres://${postgresql_role.uplink.name}:${random_password.pgsql_uplink_user_password.result}@${azurerm_postgresql_flexible_server.public_db.fqdn}:5432/${postgresql_database.uplink.name} 22 | EOT 23 | } 24 | -------------------------------------------------------------------------------- /rating.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "postgresql_database" "rating" { 2 | name = "rating" 3 | owner = postgresql_role.rating.name 4 | } 5 | 6 | resource "random_password" "pgsql_rating_user_password" { 7 | length = 24 8 | override_special = "!#%&*()-_=+[]{}:?" 9 | special = true 10 | } 11 | 12 | resource "postgresql_role" "rating" { 13 | name = "rating" 14 | login = true 15 | password = random_password.pgsql_rating_user_password.result 16 | } 17 | 18 | # This (sensitive) output is meant to be encrypted into the production secret system, to be provided as a secret to the ratings.jenkins.io application 19 | output "rating_dbconfig" { 20 | sensitive = true 21 | description = "YAML (secret) values for the Helm chart jenkins-infra/rating" 22 | value = <<-EOT 23 | database: 24 | username: "${postgresql_role.rating.name}" 25 | password: "${random_password.pgsql_rating_user_password.result}" 26 | server: "${azurerm_postgresql_flexible_server.public_db.fqdn}" 27 | name: "${postgresql_database.rating.name}" 28 | EOT 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Jenkins Infra 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /plugin-health.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "postgresql_database" "plugin_health" { 2 | name = "plugin_health" 3 | owner = postgresql_role.plugin_health.name 4 | } 5 | 6 | resource "random_password" "pgsql_plugin_health_user_password" { 7 | length = 24 8 | override_special = "!#%&*()-_=+[]{}:?" 9 | special = true 10 | } 11 | 12 | resource "postgresql_role" "plugin_health" { 13 | name = "plugin_health" 14 | login = true 15 | password = random_password.pgsql_plugin_health_user_password.result 16 | } 17 | 18 | # This (sensitive) output is meant to be encrypted into the production secret system, to be provided as a secret to the plugin-health.jenkins.io application 19 | output "plugin_health_dbconfig" { 20 | sensitive = true 21 | description = "YAML (secret) values for the Helm chart jenkins-infra/plugin-health-scoring" 22 | value = <<-EOT 23 | database: 24 | username: "${postgresql_role.plugin_health.name}" 25 | password: "${random_password.pgsql_plugin_health_user_password.result}" 26 | server: "${azurerm_postgresql_flexible_server.public_db.fqdn}" 27 | name: "${postgresql_database.plugin_health.name}" 28 | EOT 29 | } 30 | -------------------------------------------------------------------------------- /Jenkinsfile_k8s: -------------------------------------------------------------------------------- 1 | if (env.BRANCH_IS_PRIMARY) { 2 | // Only trigger a daily check on the principal branch 3 | properties([pipelineTriggers([cron('@daily')])]) 4 | } 5 | 6 | terraform( 7 | stagingCredentials: [ 8 | azureServicePrincipal( 9 | credentialsId: 'staging-terraform-azure-serviceprincipal', 10 | subscriptionIdVariable: 'ARM_SUBSCRIPTION_ID', 11 | clientIdVariable: 'ARM_CLIENT_ID', 12 | clientSecretVariable: 'ARM_CLIENT_SECRET', 13 | tenantIdVariable: 'ARM_TENANT_ID', 14 | ), 15 | file( 16 | credentialsId: 'staging-terraform-azure-backend-config', 17 | variable: 'BACKEND_CONFIG_FILE', 18 | ), 19 | ], 20 | productionCredentials: [ 21 | azureServicePrincipal( 22 | credentialsId: 'production-terraform-azure-serviceprincipal', 23 | subscriptionIdVariable: 'ARM_SUBSCRIPTION_ID', 24 | clientIdVariable: 'ARM_CLIENT_ID', 25 | clientSecretVariable: 'ARM_CLIENT_SECRET', 26 | tenantIdVariable: 'ARM_TENANT_ID', 27 | ), 28 | file( 29 | credentialsId: 'production-terraform-azure-backend-config', 30 | variable: 'BACKEND_CONFIG_FILE', 31 | ), 32 | ], 33 | publishReports: ['jenkins-infra-data-reports/azure.json'], 34 | ) 35 | -------------------------------------------------------------------------------- /docs.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "docs_jenkins_io" { 2 | name = "docs-jenkins-io" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | resource "azurerm_storage_account" "docs_jenkins_io" { 8 | name = "docsjenkinsio" 9 | resource_group_name = azurerm_resource_group.docs_jenkins_io.name 10 | location = azurerm_resource_group.docs_jenkins_io.location 11 | account_tier = "Standard" 12 | account_replication_type = "ZRS" 13 | account_kind = "StorageV2" 14 | https_traffic_only_enabled = true 15 | min_tls_version = "TLS1_2" 16 | 17 | network_rules { 18 | default_action = "Deny" 19 | virtual_network_subnet_ids = concat( 20 | [ 21 | # Required for using the resource 22 | data.azurerm_subnet.publick8s.id, 23 | ], 24 | # Required for managing the resource 25 | local.app_subnets["infra.ci.jenkins.io"].agents, 26 | ) 27 | bypass = ["AzureServices"] 28 | } 29 | 30 | tags = local.default_tags 31 | } 32 | 33 | resource "azurerm_storage_share" "docs_jenkins_io" { 34 | name = "docs-jenkins-io" 35 | storage_account_id = azurerm_storage_account.docs_jenkins_io.id 36 | quota = 5 37 | } 38 | -------------------------------------------------------------------------------- /stats.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "stats_jenkins_io" { 2 | name = "stats-jenkins-io" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | resource "azurerm_storage_account" "stats_jenkins_io" { 8 | name = "statsjenkinsio" 9 | resource_group_name = azurerm_resource_group.stats_jenkins_io.name 10 | location = azurerm_resource_group.stats_jenkins_io.location 11 | account_tier = "Standard" 12 | account_replication_type = "ZRS" 13 | account_kind = "StorageV2" 14 | https_traffic_only_enabled = true 15 | min_tls_version = "TLS1_2" 16 | 17 | network_rules { 18 | default_action = "Deny" 19 | virtual_network_subnet_ids = concat( 20 | [ 21 | # Required for using and populating the resource 22 | data.azurerm_subnet.publick8s.id, 23 | ], 24 | # Required for managing the resource 25 | local.app_subnets["infra.ci.jenkins.io"].agents, 26 | ) 27 | bypass = ["AzureServices"] 28 | } 29 | 30 | tags = local.default_tags 31 | } 32 | 33 | resource "azurerm_storage_share" "stats_jenkins_io" { 34 | name = "stats-jenkins-io" 35 | storage_account_id = azurerm_storage_account.stats_jenkins_io.id 36 | quota = 5 37 | } 38 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/mysql.yaml: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `mysql` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `mysql` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: petoju 22 | name: mysql 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: petoju/mysql 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `petoju/mysql` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - petoju/mysql 48 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/local.yaml: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `local` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `local` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: hashicorp 22 | name: local 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: hashicorp/local 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `local` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - hashicorp/local 48 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/random.yaml: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `random` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `random` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: hashicorp 22 | name: random 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: hashicorp/random 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `random` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - hashicorp/random 48 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/azuread.yaml: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `azuread` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `azuread` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: hashicorp 22 | name: azuread 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: hashicorp/azuread 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `azuread` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - hashicorp/azuread 48 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/azurerm.yaml: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `azurerm` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `azurerm` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: hashicorp 22 | name: azurerm 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: hashicorp/azurerm 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `azurerm` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - hashicorp/azurerm 48 | -------------------------------------------------------------------------------- /contributors.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "contributors_jenkins_io" { 2 | name = "contributors-jenkins-io" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | resource "azurerm_storage_account" "contributors_jenkins_io" { 8 | name = "contributorsjenkinsio" 9 | resource_group_name = azurerm_resource_group.contributors_jenkins_io.name 10 | location = azurerm_resource_group.contributors_jenkins_io.location 11 | account_tier = "Standard" 12 | account_replication_type = "ZRS" 13 | account_kind = "StorageV2" 14 | https_traffic_only_enabled = true 15 | min_tls_version = "TLS1_2" 16 | 17 | network_rules { 18 | default_action = "Deny" 19 | virtual_network_subnet_ids = concat( 20 | [ 21 | # Required for using the resource 22 | data.azurerm_subnet.publick8s.id, 23 | ], 24 | # Required for managing the resource 25 | local.app_subnets["infra.ci.jenkins.io"].agents, 26 | ) 27 | bypass = ["AzureServices"] 28 | } 29 | 30 | tags = local.default_tags 31 | } 32 | 33 | resource "azurerm_storage_share" "contributors_jenkins_io" { 34 | name = "contributors-jenkins-io" 35 | storage_account_id = azurerm_storage_account.contributors_jenkins_io.id 36 | quota = 5 37 | } 38 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/kubernetes.yaml.disabled: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `kubernetes` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `kubernetes` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: hashicorp 22 | name: kubernetes 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: hashicorp/kubernetes 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `kubernetes` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - hashicorp/kubernetes 48 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/terraform-providers/postgresql.yaml: -------------------------------------------------------------------------------- 1 | name: "Bump Terraform `postgresql` provider version" 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | lastVersion: 17 | name: Get latest version of the `postgresql` provider 18 | kind: terraform/registry 19 | spec: 20 | type: provider 21 | namespace: cyrilgdn 22 | name: postgresql 23 | 24 | targets: 25 | updateTerraformLockFile: 26 | name: Update Terraform lock file 27 | kind: terraform/lock 28 | sourceid: lastVersion 29 | spec: 30 | file: .terraform.lock.hcl 31 | provider: cyrilgdn/postgresql 32 | platforms: 33 | - linux_amd64 34 | - linux_arm64 35 | - darwin_amd64 36 | - darwin_arm64 37 | scmid: default 38 | 39 | actions: 40 | default: 41 | kind: github/pullrequest 42 | scmid: default 43 | spec: 44 | title: Bump Terraform `cyrilgdn/postgresql` provider version to {{ source "lastVersion" }} 45 | labels: 46 | - terraform-providers 47 | - cyrilgdn/postgresql 48 | -------------------------------------------------------------------------------- /updates.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "updates_jenkins_io" { 2 | name = "updates-jenkins-io" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | ## NS records for each CloudFlare zone defined in https://github.com/jenkins-infra/cloudflare/blob/main/updates.jenkins.io.tf 8 | # West Europe 9 | resource "azurerm_dns_ns_record" "updates_jenkins_io_cloudflare_zone_westeurope" { 10 | name = "westeurope.cloudflare" 11 | zone_name = data.azurerm_dns_zone.jenkinsio.name 12 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 13 | ttl = 60 14 | # Should correspond to the "zones_name_servers" output defined in https://github.com/jenkins-infra/cloudflare/blob/main/updates.jenkins.io.tf 15 | records = ["jaxson.ns.cloudflare.com", "mira.ns.cloudflare.com"] 16 | tags = local.default_tags 17 | } 18 | # East US 19 | resource "azurerm_dns_ns_record" "updates_jenkins_io_cloudflare_zone_eastamerica" { 20 | name = "eastamerica.cloudflare" 21 | zone_name = data.azurerm_dns_zone.jenkinsio.name 22 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 23 | ttl = 60 24 | # Should correspond to the "zones_name_servers" output defined in https://github.com/jenkins-infra/cloudflare/blob/main/updates.jenkins.io.tf 25 | records = ["jaxson.ns.cloudflare.com", "mira.ns.cloudflare.com"] 26 | tags = local.default_tags 27 | } 28 | -------------------------------------------------------------------------------- /reports.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "reports_jenkins_io" { 2 | name = "reports-jenkins-io" 3 | location = var.location 4 | 5 | tags = { 6 | scope = "terraform-managed" 7 | } 8 | } 9 | ## trusted.ci.jenkins.io and infra.ci.jenkins.io are using the Storage Account Key to read and write 10 | resource "azurerm_storage_account" "reports_jenkins_io" { 11 | name = "reportsjenkinsio" 12 | resource_group_name = azurerm_resource_group.reports_jenkins_io.name 13 | location = azurerm_resource_group.reports_jenkins_io.location 14 | account_tier = "Standard" 15 | account_replication_type = "ZRS" 16 | account_kind = "StorageV2" 17 | https_traffic_only_enabled = true 18 | min_tls_version = "TLS1_2" 19 | 20 | network_rules { 21 | default_action = "Deny" 22 | virtual_network_subnet_ids = concat( 23 | [ 24 | # Required for using the resource 25 | data.azurerm_subnet.publick8s.id, 26 | ], 27 | # Required for populating the resource from infra-reports 28 | local.app_subnets["infra.ci.jenkins.io"].agents, 29 | # Required for populating the resource from coretaglib and RPU 30 | local.app_subnets["trusted.ci.jenkins.io"].agents, 31 | ) 32 | bypass = ["AzureServices"] 33 | } 34 | 35 | tags = local.default_tags 36 | } 37 | resource "azurerm_storage_share" "reports_jenkins_io" { 38 | name = "reports-jenkins-io" 39 | storage_account_id = azurerm_storage_account.reports_jenkins_io.id 40 | quota = 5 41 | } 42 | -------------------------------------------------------------------------------- /keycloak.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "random_password" "pgsql_keycloak_user_password" { 2 | length = 24 3 | override_special = "!#%&*()-_=+[]{}:?" 4 | special = true 5 | } 6 | 7 | resource "postgresql_role" "keycloak" { 8 | name = "keycloak" 9 | login = true 10 | password = random_password.pgsql_keycloak_user_password.result 11 | } 12 | 13 | resource "postgresql_database" "keycloak" { 14 | name = "keycloak" 15 | owner = postgresql_role.keycloak.name 16 | } 17 | 18 | # This (sensitive) output is meant to be encrypted into the production secret system, to be provided as a secret to the Keycloak application (https://admin.accounts.jenkins.io) 19 | output "keycloak_dbconfig" { 20 | # Value of DB_PORT: 5432 is the only usable port: https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-networking 21 | ## Terraform resource does not export any port attribute: https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/postgresql_flexible_server#attributes-reference 22 | sensitive = true 23 | description = "YAML (secret) values for the Helm chart codecentric/keycloak" 24 | value = <<-EOT 25 | secrets: 26 | db: 27 | data: 28 | DB_USER: ${base64encode(postgresql_role.keycloak.name)} 29 | DB_PASSWORD: ${base64encode(random_password.pgsql_keycloak_user_password.result)} 30 | DB_VENDOR: ${base64encode("postgres")} 31 | DB_ADDR: ${base64encode(azurerm_postgresql_flexible_server.public_db.fqdn)} 32 | DB_PORT: ${base64encode("5432")} 33 | DB_DATABASE: ${base64encode(postgresql_database.keycloak.name)} 34 | EOT 35 | } 36 | -------------------------------------------------------------------------------- /plugins.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "plugins_jenkins_io" { 2 | name = "pluginsjenkinsio" 3 | location = var.location 4 | } 5 | 6 | resource "azurerm_storage_account" "plugins_jenkins_io" { 7 | name = "pluginsjenkinsio" 8 | resource_group_name = azurerm_resource_group.plugins_jenkins_io.name 9 | location = azurerm_resource_group.plugins_jenkins_io.location 10 | account_tier = "Premium" 11 | account_kind = "FileStorage" 12 | access_tier = "Hot" 13 | account_replication_type = "ZRS" 14 | min_tls_version = "TLS1_2" # default value, needed for tfsec 15 | infrastructure_encryption_enabled = true 16 | 17 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 18 | network_rules { 19 | default_action = "Deny" 20 | virtual_network_subnet_ids = concat( 21 | [ 22 | # Required for using the resource 23 | data.azurerm_subnet.publick8s.id, 24 | ], 25 | # Required for managing and populating the resource 26 | local.app_subnets["infra.ci.jenkins.io"].agents, 27 | ) 28 | bypass = ["Metrics", "Logging", "AzureServices"] 29 | } 30 | 31 | tags = local.default_tags 32 | } 33 | 34 | resource "azurerm_storage_share" "plugins_jenkins_io" { 35 | name = "plugins-jenkins-io" 36 | storage_account_id = azurerm_storage_account.plugins_jenkins_io.id 37 | quota = 100 # Minimum size when using a Premium storage account 38 | } 39 | -------------------------------------------------------------------------------- /matomo.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | ## Matomo Resources 2 | 3 | # Database - ref. https://matomo.org/faq/how-to-install/faq_23484/ 4 | resource "mysql_database" "matomo" { 5 | name = "matomo" 6 | } 7 | resource "random_password" "matomo_mysql_password" { 8 | length = 81 9 | lower = true 10 | min_lower = 1 11 | min_numeric = 1 12 | min_special = 1 13 | min_upper = 1 14 | numeric = true 15 | override_special = "_" 16 | special = true 17 | upper = true 18 | } 19 | resource "mysql_user" "matomo" { 20 | user = "matomo" 21 | host = "*" # Default "localhost" forbids access from clusters 22 | plaintext_password = random_password.matomo_mysql_password.result 23 | } 24 | resource "mysql_grant" "matomo" { 25 | user = mysql_user.matomo.user 26 | host = mysql_user.matomo.host 27 | database = mysql_database.matomo.name 28 | privileges = ["SELECT", "INSERT", "UPDATE", "DELETE", "CREATE", "INDEX", "DROP", "ALTER", "CREATE TEMPORARY TABLES", "LOCK TABLES"] 29 | } 30 | 31 | # This (sensitive) output is meant to be encrypted into the production secret system, to be provided as a secret to the matomo application 32 | output "matomo_dbconfig" { 33 | # Value of the port is fixed to 3306 (https://learn.microsoft.com/en-us/azure/mysql/flexible-server/concepts-networking and https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_flexible_server#attributes-reference) 34 | sensitive = true 35 | description = "YAML (secret) values for the Helm chart bitnami/matomo" 36 | value = <<-EOT 37 | externalDatabase: 38 | host: ${azurerm_mysql_flexible_server.public_db_mysql.fqdn} 39 | port: 3306 40 | database: ${mysql_database.matomo.name} 41 | user: ${mysql_user.matomo.user} 42 | password: ${random_password.matomo_mysql_password.result} 43 | EOT 44 | } 45 | -------------------------------------------------------------------------------- /javadoc.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | # resource "azurerm_resource_group" "javadoc_jenkins_io" { 2 | resource "azurerm_resource_group" "javadoc" { 3 | name = "javadoc-jenkins-io" 4 | location = var.location 5 | } 6 | 7 | ### TODO Remove below 8 | resource "azurerm_resource_group" "javadocjenkinsio" { 9 | name = "javadocjenkinsio" 10 | location = var.location 11 | } 12 | resource "azurerm_storage_account" "javadoc_jenkins_io" { 13 | name = "javadocjenkinsio" 14 | resource_group_name = azurerm_resource_group.javadocjenkinsio.name 15 | location = azurerm_resource_group.javadocjenkinsio.location 16 | account_tier = "Premium" 17 | account_kind = "FileStorage" 18 | access_tier = "Hot" 19 | account_replication_type = "ZRS" 20 | min_tls_version = "TLS1_2" # default value, needed for tfsec 21 | infrastructure_encryption_enabled = true 22 | 23 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 24 | network_rules { 25 | default_action = "Deny" 26 | virtual_network_subnet_ids = concat( 27 | [ 28 | # Required for using the resource 29 | data.azurerm_subnet.publick8s.id, 30 | ], 31 | # Required for managing the resource 32 | local.app_subnets["infra.ci.jenkins.io"].agents, 33 | # Required for populating the resource when a release is performed 34 | local.app_subnets["trusted.ci.jenkins.io"].agents, 35 | ) 36 | bypass = ["Metrics", "Logging", "AzureServices"] 37 | } 38 | 39 | tags = local.default_tags 40 | } 41 | 42 | resource "azurerm_storage_share" "javadoc_jenkins_io" { 43 | name = "javadoc-jenkins-io" 44 | storage_account_id = azurerm_storage_account.javadoc_jenkins_io.id 45 | quota = 100 # Minimum size when using a Premium storage account 46 | } 47 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/service-ips.yaml: -------------------------------------------------------------------------------- 1 | name: Update external service IPs 2 | 3 | scms: 4 | default: 5 | kind: github 6 | spec: 7 | user: "{{ .github.user }}" 8 | email: "{{ .github.email }}" 9 | owner: "{{ .github.owner }}" 10 | repository: "{{ .github.repository }}" 11 | token: "{{ requiredEnv .github.token }}" 12 | username: "{{ .github.username }}" 13 | branch: "{{ .github.branch }}" 14 | 15 | sources: 16 | getTrustedCiJenkinsIo: 17 | kind: json 18 | spec: 19 | file: https://reports.jenkins.io/jenkins-infra-data-reports/azure-net.json 20 | key: .trusted\.ci\.jenkins\.io.outbound_ips 21 | transformers: 22 | - trimprefix: "[" 23 | - trimsuffix: "]" 24 | 25 | getInfraCiJenkinsIo: 26 | kind: json 27 | spec: 28 | file: https://reports.jenkins.io/jenkins-infra-data-reports/azure-net.json 29 | key: .infra\.ci\.jenkins\.io.outbound_ips 30 | transformers: 31 | - trimprefix: "[" 32 | - trimsuffix: "]" 33 | 34 | getPrivateVpn: 35 | kind: json 36 | spec: 37 | file: https://reports.jenkins.io/jenkins-infra-data-reports/azure-net.json 38 | key: .private\.vpn\.jenkins\.io.outbound_ips 39 | transformers: 40 | - trimprefix: "[" 41 | - trimsuffix: "]" 42 | 43 | targets: 44 | setTrustedCiJenkinsIo: 45 | sourceid: getTrustedCiJenkinsIo 46 | name: Update trusted.ci.jenkins.io outbound IPs 47 | kind: hcl 48 | spec: 49 | file: locals.tf 50 | path: locals.outbound_ips_trusted_ci_jenkins_io 51 | scmid: default 52 | 53 | setInfraCiJenkinsIo: 54 | sourceid: getInfraCiJenkinsIo 55 | name: Update infra.ci.jenkins.io outbound IPs 56 | kind: hcl 57 | spec: 58 | file: locals.tf 59 | path: locals.outbound_ips_infra_ci_jenkins_io 60 | scmid: default 61 | 62 | setPrivateVpn: 63 | sourceid: getPrivateVpn 64 | name: Update private.vpn.jenkins.io outbound IPs 65 | kind: hcl 66 | spec: 67 | file: locals.tf 68 | path: locals.outbound_ips_private_vpn_jenkins_io 69 | scmid: default 70 | 71 | actions: 72 | default: 73 | kind: github/pullrequest 74 | scmid: default 75 | title: Update the service IPs 76 | spec: 77 | labels: 78 | - dependencies 79 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/updatecli-end-dates.yaml.tpl: -------------------------------------------------------------------------------- 1 | {{ range $key, $val := .updatecli_end_dates }} 2 | {{ $hclFile := printf "%s%s" $key ".tf" }} 3 | {{ $hclKey := "module.controller_service_principal_end_date" }} 4 | {{ if (and $val $val.custom_hcl_key) }} 5 | {{ $hclKey = $val.custom_hcl_key }} 6 | {{ end }} 7 | --- 8 | # yamllint disable rule:line-length 9 | name: "Generate new end date for the {{ $key }} Azure AD Application password" 10 | 11 | scms: 12 | default: 13 | kind: github 14 | spec: 15 | user: "{{ $.github.user }}" 16 | email: "{{ $.github.email }}" 17 | owner: "{{ $.github.owner }}" 18 | repository: "{{ $.github.repository }}" 19 | token: "{{ requiredEnv $.github.token }}" 20 | username: "{{ $.github.username }}" 21 | branch: "{{ $.github.branch }}" 22 | 23 | sources: 24 | currentEndDate: 25 | name: Get current `end_date` date 26 | kind: hcl 27 | spec: 28 | file: {{ $hclFile }} 29 | path: {{ $hclKey }} 30 | nextEndDate: 31 | name: Prepare next `end_date` date within 3 months 32 | kind: shell 33 | spec: 34 | command: bash ./updatecli/scripts/dateadd.sh 35 | environments: 36 | - name: PATH 37 | 38 | conditions: 39 | checkIfEndDateSoonExpired: 40 | kind: shell 41 | sourceid: currentEndDate 42 | spec: 43 | # Current end_date date value passed as argument 44 | command: bash ./updatecli/scripts/datediff.sh 45 | environments: 46 | - name: PATH 47 | 48 | targets: 49 | updateNextEndDate: 50 | name: Update Terraform file `{{ $key }}.tf` with new expiration date 51 | kind: hcl 52 | sourceid: nextEndDate 53 | spec: 54 | file: {{ $hclFile }} 55 | path: {{ $hclKey }} 56 | scmid: default 57 | 58 | actions: 59 | default: 60 | kind: github/pullrequest 61 | scmid: default 62 | spec: 63 | title: 'Azure AD Application password for updatecli in `{{ $key }}` expires on `{{ source "currentEndDate" }}`' 64 | description: | 65 | This PR updates the Azure AD application password used in `{{ $key }}` for updatecli. 66 | 67 | The current end date is set to `{{ source "currentEndDate" }}`. 68 | 69 | {{ $val.doc_how_to_get_credential | indent 8 }} 70 | 71 | labels: 72 | - azure-ad-application 73 | - end-dates 74 | - {{ $key }} 75 | {{ end }} 76 | -------------------------------------------------------------------------------- /ldap.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "ldap_jenkins_io" { 2 | name = "ldap-jenkins-io" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | resource "azurerm_managed_disk" "ldap_jenkins_io_data" { 8 | name = "ldap-jenkins-io-data" 9 | location = azurerm_resource_group.ldap_jenkins_io.location 10 | resource_group_name = azurerm_resource_group.ldap_jenkins_io.name 11 | # ZRS to ensure we can move service across AZs 12 | # Standard because it is enough for LDAP's IOPS and I/O bandwidth 13 | # Ref. https://azure.microsoft.com/en-us/pricing/details/managed-disks/ 14 | storage_account_type = "StandardSSD_ZRS" 15 | create_option = "Empty" 16 | # LDAP data set is between 300 and 500 Mb 17 | # Class E1 (4G) only allow 7800 paid transactions per hour, while LDAP may peak at 8500 sometimes so E2 it is 18 | # Ref. https://azure.microsoft.com/en-us/pricing/details/managed-disks/ 19 | disk_size_gb = 8 20 | tags = local.default_tags 21 | } 22 | resource "azurerm_storage_account" "ldap_jenkins_io" { 23 | name = "ldapjenkinsio" 24 | resource_group_name = azurerm_resource_group.ldap_jenkins_io.name 25 | location = azurerm_resource_group.ldap_jenkins_io.location 26 | account_tier = "Standard" 27 | account_replication_type = "ZRS" 28 | account_kind = "StorageV2" 29 | https_traffic_only_enabled = true 30 | min_tls_version = "TLS1_2" # default value, needed for tfsec 31 | infrastructure_encryption_enabled = true # LDAP data is sensitive, even if password are encrypted 32 | 33 | network_rules { 34 | default_action = "Deny" 35 | virtual_network_subnet_ids = concat( 36 | [ 37 | # Required for using the resource 38 | data.azurerm_subnet.publick8s.id, 39 | ], 40 | # Required for managing the resource 41 | local.app_subnets["infra.ci.jenkins.io"].agents, 42 | ) 43 | bypass = ["AzureServices"] 44 | } 45 | 46 | tags = local.default_tags 47 | } 48 | resource "azurerm_storage_share" "ldap_jenkins_io_backups" { 49 | name = "ldap" 50 | storage_account_id = azurerm_storage_account.ldap_jenkins_io.id 51 | # Unless this is a Premium Storage, we only pay for the storage we consume 52 | quota = 10 53 | } 54 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/fs-sp-writer-end-dates_trusted.ci.jenkins.io.tf.tpl: -------------------------------------------------------------------------------- 1 | {{ range $key, $val := .end_dates.trusted_ci_jenkins_io }} 2 | --- 3 | # yamllint disable rule:line-length 4 | name: "Generate new end date for {{ $val.service }} File Share service principal writer on trusted.ci.jenkins.io" 5 | 6 | scms: 7 | default: 8 | kind: github 9 | spec: 10 | user: "{{ $.github.user }}" 11 | email: "{{ $.github.email }}" 12 | owner: "{{ $.github.owner }}" 13 | repository: "{{ $.github.repository }}" 14 | token: "{{ requiredEnv $.github.token }}" 15 | username: "{{ $.github.username }}" 16 | branch: "{{ $.github.branch }}" 17 | 18 | sources: 19 | currentEndDate: 20 | name: Get current `end_date` date 21 | kind: hcl 22 | spec: 23 | file: trusted.ci.jenkins.io.tf 24 | path: module.{{ $key }}.service_principal_end_date 25 | nextEndDate: 26 | name: Prepare next `end_date` date within 3 months 27 | kind: shell 28 | spec: 29 | command: bash ./updatecli/scripts/dateadd.sh 30 | environments: 31 | - name: PATH 32 | 33 | conditions: 34 | checkIfEndDateSoonExpired: 35 | kind: shell 36 | sourceid: currentEndDate 37 | spec: 38 | # Current end_date date value passed as argument 39 | command: bash ./updatecli/scripts/datediff.sh 40 | environments: 41 | - name: PATH 42 | 43 | targets: 44 | updateNextEndDate: 45 | name: 'New end date for `{{ $val.service }}` File Share service principal writer on `trusted.ci.jenkins.io` (current: {{ source "currentEndDate" }})' 46 | kind: hcl 47 | sourceid: nextEndDate 48 | spec: 49 | file: trusted.ci.jenkins.io.tf 50 | path: module.{{ $key }}.service_principal_end_date 51 | scmid: default 52 | 53 | actions: 54 | default: 55 | kind: github/pullrequest 56 | scmid: default 57 | spec: 58 | title: 'Azure File Share Principal `{{ $val.service }}` on `trusted.ci.jenkins.io` expires on `{{ source "currentEndDate" }}`' 59 | description: | 60 | This PR updates the end date of {{ $val.service }} File Share service principal writer used in trusted.ci.jenkins.io. 61 | 62 | The current end date is set to `{{ source "currentEndDate" }}`. 63 | 64 | {{ $val.doc_how_to_get_credential | indent 8 }} 65 | 66 | labels: 67 | - terraform 68 | - "{{ $val.service }}" 69 | - end-dates 70 | - trusted.ci.jenkins.io 71 | {{ end }} 72 | -------------------------------------------------------------------------------- /archives.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "archives" { 2 | name = "archives" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | resource "azurerm_storage_account" "archives" { 8 | name = "jenkinsinfraarchives" 9 | resource_group_name = azurerm_resource_group.archives.name 10 | location = azurerm_resource_group.archives.location 11 | account_tier = "Standard" 12 | account_replication_type = "GRS" # recommended for backups 13 | # https://learn.microsoft.com/en-gb/azure/storage/common/infrastructure-encryption-enable 14 | infrastructure_encryption_enabled = true 15 | min_tls_version = "TLS1_2" # default value, needed for tfsec 16 | 17 | network_rules { 18 | default_action = "Deny" 19 | virtual_network_subnet_ids = concat( 20 | # Required for managing the resource 21 | local.app_subnets["infra.ci.jenkins.io"].agents, 22 | ) 23 | bypass = ["AzureServices"] 24 | } 25 | 26 | tags = local.default_tags 27 | } 28 | 29 | ## Archived items 30 | # Container for the logs archive (2019 -> 2025) of the legacy `updates.jenkins.io` service which used to be in the 'pkg' CloudBees AWS VM 31 | resource "azurerm_storage_container" "legacy_updatesjio_logs" { 32 | name = "legacy-updatesjio-logs" 33 | storage_account_id = azurerm_storage_account.archives.id 34 | container_access_type = "private" 35 | metadata = merge(local.default_tags, { 36 | helpdesk = "https://github.com/jenkins-infra/helpdesk/issues/2649" 37 | }) 38 | } 39 | 40 | # Container for the dump of confluence databases 41 | resource "azurerm_storage_container" "confluence_dumps" { 42 | name = "confluence-databases-dump" 43 | storage_account_id = azurerm_storage_account.archives.id 44 | container_access_type = "private" 45 | metadata = merge(local.default_tags, { 46 | helpdesk = "https://github.com/jenkins-infra/helpdesk/issues/3249" 47 | }) 48 | } 49 | 50 | # Container for the dump of confluence databases - ref. https://github.com/jenkins-infra/helpdesk/issues/4667 51 | resource "azurerm_storage_container" "uplink_db_pre_20250521" { 52 | name = "uplink-db-pre-20250521" 53 | storage_account_id = azurerm_storage_account.archives.id 54 | container_access_type = "private" 55 | metadata = merge(local.default_tags, { 56 | helpdesk = "https://github.com/jenkins-infra/helpdesk/issues/3249" 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /updatecli/values.yaml: -------------------------------------------------------------------------------- 1 | github: 2 | user: "Jenkins Infra Bot (updatecli)" 3 | email: "60776566+jenkins-infra-bot@users.noreply.github.com" 4 | token: "UPDATECLI_GITHUB_TOKEN" 5 | branch: "main" 6 | owner: "jenkins-infra" 7 | repository: "azure" 8 | # Also used by terraform in locals.tf 9 | end_dates: 10 | infra_ci_jenkins_io: 11 | infraci_contributorsjenkinsio_fileshare_serviceprincipal_writer: 12 | end_date: "2026-02-15T00:00:00Z" 13 | service: "contributors.jenkins.io" 14 | secret: "CONTRIBUTORS_SERVICE_PRINCIPAL_WRITER_CLIENT_SECRET" 15 | infraci_docsjenkinsio_fileshare_serviceprincipal_writer: 16 | end_date: "2026-03-19T00:00:00Z" 17 | service: "docs.jenkins.io" 18 | secret: "DOCS_SERVICE_PRINCIPAL_WRITER_CLIENT_SECRET" 19 | infraci_pluginsjenkinsio_fileshare_serviceprincipal_writer: 20 | end_date: "2026-02-15T00:00:00Z" 21 | service: "plugins.jenkins.io" 22 | secret: "INFRACI_PLUGINSJENKINSIO_FILESHARE_SERVICE_PRINCIPAL_WRITER_PASSWORD" 23 | infraci_statsjenkinsio_fileshare_serviceprincipal_writer: 24 | end_date: "2026-02-15T00:00:00Z" 25 | service: "stats.jenkins.io" 26 | secret: "STATS_SERVICE_PRINCIPAL_WRITER_CLIENT_SECRET" 27 | updatecli_end_dates: 28 | infra.ci.jenkins.io: 29 | custom_hcl_key: resource.azuread_application_password.updatecli_infra_ci_jenkins_io.end_date 30 | doc_how_to_get_credential: | 31 | > [!IMPORTANT] 32 | > 33 | > ⚠️ Merging this PR will prevent updatecli to use `az` until the credential is updated on the controller. 34 | 35 | You'll have to update the credential on infra.ci.jenkins.io's encrypted secrets: 36 | 37 | - Update the secret value in jenkins-infra/chart-secrets (or kubernetes-management/secrets), add, commit and push the change 38 | - Trigger a build of the `kubernetes-management` job on infra.ci.jenkins.io to ensure secret value is updated in Kubernetes secrets 39 | - Finally, trigger a reload from jcasc or a controller restart (pod delete, or rollout) to make sure secrets are used to update the Jenkins credential. 40 | - test by replaying a build on main `https://infra.ci.jenkins.io/job/updatecli/job/packer-images/job/main/` and check the logs for an azure check (⚠️ do not rely on green result) 41 | 42 | The new password value, once the PR is merged and deployed, can be retrieved from the Terraform state, 43 | by searching for `azuread_application_password.updatecli_infra_ci_jenkins_io.value`. 44 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/packer-resources-azurevm-end-dates.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "Generate new end date for the packer Azure AD Application password" 3 | 4 | scms: 5 | default: 6 | kind: github 7 | spec: 8 | user: "{{ .github.user }}" 9 | email: "{{ .github.email }}" 10 | owner: "{{ .github.owner }}" 11 | repository: "{{ .github.repository }}" 12 | token: "{{ requiredEnv .github.token }}" 13 | username: "{{ .github.username }}" 14 | branch: "{{ .github.branch }}" 15 | 16 | sources: 17 | currentEndDate: 18 | name: Get current `end_date` date 19 | kind: hcl 20 | spec: 21 | file: packer-resources.tf 22 | path: resource.azuread_application_password.packer.end_date 23 | nextEndDate: 24 | name: Prepare next `end_date` date within 3 months 25 | kind: shell 26 | spec: 27 | command: bash ./updatecli/scripts/dateadd.sh 28 | environments: 29 | - name: PATH 30 | 31 | conditions: 32 | checkIfEndDateSoonExpired: 33 | kind: shell 34 | sourceid: currentEndDate 35 | spec: 36 | command: bash ./updatecli/scripts/datediff.sh 37 | environments: 38 | - name: PATH 39 | 40 | targets: 41 | updateNextEndDate: 42 | name: Update Terraform file `packer-resources.tf` with new expiration date 43 | kind: hcl 44 | sourceid: nextEndDate 45 | spec: 46 | file: packer-resources.tf 47 | path: resource.azuread_application_password.packer.end_date 48 | scmid: default 49 | 50 | actions: 51 | default: 52 | kind: github/pullrequest 53 | scmid: default 54 | spec: 55 | title: 'Extend Azure AD Application password validity for packer-resources (current end date: {{ source "currentEndDate" }})' 56 | description: | 57 | This PR generates a new Azure AD application password with a new end date for the packer-resources application (to allow building VM images). 58 | Once this PR is merged and deployed with success by Terraform (on infra.ci.jenkins.io), 59 | you can retrieve the new password value from the Terraform state with `terraform show -json` 60 | then searching for the new password in `values.value` of the `resource.azuread_application_password.packer` section (do NOT save it anywhere!) 61 | and (manually) update the packer credential as needed. 62 | Finally, verify that the new credential works by running a test Packer build. 63 | labels: 64 | - azure-ad-application 65 | - end-dates 66 | - packer-resources 67 | -------------------------------------------------------------------------------- /data-storage-jenkins-io.tf: -------------------------------------------------------------------------------- 1 | # Storage account 2 | resource "azurerm_resource_group" "data_storage_jenkins_io" { 3 | name = "data-storage" 4 | location = var.location 5 | tags = local.default_tags 6 | } 7 | 8 | resource "azurerm_storage_account" "data_storage_jenkins_io" { 9 | name = "datastoragejenkinsio" 10 | resource_group_name = azurerm_resource_group.data_storage_jenkins_io.name 11 | location = azurerm_resource_group.data_storage_jenkins_io.location 12 | 13 | account_tier = "Premium" 14 | account_kind = "FileStorage" 15 | access_tier = "Hot" 16 | account_replication_type = "ZRS" 17 | min_tls_version = "TLS1_2" # default value, needed for tfsec 18 | infrastructure_encryption_enabled = true 19 | # Disabled for NFS - https://learn.microsoft.com/en-us/azure/storage/common/storage-require-secure-transfer?toc=%2Fazure%2Fstorage%2Ffiles%2Ftoc.json 20 | https_traffic_only_enabled = false 21 | 22 | tags = local.default_tags 23 | 24 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 25 | network_rules { 26 | default_action = "Deny" 27 | # Only NFS share means only private network access - https://learn.microsoft.com/en-us/azure/storage/files/files-nfs-protocol#security-and-networking 28 | virtual_network_subnet_ids = concat( 29 | [ 30 | # Required for using the resource 31 | data.azurerm_subnet.publick8s.id, 32 | # Allows release.ci.jenkins.io agents to access the mount 33 | data.azurerm_subnet.privatek8s_release_tier.id, 34 | ], 35 | # Required for managing the resource 36 | local.app_subnets["infra.ci.jenkins.io"].agents, 37 | # Required for populating the resource 38 | local.app_subnets["trusted.ci.jenkins.io"].agents, 39 | ) 40 | bypass = ["Metrics", "Logging", "AzureServices"] 41 | } 42 | } 43 | # This storage account is expected to replace both "data_storage_jenkins_io_content" and "data_storage_jenkins_io_redirects" 44 | resource "azurerm_storage_share" "data_storage_jenkins_io" { 45 | name = "data-storage-jenkins-io" 46 | storage_account_id = azurerm_storage_account.data_storage_jenkins_io.id 47 | quota = 750 # Minimum size of premium is 100 - https://learn.microsoft.com/en-us/azure/storage/files/understanding-billing#provisioning-method 48 | enabled_protocol = "NFS" # Require a Premium Storage Account 49 | } 50 | -------------------------------------------------------------------------------- /providers.tf: -------------------------------------------------------------------------------- 1 | # Configure the Microsoft Azure Provider 2 | provider "azurerm" { 3 | subscription_id = "dff2ec18-6a8e-405c-8e45-b7df7465acf0" 4 | resource_provider_registrations = "none" 5 | features {} 6 | } 7 | 8 | provider "kubernetes" { 9 | alias = "privatek8s" 10 | host = local.aks_clusters_outputs.privatek8s.cluster_hostname 11 | client_certificate = base64decode(azurerm_kubernetes_cluster.privatek8s.kube_config.0.client_certificate) 12 | client_key = base64decode(azurerm_kubernetes_cluster.privatek8s.kube_config.0.client_key) 13 | cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.privatek8s.kube_config.0.cluster_ca_certificate) 14 | } 15 | 16 | provider "kubernetes" { 17 | alias = "publick8s" 18 | host = local.aks_clusters_outputs.publick8s.cluster_hostname 19 | client_certificate = base64decode(azurerm_kubernetes_cluster.publick8s.kube_config.0.client_certificate) 20 | client_key = base64decode(azurerm_kubernetes_cluster.publick8s.kube_config.0.client_key) 21 | cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.publick8s.kube_config.0.cluster_ca_certificate) 22 | } 23 | 24 | provider "kubernetes" { 25 | alias = "infracijenkinsio_agents_2" 26 | host = local.aks_clusters_outputs.infracijenkinsio_agents_2.cluster_hostname 27 | client_certificate = base64decode(azurerm_kubernetes_cluster.infracijenkinsio_agents_2.kube_config.0.client_certificate) 28 | client_key = base64decode(azurerm_kubernetes_cluster.infracijenkinsio_agents_2.kube_config.0.client_key) 29 | cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.infracijenkinsio_agents_2.kube_config.0.cluster_ca_certificate) 30 | } 31 | 32 | provider "postgresql" { 33 | /** 34 | Reaching this DB requires: 35 | - VPN access (with proper routing) 36 | - The following line added in your `/etc/hosts` as there are no public DNS: `10.253.0.4 public-db.postgres.database.azure.com` 37 | **/ 38 | host = azurerm_postgresql_flexible_server.public_db.fqdn 39 | username = local.public_db_pgsql_admin_login 40 | password = random_password.public_db_pgsql_admin_password.result 41 | superuser = false 42 | } 43 | 44 | provider "mysql" { 45 | /** 46 | Reaching this DB requires: 47 | - VPN access (with proper routing) 48 | - The following line added in your `/etc/hosts` as there are no public DNS: `10.253.1.4 public-db-mysql.mysql.database.azure.com` 49 | **/ 50 | endpoint = "${azurerm_mysql_flexible_server.public_db_mysql.fqdn}:3306" 51 | username = local.public_db_mysql_admin_login 52 | password = random_password.public_db_mysql_admin_password.result 53 | tls = true # Mandatory for Azure MySQL Flexible instances 54 | } 55 | -------------------------------------------------------------------------------- /updatecli/updatecli.d/fs-sp-writer-end-dates_infra.ci.jenkins.io.tf.tpl: -------------------------------------------------------------------------------- 1 | {{ range $key, $val := .end_dates.infra_ci_jenkins_io }} 2 | --- 3 | # yamllint disable rule:line-length 4 | name: "Generate new end date for {{ $val.service }} File Share service principal writer on infra.ci.jenkins.io" 5 | 6 | scms: 7 | default: 8 | kind: github 9 | spec: 10 | user: "{{ $.github.user }}" 11 | email: "{{ $.github.email }}" 12 | owner: "{{ $.github.owner }}" 13 | repository: "{{ $.github.repository }}" 14 | token: "{{ requiredEnv $.github.token }}" 15 | username: "{{ $.github.username }}" 16 | branch: "{{ $.github.branch }}" 17 | 18 | sources: 19 | currentEndDate: 20 | name: Get current `end_date` date 21 | kind: yaml 22 | spec: 23 | file: updatecli/values.yaml 24 | key: $.end_dates.infra_ci_jenkins_io.{{ $key }}.end_date 25 | nextEndDate: 26 | name: Prepare next `end_date` date within 3 months 27 | kind: shell 28 | spec: 29 | command: bash ./updatecli/scripts/dateadd.sh 30 | environments: 31 | - name: PATH 32 | 33 | conditions: 34 | checkIfEndDateSoonExpired: 35 | kind: shell 36 | sourceid: currentEndDate 37 | spec: 38 | # Current end_date date value passed as argument 39 | command: bash ./updatecli/scripts/datediff.sh 40 | environments: 41 | - name: PATH 42 | 43 | targets: 44 | updateNextEndDate: 45 | name: 'New end date for `{{ $val.service }}` File Share service principal writer on `infra.ci.jenkins.io` (current: {{ source "currentEndDate" }})' 46 | kind: yaml 47 | sourceid: nextEndDate 48 | spec: 49 | file: updatecli/values.yaml 50 | key: $.end_dates.infra_ci_jenkins_io.{{ $key }}.end_date 51 | scmid: default 52 | 53 | actions: 54 | default: 55 | kind: github/pullrequest 56 | scmid: default 57 | spec: 58 | title: 'New end date for `{{ $val.service }}` File Share service principal writer on `infra.ci.jenkins.io` (current: {{ source "currentEndDate" }})' 59 | description: | 60 | This PR updates the end date of {{ $val.service }} File Share service principal writer on infra.ci.jenkins.io. 61 | 62 | The current end date is set to `{{ $val.end_date }}`. 63 | 64 | After merging this PR, a new password will be generated. 65 | 66 | > [!IMPORTANT] 67 | > You'll have to ensure that `{{ $val.secret }}` is updated with this new password 68 | > in https://github.com/jenkins-infra/charts-secrets/blob/main/config/infra.ci.jenkins.io/jenkins-secrets.yaml. 69 | 70 | If you don't, the build of {{ $val.service }} on infra.ci.jenkins.io won't be able to update the website content anymore. 71 | labels: 72 | - terraform 73 | - "{{ $val.service }}" 74 | - end-dates 75 | - infra.ci.jenkins.io 76 | {{ end }} 77 | -------------------------------------------------------------------------------- /README.adoc: -------------------------------------------------------------------------------- 1 | = Jenkins infra on Azure 2 | :tip-caption: :bulb: 3 | :note-caption: :information_source: 4 | :important-caption: :heavy_exclamation_mark: 5 | :caution-caption: :fire: 6 | :warning-caption: :warning: 7 | :toc: 8 | :private_repo_name: terraform-states 9 | :private_repo_url: https://github.com/jenkins-infra/{private_repo_name} 10 | 11 | This repository hosts the infrastructure-as-code definition for all the link:https://azure.com/[Azure hosted] resources for the link:https://www.jenkins.io/projects/infrastructure/[Jenkins Infrastructure Project]. 12 | 13 | See also https://github.com/jenkins-infra/azure-net for all global network related resources. 14 | 15 | == Requirements 16 | 17 | In order to use this repository to provision the Jenkins infrastructure on azure, you need: 18 | 19 | * An `Azure` account 20 | * The requirements (of the shared tools) listed at link:https://github.com/jenkins-infra/shared-tools/tree/main/terraform#requirements[shared-tools/terraform#requirements] 21 | * The link:https://developer.hashicorp.com/terraform/language/settings/backends/azurerm[Terraform AzureRM Backend Configuration] on a local file named `backend-config`: 22 | ** The content can be retrieved from the outputs of the link:{private_repo_url}[(private) repository {private_repo_name}] 23 | ** This file (`backend-config`) is git-ignored 24 | 25 | * The git command line to allow cloning the repository and its submodule link:https://github.com/jenkins-infra/shared-tools[shared-tools] 26 | ** This repository has submodules. Once you cloned the repository, execute the following command to obtain the shared tools: 27 | 28 | [source,bash] 29 | ---- 30 | git submodule update --init --recursive 31 | ---- 32 | 33 | * Routing to database (private endpoints and private DNSes): 34 | ** VPN access is required with routing to the database subnets set up to your user 35 | ** As there are no public DNS, set up your local `/etc/hosts` (check the `providers.tf` for details) 36 | 37 | == HowTo 38 | 39 | === Provision 40 | 41 | IMPORTANT: Don't blindly execute the terraform code located in this repository on your own account as it may lead your account bill to significantly increase. 42 | 43 | 44 | Once you've fulfilled the <>, you may execute any command from https://github.com/jenkins-infra/shared-tools/blob/main/terraform/README.adoc#available-commands by adding the correct flag `--directory` pointing to `.shared-tools/terraform/`: 45 | 46 | [source,bash] 47 | ---- 48 | make --directory=.shared-tools/terraform help 49 | make --directory=.shared-tools/terraform lint 50 | # ... 51 | ---- 52 | 53 | 54 | A usual change to this repository looks like the following: 55 | 56 | * Fork the repository and clone it locally 57 | * Follow the <> steps to obtain the shared tools 58 | * Start by running a full `make --directory=.shared-tools/terraform validate` command to ensure that you work on a sane base (should generate a report TXT file with no changes to be applied) 59 | * Edit the Terraform project files 60 | * Run the command `make --directory=.shared-tools/terraform validate` again to ensure that your changes are OK 61 | * Commit, push and open a pull request to let the Jenkins pipeline run the test + plan (as per https://github.com/jenkins-infra/shared-tools/blob/main/terraform/README.adoc#jenkins-pipeline) 62 | -------------------------------------------------------------------------------- /postgres-public-db.tf: -------------------------------------------------------------------------------- 1 | # NOTE: managing DB resources requires routes to database (private endpoints and private DNSes): 2 | # * Either: 3 | # ** VPN access is required with routing to the database subnets set up to your user, 4 | # ** OR running terraform in a subnet with a private endpoint access/routing to the DB subnet 5 | # * Also, as there are no public DNS, either: 6 | # ** Set up your local `/etc/hosts` (check the `providers.tf` for details), 7 | # ** OR have your subnet set up to use the private DNS records 8 | 9 | ###### 10 | # Dedicated subnet is reserved as "delegated" for the pgsql server on the public network 11 | # Ref. https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-networking 12 | # Defined in https://github.com/jenkins-infra/azure-net/blob/main/vnets.tf 13 | data "azurerm_subnet" "public_db_vnet_postgres_tier" { 14 | name = "${data.azurerm_virtual_network.public_db.name}-postgres-tier" 15 | virtual_network_name = data.azurerm_virtual_network.public_db.name 16 | resource_group_name = data.azurerm_resource_group.public.name 17 | } 18 | resource "azurerm_network_security_group" "db_pgsql_tier" { 19 | name = "${data.azurerm_virtual_network.public_db.name}-postgres" 20 | location = var.location 21 | resource_group_name = data.azurerm_resource_group.public.name 22 | } 23 | resource "azurerm_subnet_network_security_group_association" "db_pgsql_tier" { 24 | subnet_id = data.azurerm_subnet.public_db_vnet_postgres_tier.id 25 | network_security_group_id = azurerm_network_security_group.db_pgsql_tier.id 26 | } 27 | # Used by 'local.public_db_pgsql_admin_login' (which is itself needed by the postgres provider) 28 | resource "random_password" "public_db_pgsql_admin_login" { 29 | length = 14 30 | special = false 31 | upper = false 32 | } 33 | resource "random_password" "public_db_pgsql_admin_password" { 34 | length = 24 35 | } 36 | resource "azurerm_postgresql_flexible_server" "public_db" { 37 | name = "public-db" 38 | resource_group_name = data.azurerm_resource_group.public.name 39 | location = var.location 40 | public_network_access_enabled = false 41 | administrator_login = local.public_db_pgsql_admin_login 42 | administrator_password = random_password.public_db_pgsql_admin_password.result 43 | sku_name = "B_Standard_B1ms" # 1vCore / 2 Gb - https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-b-series-burstable 44 | storage_mb = "262144" 45 | storage_tier = "P15" 46 | version = "13" 47 | zone = "1" 48 | private_dns_zone_id = azurerm_private_dns_zone.public_db_pgsql.id 49 | delegated_subnet_id = data.azurerm_subnet.public_db_vnet_postgres_tier.id 50 | 51 | depends_on = [ 52 | /** 53 | The network link from private pod is required to allow the provider "postgresql" 54 | to connect to this server from the private Jenkins agents where terraform runs 55 | (or through VPN tunnelling) 56 | **/ 57 | azurerm_private_dns_zone_virtual_network_link.public_db_pgsql["private-vnet"], 58 | azurerm_private_dns_zone_virtual_network_link.public_db_pgsql["infracijenkinsio-vnet"], 59 | ] 60 | } 61 | resource "azurerm_private_dns_zone" "public_db_pgsql" { 62 | name = "public-db-pgsql.jenkins-infra.postgres.database.azure.com" 63 | resource_group_name = data.azurerm_resource_group.public.name 64 | } 65 | resource "azurerm_private_dns_zone_virtual_network_link" "public_db_pgsql" { 66 | for_each = { 67 | "public-vnet" = data.azurerm_virtual_network.public.id, 68 | "publicdb-vnet" = data.azurerm_virtual_network.public_db.id, 69 | "private-vnet" = data.azurerm_virtual_network.private.id, 70 | "infracijenkinsio-vnet" = data.azurerm_virtual_network.infra_ci_jenkins_io.id, 71 | } 72 | name = "${each.key}-to-publicdbpgsql" 73 | resource_group_name = data.azurerm_resource_group.public.name 74 | private_dns_zone_name = azurerm_private_dns_zone.public_db_pgsql.name 75 | virtual_network_id = each.value 76 | } 77 | -------------------------------------------------------------------------------- /release.ci.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "release_ci_jenkins_io_controller" { 2 | name = "release-ci-jenkins-io-controller" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | resource "azurerm_user_assigned_identity" "release_ci_jenkins_io_controller" { 7 | location = azurerm_resource_group.release_ci_jenkins_io_controller.location 8 | name = "releasecijenkinsiocontroller" 9 | resource_group_name = azurerm_resource_group.release_ci_jenkins_io_controller.name 10 | } 11 | resource "azurerm_managed_disk" "release_ci_jenkins_io_data" { 12 | name = "release-ci-jenkins-io-data" 13 | location = azurerm_resource_group.release_ci_jenkins_io_controller.location 14 | resource_group_name = azurerm_resource_group.release_ci_jenkins_io_controller.name 15 | storage_account_type = "StandardSSD_ZRS" 16 | create_option = "Empty" 17 | disk_size_gb = 64 18 | tags = local.default_tags 19 | } 20 | # Required to allow AKS CSI driver to access the Azure disk 21 | resource "azurerm_role_definition" "release_ci_jenkins_io_controller_disk_reader" { 22 | name = "ReadReleaseCIDisk" 23 | scope = azurerm_resource_group.release_ci_jenkins_io_controller.id 24 | 25 | permissions { 26 | actions = [ 27 | "Microsoft.Compute/disks/read", 28 | "Microsoft.Compute/disks/write", 29 | ] 30 | } 31 | } 32 | resource "azurerm_role_assignment" "release_ci_jenkins_io_controller_disk_reader" { 33 | scope = azurerm_resource_group.release_ci_jenkins_io_controller.id 34 | role_definition_id = azurerm_role_definition.release_ci_jenkins_io_controller_disk_reader.role_definition_resource_id 35 | principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id 36 | } 37 | resource "azurerm_user_assigned_identity" "release_ci_jenkins_io_agents" { 38 | location = var.location 39 | name = "release-ci-jenkins-io-agents" 40 | resource_group_name = azurerm_kubernetes_cluster.privatek8s.resource_group_name 41 | } 42 | resource "azurerm_role_assignment" "release_ci_jenkins_io_azurevm_agents_write_buildsreports_share" { 43 | scope = azurerm_storage_account.builds_reports_jenkins_io.id 44 | # Allow writing 45 | role_definition_name = "Storage File Data Privileged Contributor" 46 | principal_id = azurerm_user_assigned_identity.release_ci_jenkins_io_agents.principal_id 47 | } 48 | 49 | resource "azurerm_resource_group" "prodreleasecore" { 50 | name = "prodreleasecore" 51 | location = var.location 52 | tags = local.default_tags 53 | } 54 | resource "azurerm_key_vault" "prodreleasecore" { 55 | tenant_id = data.azurerm_client_config.current.tenant_id 56 | name = "prodreleasecore" 57 | location = var.location 58 | resource_group_name = azurerm_resource_group.prodreleasecore.name 59 | sku_name = "standard" 60 | 61 | enabled_for_disk_encryption = false 62 | soft_delete_retention_days = 90 63 | purge_protection_enabled = false 64 | rbac_authorization_enabled = false 65 | enabled_for_deployment = false 66 | enabled_for_template_deployment = false 67 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 68 | public_network_access_enabled = true 69 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 70 | network_acls { 71 | bypass = "AzureServices" 72 | default_action = "Deny" 73 | virtual_network_subnet_ids = local.app_subnets["release.ci.jenkins.io"].agents 74 | } 75 | 76 | # releasecore Entra Application 77 | access_policy { 78 | tenant_id = data.azurerm_client_config.current.tenant_id 79 | object_id = "b6d73004-673f-4099-aa80-30e6e9dae314" 80 | 81 | certificate_permissions = [ 82 | "Get", 83 | "List", 84 | "GetIssuers", 85 | "ListIssuers", 86 | ] 87 | 88 | key_permissions = [ 89 | "Get", 90 | "List", 91 | "Decrypt", 92 | "Verify", 93 | "Encrypt", 94 | ] 95 | secret_permissions = [ 96 | "Get", 97 | "List", 98 | ] 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /mysql-public-db.tf: -------------------------------------------------------------------------------- 1 | # NOTE: managing DB resources requires routes to database (private endpoints and private DNSes): 2 | # * Either: 3 | # ** VPN access is required with routing to the database subnets set up to your user, 4 | # ** OR running terraform in a subnet with a private endpoint access/routing to the DB subnet 5 | # * Also, as there are no public DNS, either: 6 | # ** Set up your local `/etc/hosts` (check the `providers.tf` for details), 7 | # ** OR have your subnet set up to use the private DNS records 8 | ###### 9 | # Dedicated subnet is reserved as "delegated" for the mysql server on the public network 10 | # Defined in https://github.com/jenkins-infra/azure-net/blob/main/vnets.tf 11 | data "azurerm_subnet" "public_db_vnet_mysql_tier" { 12 | name = "${data.azurerm_virtual_network.public_db.name}-mysql-tier" 13 | virtual_network_name = data.azurerm_virtual_network.public_db.name 14 | resource_group_name = data.azurerm_resource_group.public.name 15 | } 16 | resource "azurerm_network_security_group" "db_mysql_tier" { 17 | name = "${data.azurerm_virtual_network.public_db.name}-mysql" 18 | location = var.location 19 | resource_group_name = data.azurerm_resource_group.public.name 20 | } 21 | resource "azurerm_subnet_network_security_group_association" "db_mysql_tier" { 22 | subnet_id = data.azurerm_subnet.public_db_vnet_mysql_tier.id 23 | network_security_group_id = azurerm_network_security_group.db_mysql_tier.id 24 | } 25 | # Used by 'local.public_db_mysql_admin_login' (which is itself needed by the mysql provider) 26 | resource "random_password" "public_db_mysql_admin_login" { 27 | length = 14 28 | special = false 29 | upper = false 30 | } 31 | # Generate random value for the login password see https://learn.microsoft.com/en-us/azure/mysql/flexible-server/quickstart-create-terraform?tabs=azure-cli#implement-the-terraform-code 32 | resource "random_password" "public_db_mysql_admin_password" { 33 | length = 8 34 | lower = true 35 | min_lower = 1 36 | min_numeric = 1 37 | min_special = 1 38 | min_upper = 1 39 | numeric = true 40 | override_special = "_" 41 | special = true 42 | upper = true 43 | } 44 | 45 | # Manages the MySQL Flexible Server https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/mysql_flexible_server 46 | resource "azurerm_mysql_flexible_server" "public_db_mysql" { 47 | name = "public-db-mysql" 48 | resource_group_name = data.azurerm_resource_group.public.name 49 | location = var.location 50 | administrator_login = local.public_db_mysql_admin_login 51 | administrator_password = random_password.public_db_mysql_admin_password.result 52 | sku_name = "B_Standard_B1ms" # 1vCore / 2 Gb - https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-b-series-burstable 53 | version = "8.0.21" 54 | zone = "1" 55 | geo_redundant_backup_enabled = false 56 | private_dns_zone_id = azurerm_private_dns_zone.public_db_mysql.id 57 | delegated_subnet_id = data.azurerm_subnet.public_db_vnet_mysql_tier.id 58 | 59 | depends_on = [ 60 | /** 61 | The network link from private pod is required to allow the provider "mysql" 62 | to connect to this server from the private Jenkins agents where terraform runs 63 | (or through VPN tunnelling) 64 | **/ 65 | azurerm_private_dns_zone_virtual_network_link.public_db_mysql["private-vnet"], 66 | azurerm_private_dns_zone_virtual_network_link.public_db_mysql["infracijenkinsio-vnet"], 67 | ] 68 | } 69 | 70 | # Enables you to manage Private DNS zones within Azure DNS 71 | resource "azurerm_private_dns_zone" "public_db_mysql" { 72 | name = "public-db-mysql.jenkins-infra.mysql.database.azure.com" 73 | resource_group_name = data.azurerm_resource_group.public.name 74 | } 75 | 76 | # Enables you to manage Private DNS zone Virtual Network Links 77 | resource "azurerm_private_dns_zone_virtual_network_link" "public_db_mysql" { 78 | for_each = { 79 | "public-vnet" = data.azurerm_virtual_network.public.id, 80 | "publicdb-vnet" = data.azurerm_virtual_network.public_db.id, 81 | "private-vnet" = data.azurerm_virtual_network.private.id, 82 | "infracijenkinsio-vnet" = data.azurerm_virtual_network.infra_ci_jenkins_io.id, 83 | } 84 | name = "${each.key}-to-publicdbmysql" 85 | resource_group_name = data.azurerm_resource_group.public.name 86 | private_dns_zone_name = azurerm_private_dns_zone.public_db_mysql.name 87 | virtual_network_id = each.value 88 | } 89 | -------------------------------------------------------------------------------- /packer-resources.tf: -------------------------------------------------------------------------------- 1 | # Azure Resources required or used by the repository jenkins-infra/packer-images 2 | 3 | resource "azuread_application" "packer" { 4 | display_name = "packer" 5 | owners = [ 6 | data.azuread_service_principal.terraform_production.object_id, # terraform-production Service Principal, used by the CI system 7 | ] 8 | tags = [for key, value in local.default_tags : "${key}:${value}"] 9 | required_resource_access { 10 | resource_app_id = "00000003-0000-0000-c000-000000000000" # Microsoft Graph 11 | 12 | resource_access { 13 | id = "e1fe6dd8-ba31-4d61-89e7-88639da4683d" # User.Read 14 | type = "Scope" 15 | } 16 | } 17 | 18 | web { 19 | homepage_url = "https://github.com/jenkins-infra/azure" 20 | } 21 | } 22 | 23 | resource "azuread_service_principal" "packer" { 24 | client_id = azuread_application.packer.client_id 25 | app_role_assignment_required = false 26 | owners = [ 27 | data.azuread_service_principal.terraform_production.object_id, # terraform-production Service Principal, used by the CI system 28 | ] 29 | } 30 | 31 | resource "azuread_application_password" "packer" { 32 | display_name = "packer-tf-managed" 33 | application_id = azuread_application.packer.id 34 | end_date = "2026-05-01T00:00:00Z" 35 | } 36 | 37 | ## Dev Resources are used by the pull requests in jenkins-infra/packer-images 38 | ## Staging Resources are used by the "main" branch builds 39 | ## Prod Resources are used for final Packer artifacts 40 | resource "azurerm_resource_group" "packer_images_cdf" { 41 | for_each = local.shared_galleries 42 | 43 | name = "${each.key}-packer-images" 44 | location = data.azurerm_virtual_network.infra_ci_jenkins_io.location # Location of the packer subnet in infra.ci 45 | } 46 | 47 | resource "azurerm_resource_group" "packer_builds_cdf" { 48 | for_each = local.shared_galleries 49 | 50 | name = "${each.key}-packer-builds" 51 | location = data.azurerm_virtual_network.infra_ci_jenkins_io.location # Location of the packer subnet in infra.ci 52 | } 53 | 54 | resource "azurerm_shared_image_gallery" "packer_images_cdf" { 55 | for_each = local.shared_galleries 56 | 57 | name = "${each.key}_packer_images" 58 | resource_group_name = azurerm_resource_group.packer_images_cdf[each.key].name 59 | location = data.azurerm_virtual_network.infra_ci_jenkins_io.location # Location of the packer subnet in infra.ci 60 | description = each.value.description 61 | 62 | tags = { 63 | scope = "terraform-managed" 64 | } 65 | } 66 | 67 | # Note that Terraform does NOT manage image versions (it's packer-based). 68 | resource "azurerm_shared_image" "jenkins_agent_images_cdf" { 69 | # Generate a list of images in the form "_" 70 | for_each = toset( 71 | distinct( 72 | flatten([ 73 | for gallery_key, gallery_value in local.shared_galleries : [ 74 | for image_key in gallery_value.images : "${gallery_key}_${image_key}" 75 | ] 76 | ]) 77 | ) 78 | ) 79 | 80 | name = format("jenkins-agent-%s", split("_", each.value)[1]) 81 | gallery_name = azurerm_shared_image_gallery.packer_images_cdf[split("_", each.value)[0]].name 82 | resource_group_name = azurerm_resource_group.packer_images_cdf[split("_", each.value)[0]].name 83 | location = data.azurerm_virtual_network.infra_ci_jenkins_io.location # Location of the packer subnet in infra.ci 84 | 85 | architecture = length(regexall(".+arm64", split("_", each.value)[1])) > 0 ? "Arm64" : "x64" 86 | 87 | hyper_v_generation = "V2" 88 | os_type = length(regexall(".*windows.*", lower(split("_", each.value)[1]))) > 0 ? "Windows" : "Linux" 89 | specialized = false 90 | trusted_launch_enabled = false 91 | 92 | lifecycle { 93 | ignore_changes = [ 94 | eula, accelerated_network_support_enabled 95 | ] 96 | } 97 | 98 | identifier { 99 | publisher = format("jenkins-agent-%s", split("_", each.value)[1]) 100 | offer = format("jenkins-agent-%s", split("_", each.value)[1]) 101 | sku = format("jenkins-agent-%s", split("_", each.value)[1]) 102 | } 103 | 104 | tags = { 105 | scope = "terraform-managed" 106 | } 107 | } 108 | 109 | # Allow packer Service Principal to manage AzureRM resources inside the packer resource groups 110 | resource "azurerm_role_assignment" "packer_role_images_assignement_cdf" { 111 | for_each = azurerm_resource_group.packer_images_cdf 112 | 113 | scope = each.value.id 114 | role_definition_name = "Contributor" 115 | principal_id = azuread_service_principal.packer.object_id 116 | } 117 | # Allow packer Service Principal to manage AzureRM resources inside the packer resource groups 118 | resource "azurerm_role_assignment" "packer_role_builds_assignement_cdf" { 119 | for_each = azurerm_resource_group.packer_builds_cdf 120 | 121 | scope = each.value.id 122 | role_definition_name = "Contributor" 123 | principal_id = azuread_service_principal.packer.object_id 124 | } 125 | resource "azurerm_role_assignment" "packer_role_manage_subnet_cdf" { 126 | scope = data.azurerm_subnet.infra_ci_jenkins_io_packer_builds.id 127 | role_definition_name = "Network Contributor" 128 | principal_id = azuread_service_principal.packer.object_id 129 | } 130 | -------------------------------------------------------------------------------- /vnets.tf: -------------------------------------------------------------------------------- 1 | # The resources groups and virtual networks below are defined here: 2 | # https://github.com/jenkins-infra/azure-net/blob/main/vnets.tf 3 | 4 | ################################################################################ 5 | ## Resource Groups 6 | ################################################################################ 7 | # Defined in https://github.com/jenkins-infra/azure-net/blob/main/vnets.tf 8 | data "azurerm_resource_group" "public" { 9 | name = "public" 10 | } 11 | data "azurerm_resource_group" "private" { 12 | name = "private" 13 | } 14 | data "azurerm_resource_group" "infra_ci_jenkins_io" { 15 | name = "infra-ci-jenkins-io" 16 | } 17 | data "azurerm_resource_group" "cert_ci_jenkins_io" { 18 | name = "cert-ci-jenkins-io" 19 | } 20 | data "azurerm_resource_group" "trusted_ci_jenkins_io" { 21 | name = "trusted-ci-jenkins-io" 22 | } 23 | 24 | ################################################################################ 25 | ## Virtual Networks 26 | ################################################################################ 27 | # Defined in https://github.com/jenkins-infra/azure-net/blob/main/vnets.tf 28 | data "azurerm_virtual_network" "public" { 29 | name = "${data.azurerm_resource_group.public.name}-vnet" 30 | resource_group_name = data.azurerm_resource_group.public.name 31 | } 32 | data "azurerm_virtual_network" "private" { 33 | name = "${data.azurerm_resource_group.private.name}-vnet" 34 | resource_group_name = data.azurerm_resource_group.private.name 35 | } 36 | # Reference to the PostgreSQL/MySql dedicated network external resources 37 | data "azurerm_virtual_network" "public_db" { 38 | name = "${data.azurerm_resource_group.public.name}-db-vnet" 39 | resource_group_name = data.azurerm_resource_group.public.name 40 | } 41 | data "azurerm_virtual_network" "infra_ci_jenkins_io" { 42 | name = "${data.azurerm_resource_group.infra_ci_jenkins_io.name}-vnet" 43 | resource_group_name = data.azurerm_resource_group.infra_ci_jenkins_io.name 44 | } 45 | data "azurerm_virtual_network" "cert_ci_jenkins_io" { 46 | name = "${data.azurerm_resource_group.cert_ci_jenkins_io.name}-vnet" 47 | resource_group_name = data.azurerm_resource_group.cert_ci_jenkins_io.name 48 | } 49 | data "azurerm_virtual_network" "trusted_ci_jenkins_io" { 50 | name = "trusted-ci-jenkins-io-vnet" 51 | resource_group_name = data.azurerm_resource_group.trusted_ci_jenkins_io.name 52 | } 53 | 54 | ################################################################################ 55 | ## SUB NETWORKS 56 | ################################################################################ 57 | # Defined in https://github.com/jenkins-infra/azure-net/blob/main/vpn.tf 58 | data "azurerm_subnet" "private_vnet_data_tier" { 59 | name = "${data.azurerm_virtual_network.private.name}-data-tier" 60 | virtual_network_name = data.azurerm_virtual_network.private.name 61 | resource_group_name = data.azurerm_resource_group.private.name 62 | } 63 | data "azurerm_subnet" "infra_ci_jenkins_io_ephemeral_agents" { 64 | name = "${data.azurerm_virtual_network.infra_ci_jenkins_io.name}-ephemeral-agents" 65 | virtual_network_name = data.azurerm_virtual_network.infra_ci_jenkins_io.name 66 | resource_group_name = data.azurerm_virtual_network.infra_ci_jenkins_io.resource_group_name 67 | } 68 | data "azurerm_subnet" "infracijenkinsio_agents_2" { 69 | name = "${data.azurerm_virtual_network.infra_ci_jenkins_io.name}-kubernetes-agents" 70 | resource_group_name = data.azurerm_virtual_network.infra_ci_jenkins_io.resource_group_name 71 | virtual_network_name = data.azurerm_virtual_network.infra_ci_jenkins_io.name 72 | } 73 | data "azurerm_subnet" "cert_ci_jenkins_io_controller" { 74 | name = "${data.azurerm_virtual_network.cert_ci_jenkins_io.name}-controller" 75 | virtual_network_name = data.azurerm_virtual_network.cert_ci_jenkins_io.name 76 | resource_group_name = data.azurerm_virtual_network.cert_ci_jenkins_io.resource_group_name 77 | } 78 | data "azurerm_subnet" "cert_ci_jenkins_io_ephemeral_agents" { 79 | name = "${data.azurerm_virtual_network.cert_ci_jenkins_io.name}-ephemeral-agents" 80 | virtual_network_name = data.azurerm_virtual_network.cert_ci_jenkins_io.name 81 | resource_group_name = data.azurerm_virtual_network.cert_ci_jenkins_io.resource_group_name 82 | } 83 | data "azurerm_subnet" "trusted_ci_jenkins_io_controller" { 84 | name = "${data.azurerm_virtual_network.trusted_ci_jenkins_io.name}-controller" 85 | virtual_network_name = data.azurerm_virtual_network.trusted_ci_jenkins_io.name 86 | resource_group_name = data.azurerm_resource_group.trusted_ci_jenkins_io.name 87 | } 88 | data "azurerm_subnet" "trusted_ci_jenkins_io_permanent_agents" { 89 | name = "${data.azurerm_virtual_network.trusted_ci_jenkins_io.name}-permanent-agents" 90 | virtual_network_name = data.azurerm_virtual_network.trusted_ci_jenkins_io.name 91 | resource_group_name = data.azurerm_resource_group.trusted_ci_jenkins_io.name 92 | } 93 | data "azurerm_subnet" "trusted_ci_jenkins_io_ephemeral_agents" { 94 | name = "${data.azurerm_virtual_network.trusted_ci_jenkins_io.name}-ephemeral-agents" 95 | resource_group_name = data.azurerm_resource_group.trusted_ci_jenkins_io.name 96 | virtual_network_name = data.azurerm_virtual_network.trusted_ci_jenkins_io.name 97 | } 98 | data "azurerm_subnet" "infra_ci_jenkins_io_packer_builds" { 99 | name = "${data.azurerm_virtual_network.infra_ci_jenkins_io.name}-packer-builds" 100 | virtual_network_name = data.azurerm_virtual_network.infra_ci_jenkins_io.name 101 | resource_group_name = data.azurerm_virtual_network.infra_ci_jenkins_io.resource_group_name 102 | } 103 | data "azurerm_subnet" "privatek8s_tier" { 104 | name = "privatek8s-tier" 105 | resource_group_name = data.azurerm_resource_group.private.name 106 | virtual_network_name = data.azurerm_virtual_network.private.name 107 | } 108 | data "azurerm_subnet" "privatek8s_release_tier" { 109 | name = "privatek8s-release-tier" 110 | resource_group_name = data.azurerm_resource_group.private.name 111 | virtual_network_name = data.azurerm_virtual_network.private.name 112 | } 113 | data "azurerm_subnet" "privatek8s_infra_ci_controller_tier" { 114 | name = "privatek8s-infraci-ctrl-tier" 115 | resource_group_name = data.azurerm_resource_group.private.name 116 | virtual_network_name = data.azurerm_virtual_network.private.name 117 | } 118 | data "azurerm_subnet" "privatek8s_release_ci_controller_tier" { 119 | name = "privatek8s-releaseci-ctrl-tier" 120 | resource_group_name = data.azurerm_resource_group.private.name 121 | virtual_network_name = data.azurerm_virtual_network.private.name 122 | } 123 | -------------------------------------------------------------------------------- /agent.trusted.ci.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | #################################################################################### 2 | ## Resources for the permanent agent VM 3 | #################################################################################### 4 | resource "azurerm_resource_group" "permanent_agents_trusted_ci_jenkins_io" { 5 | name = "permanent-agents-trusted-ci-jenkins-io" 6 | location = var.location 7 | tags = local.default_tags 8 | } 9 | resource "azurerm_network_interface" "agent_trusted_ci_jenkins_io" { 10 | name = "agent-trusted-ci-jenkins-io" 11 | location = azurerm_resource_group.permanent_agents_trusted_ci_jenkins_io.location 12 | resource_group_name = azurerm_resource_group.permanent_agents_trusted_ci_jenkins_io.name 13 | tags = local.default_tags 14 | 15 | ip_configuration { 16 | name = "internal" 17 | subnet_id = data.azurerm_subnet.trusted_ci_jenkins_io_permanent_agents.id 18 | private_ip_address_allocation = "Dynamic" 19 | } 20 | } 21 | resource "azurerm_linux_virtual_machine" "agent_trusted_ci_jenkins_io" { 22 | name = "agent.trusted.ci.jenkins.io" 23 | resource_group_name = azurerm_resource_group.permanent_agents_trusted_ci_jenkins_io.name 24 | location = azurerm_resource_group.permanent_agents_trusted_ci_jenkins_io.location 25 | tags = local.default_tags 26 | size = "Standard_B2s" 27 | admin_username = local.admin_username 28 | zone = "1" # We need a zonale deployment to attach a Premium_SSD_v2 data disk 29 | disable_password_authentication = true 30 | network_interface_ids = [ 31 | azurerm_network_interface.agent_trusted_ci_jenkins_io.id, 32 | ] 33 | 34 | admin_ssh_key { 35 | username = local.admin_username 36 | public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC5K7Ro7jBl5Kc68RdzG6EXHstIBFSxO5Da8SQJSMeCbb4cHTYuBBH8jNsAFcnkN64kEu+YhmlxaWEVEIrPgfGfs13ZL7v9p+Nt76tsz6gnVdAy2zCz607pAWe7p4bBn6T9zdZcBSnvjawO+8t/5ue4ngcfAjanN5OsOgLeD6yqVyP8YTERjW78jvp2TFrIYmgWMI5ES1ln32PQmRZwc1eAOsyGJW/YIBdOxaSkZ41qUvb9b3dCorGuCovpSK2EeNphjLPpVX/NRpVY4YlDqAcTCdLdDrEeVqkiA/VDCYNhudZTDa8f1iHwBE/GEtlKmoO6dxJ5LAkRk3RIVHYrmI6XXSw5l0tHhW5D12MNwzUfDxQEzBpGK5iSfOBt5zJ5OiI9ftnsq/GV7vCXfvMVGDLUC551P5/s/wM70QmHwhlGQNLNeJxRTvd6tL11bof3K+29ivFYUmpU17iVxYOWhkNY86WyngHU6Ux0zaczF3H6H0tpg1Ca/cFO428AVPw/RTJpcAe6OVKq5zwARNApQ/p6fJKUAdXap+PpQGZlQhPLkUbwtFXGTrpX9ePTcdzryCYjgrZouvy4ZMzruJiIbFUH8mRY3xVREVaIsJakruvgw3b14oQgcB4BwYVBBqi62xIvbRzAv7Su9t2jK6OR2z3sM/hLJRqIJ5oILMORa7XqrQ== smerle@MacBook-Pro-de-Stephane.local" 37 | } 38 | 39 | user_data = base64encode( 40 | templatefile("./.shared-tools/terraform/cloudinit.tftpl", { 41 | hostname = "agent.trusted.ci.jenkins.io", 42 | admin_username = local.admin_username, 43 | } 44 | )) 45 | computer_name = "agent.trusted.ci.jenkins.io" 46 | 47 | # Encrypt all disks (ephemeral, temp dirs and data volumes) - https://learn.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal?tabs=azure-powershell 48 | encryption_at_host_enabled = true 49 | 50 | os_disk { 51 | caching = "ReadWrite" 52 | storage_account_type = "StandardSSD_LRS" 53 | disk_size_gb = "32" # Minimum size with Ubuntu base image 54 | } 55 | 56 | source_image_reference { 57 | publisher = "Canonical" 58 | offer = "0001-com-ubuntu-minimal-jammy" 59 | sku = "minimal-22_04-lts-gen2" 60 | version = "latest" 61 | } 62 | 63 | identity { 64 | type = "UserAssigned" 65 | identity_ids = [ 66 | azurerm_user_assigned_identity.trusted_ci_jenkins_io_azurevm_agents_jenkins.id, 67 | ] 68 | } 69 | } 70 | resource "azurerm_managed_disk" "agent_trusted_ci_jenkins_io_data" { 71 | name = "agent-trusted-ci-jenkins-io-data" 72 | location = azurerm_resource_group.permanent_agents_trusted_ci_jenkins_io.location 73 | resource_group_name = azurerm_resource_group.permanent_agents_trusted_ci_jenkins_io.name 74 | zone = azurerm_linux_virtual_machine.agent_trusted_ci_jenkins_io.zone 75 | storage_account_type = "PremiumV2_LRS" 76 | create_option = "Empty" 77 | disk_size_gb = "580" 78 | 79 | tags = local.default_tags 80 | } 81 | resource "azurerm_virtual_machine_data_disk_attachment" "agent_trusted_ci_jenkins_io_data" { 82 | managed_disk_id = azurerm_managed_disk.agent_trusted_ci_jenkins_io_data.id 83 | virtual_machine_id = azurerm_linux_virtual_machine.agent_trusted_ci_jenkins_io.id 84 | lun = "20" 85 | caching = "None" # Caching not supported with "PremiumV2_LRS" 86 | } 87 | 88 | #################################################################################### 89 | ## Network Security Group and rules 90 | #################################################################################### 91 | resource "azurerm_subnet_network_security_group_association" "trusted_ci_permanent_agent" { 92 | subnet_id = data.azurerm_subnet.trusted_ci_jenkins_io_permanent_agents.id 93 | network_security_group_id = module.trusted_ci_jenkins_io.controller_nsg_id 94 | } 95 | 96 | # Ignore the rule as it does not detect the IP restriction to only update.jenkins.io"s host 97 | #trivy:ignore:azure-network-no-public-egress 98 | resource "azurerm_network_security_rule" "allow_outbound_ssh_from_permanent_agent_to_pkg" { 99 | name = "allow-outbound-ssh-from-permanent-agent-to-pkg" 100 | priority = 4080 101 | direction = "Outbound" 102 | access = "Allow" 103 | protocol = "Tcp" 104 | source_port_range = "*" 105 | destination_port_range = "22" 106 | source_address_prefixes = [ 107 | azurerm_linux_virtual_machine.agent_trusted_ci_jenkins_io.private_ip_address, 108 | ] 109 | destination_address_prefix = local.external_services["pkg.origin.jenkins.io"] 110 | resource_group_name = module.trusted_ci_jenkins_io.controller_resourcegroup_name 111 | network_security_group_name = module.trusted_ci_jenkins_io.controller_nsg_name 112 | } 113 | 114 | resource "azurerm_network_security_rule" "allow_inbound_ssh_from_controller_to_permanent_agent" { 115 | name = "allow-inbound-ssh-from-controller-to-permanent-agent" 116 | priority = 3600 117 | direction = "Inbound" 118 | access = "Allow" 119 | protocol = "Tcp" 120 | source_port_range = "*" 121 | destination_port_range = "22" 122 | source_address_prefix = module.trusted_ci_jenkins_io.controller_private_ipv4 123 | destination_address_prefixes = [ 124 | azurerm_linux_virtual_machine.agent_trusted_ci_jenkins_io.private_ip_address, 125 | ] 126 | resource_group_name = module.trusted_ci_jenkins_io.controller_resourcegroup_name 127 | network_security_group_name = module.trusted_ci_jenkins_io.controller_nsg_name 128 | } 129 | 130 | #################################################################################### 131 | ## Public DNS records 132 | #################################################################################### 133 | resource "azurerm_dns_a_record" "trusted_permanent_agent" { 134 | name = "agent" 135 | zone_name = module.trusted_ci_jenkins_io_letsencrypt.zone_name 136 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 137 | ttl = 60 138 | records = [azurerm_linux_virtual_machine.agent_trusted_ci_jenkins_io.private_ip_address] 139 | } 140 | -------------------------------------------------------------------------------- /puppet.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "puppet_jenkins_io" { 2 | name = "puppet.jenkins.io" 3 | location = "East US 2" 4 | tags = local.default_tags 5 | } 6 | resource "azurerm_public_ip" "puppet_jenkins_io" { 7 | name = "puppet.jenkins.io" 8 | location = azurerm_resource_group.puppet_jenkins_io.location 9 | resource_group_name = azurerm_resource_group.puppet_jenkins_io.name 10 | allocation_method = "Static" 11 | sku = "Standard" 12 | tags = local.default_tags 13 | } 14 | resource "azurerm_management_lock" "puppet_jenkins_io_publicip" { 15 | name = "puppet.jenkins.io-publicip" 16 | scope = azurerm_public_ip.puppet_jenkins_io.id 17 | lock_level = "CanNotDelete" 18 | notes = "Locked because this is a sensitive resource that should not be removed" 19 | } 20 | # Defined in https://github.com/jenkins-infra/azure-net/tree/main/vnets.tf 21 | data "azurerm_subnet" "dmz" { 22 | name = "${data.azurerm_virtual_network.private.name}-dmz" 23 | resource_group_name = data.azurerm_resource_group.private.name 24 | virtual_network_name = data.azurerm_virtual_network.private.name 25 | } 26 | resource "azurerm_network_interface" "puppet_jenkins_io" { 27 | name = "puppet.jenkins.io" 28 | location = azurerm_resource_group.puppet_jenkins_io.location 29 | resource_group_name = azurerm_resource_group.puppet_jenkins_io.name 30 | tags = local.default_tags 31 | 32 | ip_configuration { 33 | name = "external" 34 | private_ip_address_allocation = "Dynamic" 35 | public_ip_address_id = azurerm_public_ip.puppet_jenkins_io.id 36 | subnet_id = data.azurerm_subnet.dmz.id 37 | } 38 | } 39 | data "azurerm_network_security_group" "private_dmz" { 40 | name = "${data.azurerm_virtual_network.private.name}-dmz" 41 | # location = data.azurerm_resource_group.private.name.location 42 | resource_group_name = data.azurerm_resource_group.private.name 43 | } 44 | ## Inbound Rules (different set of priorities than Outbound rules) ## 45 | #trivy:ignore:azure-network-no-public-ingress 46 | resource "azurerm_network_security_rule" "allow_inbound_webhooks_from_github_to_puppet" { 47 | name = "allow-inbound-webhooks-from-github-to-puppet" 48 | priority = 3999 49 | direction = "Inbound" 50 | access = "Allow" 51 | protocol = "Tcp" 52 | source_port_range = "*" 53 | # https://github.com/jenkins-infra/jenkins-infra/blob/51c90220ec19ed688a0605dce4a98eddd212844a/dist/profile/manifests/r10k.pp 54 | # https://forge.puppet.com/modules/puppet/r10k/readme 55 | destination_port_range = "8088" # r10k webhook default port 56 | source_address_prefixes = local.github_ips.webhooks 57 | destination_address_prefix = azurerm_linux_virtual_machine.puppet_jenkins_io.private_ip_address 58 | resource_group_name = data.azurerm_resource_group.private.name 59 | network_security_group_name = data.azurerm_network_security_group.private_dmz.name 60 | } 61 | resource "azurerm_network_security_rule" "allow_inbound_ssh_from_admins_to_puppet" { 62 | name = "allow-inbound-ssh-from-admins-to-puppet" 63 | priority = 4000 64 | direction = "Inbound" 65 | access = "Allow" 66 | protocol = "Tcp" 67 | source_port_range = "*" 68 | destination_port_range = "22" 69 | source_address_prefixes = flatten( 70 | concat( 71 | [for key, value in local.admin_public_ips : value] 72 | ) 73 | ) 74 | destination_address_prefix = azurerm_linux_virtual_machine.puppet_jenkins_io.private_ip_address 75 | resource_group_name = data.azurerm_resource_group.private.name 76 | network_security_group_name = data.azurerm_network_security_group.private_dmz.name 77 | } 78 | #trivy:ignore:azure-network-no-public-ingress 79 | resource "azurerm_network_security_rule" "allow_inbound_puppet_from_vms" { 80 | name = "allow-inbound-puppet-from-vms" 81 | priority = 4001 82 | direction = "Inbound" 83 | access = "Allow" 84 | protocol = "Tcp" 85 | source_port_range = "*" 86 | destination_port_range = "8140" 87 | source_address_prefix = "Internet" # TODO: restrict to only our VM outbound IPs 88 | destination_address_prefix = azurerm_linux_virtual_machine.puppet_jenkins_io.private_ip_address 89 | resource_group_name = data.azurerm_resource_group.private.name 90 | network_security_group_name = data.azurerm_network_security_group.private_dmz.name 91 | } 92 | ## Outbound Rules (different set of priorities than Inbound rules) ## 93 | ## Do not tag internet egress as a security issue (we want puppet master to access to internet) 94 | #trivy:ignore:avd-azu-0051 95 | resource "azurerm_network_security_rule" "allow_outbound_http_from_puppet_to_internet" { 96 | name = "allow-outbound-http-from-puppet-to-internet" 97 | priority = 4002 98 | direction = "Outbound" 99 | access = "Allow" 100 | protocol = "Tcp" 101 | source_port_range = "*" 102 | source_address_prefix = azurerm_linux_virtual_machine.puppet_jenkins_io.private_ip_address 103 | destination_port_ranges = ["80", "443"] 104 | destination_address_prefix = "Internet" 105 | resource_group_name = data.azurerm_resource_group.private.name 106 | network_security_group_name = data.azurerm_network_security_group.private_dmz.name 107 | } 108 | resource "azurerm_linux_virtual_machine" "puppet_jenkins_io" { 109 | name = "puppet.jenkins.io" 110 | resource_group_name = azurerm_resource_group.puppet_jenkins_io.name 111 | location = azurerm_resource_group.puppet_jenkins_io.location 112 | tags = local.default_tags 113 | size = "Standard_B2ms" 114 | admin_username = local.admin_username 115 | disable_password_authentication = true 116 | network_interface_ids = [ 117 | azurerm_network_interface.puppet_jenkins_io.id, 118 | ] 119 | 120 | admin_ssh_key { 121 | username = local.admin_username 122 | public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC54ZdYuBsHL4gtLA40hF55HdwB6g//lu5VOdpSaMP3z+dvQUUYGF+6CRxvmmr2j9+bxD/8+aJY8mBqQU2dLhhwjQIOl2gZCisWhNBGM+4oX7N/BjCAF4vc7oN5obrbI+rjauwoN0rUdT5jVvAXspVXx9Hl3ZlT/oCqogLzzbG7r8nJXNGfDASyKjRnOhjraTVhYnttgkOsQgMVNua5KuDGmtJQeshCysBZ16A3qOTblTDebUybbSjtgpRmYyfVAQqSqMTQygR2RrpbGvNj77L79z05a0TpBbDluDNLkjVAlrZ7FmNd7M4jyuLAwPStM3tHnPkXAPPVucO5cPI3l5KJNRNUxX37jRFU7tdN7NbSku8qxxoyFal67PvVU01+6xGlc5JbPVaUd621JYH8je5g+y4VMhv2o06FH5D7NXXHf809qR32xUbvPMOcBKjBZYDX+1DgHH2hMm3ezlcKgh707XQGAAIAvM5rZPXfe4MpgF9s0XEB4MXMhLSyNJ2uros=" 123 | } 124 | computer_name = "puppet.jenkins.io" 125 | 126 | # Encrypt all disks (ephemeral, temp dirs and data volumes) - https://learn.microsoft.com/en-us/azure/virtual-machines/disks-enable-host-based-encryption-portal?tabs=azure-powershell 127 | encryption_at_host_enabled = true 128 | 129 | os_disk { 130 | caching = "ReadWrite" 131 | storage_account_type = "StandardSSD_LRS" 132 | disk_size_gb = 32 # Minimal size for ubuntu 20.04 image 133 | } 134 | 135 | source_image_reference { 136 | publisher = "Canonical" 137 | offer = "0001-com-ubuntu-minimal-focal" 138 | sku = "minimal-20_04-lts-gen2" 139 | version = "latest" 140 | } 141 | } 142 | 143 | resource "azurerm_dns_a_record" "azure_puppet_jenkins_io" { 144 | name = "azure.puppet" 145 | zone_name = data.azurerm_dns_zone.jenkinsio.name 146 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 147 | ttl = 60 148 | records = [azurerm_public_ip.puppet_jenkins_io.ip_address] 149 | tags = local.default_tags 150 | } 151 | 152 | resource "azurerm_dns_cname_record" "jenkinsio_target_puppet_jenkins_io" { 153 | name = "puppet" 154 | zone_name = data.azurerm_dns_zone.jenkinsio.name 155 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 156 | ttl = 60 157 | record = azurerm_dns_a_record.azure_puppet_jenkins_io.fqdn 158 | tags = local.default_tags 159 | } 160 | -------------------------------------------------------------------------------- /cert.ci.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | module "cert_ci_jenkins_io" { 2 | source = "./.shared-tools/terraform/modules/azure-jenkinsinfra-controller" 3 | 4 | providers = { 5 | azurerm = azurerm 6 | azurerm.dns = azurerm 7 | azuread = azuread 8 | } 9 | 10 | service_fqdn = module.cert_ci_jenkins_io_letsencrypt.zone_name 11 | location = data.azurerm_resource_group.cert_ci_jenkins_io.location 12 | admin_username = local.admin_username 13 | admin_ssh_publickey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDDpxwvySus2OWViWfJ02XMYr+Qa/uPADhjt/4el2SmEf7NlJXzq5vc8imcw8YxQZKwuuKJhonlTYTpk1Cjka4bJKWNOSQ8+Kx0O2ZnNjKn3ZETWJB90bZXHVqbrNHDtu6lN6S/yRW9Q+6fuDbHBW0MXWI8Lsv+bU5v8Zll6m62rc00/I/IT9c1TX1qjCtjf5XHMFw7nVxQiTX2Zf5UKG3RI7mkCMDIvx2H9kXdzM8jtYwATZPHKHuLzffARmvy1FpNPVuLLEGYE3hljP82rll1WZbbl1ZrhjzbFUUYO4fsA7AOQHWhHiVLvtnreB269JOl/ZkHgk37zcdwJMkqKpqoEbjP9z8PURf5uMA7TiDGcpgcFMzoaFk1ueqoHM2JaM2AZQAkPhbUfT7MSOFYRx91OEg5pg5N17zNeaBM6fyxl3v7mkxSOTkKlzjAXPRyo7XsosUVQ4qb4DfsAAJ0Rynts2olRQLEzJku0ZxbbXotuoppI8HivRl7PoTsAASJRpc=" 14 | controller_network_name = data.azurerm_virtual_network.cert_ci_jenkins_io.name 15 | controller_network_rg_name = data.azurerm_resource_group.cert_ci_jenkins_io.name 16 | controller_subnet_name = data.azurerm_subnet.cert_ci_jenkins_io_controller.name 17 | controller_data_disk_size_gb = 128 18 | controller_vm_size = "Standard_B2s" 19 | default_tags = local.default_tags 20 | 21 | jenkins_infra_ips = { 22 | ldap_ipv4 = azurerm_public_ip.publick8s_ips["publick8s-ldap-ipv4"].ip_address, 23 | puppet_ipv4 = azurerm_public_ip.puppet_jenkins_io.ip_address, 24 | privatevpn_subnet = data.azurerm_subnet.private_vnet_data_tier.address_prefixes, 25 | } 26 | 27 | controller_service_principal_ids = [ 28 | data.azuread_service_principal.terraform_production.object_id, 29 | ] 30 | controller_packer_rg_ids = [ 31 | azurerm_resource_group.packer_images_cdf["prod"].id, 32 | ] 33 | 34 | agent_ip_prefixes = concat( 35 | data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.address_prefixes, 36 | ) 37 | } 38 | 39 | module "cert_ci_jenkins_io_azurevm_agents" { 40 | source = "./.shared-tools/terraform/modules/azure-jenkinsinfra-azurevm-agents" 41 | 42 | service_fqdn = module.cert_ci_jenkins_io.service_fqdn 43 | service_short_stripped_name = module.cert_ci_jenkins_io.service_short_stripped_name 44 | ephemeral_agents_network_rg_name = data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.resource_group_name 45 | ephemeral_agents_network_name = data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.virtual_network_name 46 | ephemeral_agents_subnet_name = data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.name 47 | controller_rg_name = module.cert_ci_jenkins_io.controller_resourcegroup_name 48 | controller_ips = compact([module.cert_ci_jenkins_io.controller_public_ipv4]) 49 | controller_service_principal_id = module.cert_ci_jenkins_io.controller_service_principal_id 50 | default_tags = local.default_tags 51 | jenkins_infra_ips = { 52 | privatevpn_subnet = data.azurerm_subnet.private_vnet_data_tier.address_prefixes 53 | } 54 | } 55 | 56 | resource "azurerm_user_assigned_identity" "cert_ci_jenkins_io_jenkins_agents" { 57 | location = data.azurerm_virtual_network.cert_ci_jenkins_io.location 58 | name = "cert-ci-jenkins-io-agents" 59 | resource_group_name = data.azurerm_virtual_network.cert_ci_jenkins_io.resource_group_name 60 | } 61 | # The Controller identity must be able to operate this identity to assign it to VM agents - https://plugins.jenkins.io/azure-vm-agents/#plugin-content-roles-required-by-feature 62 | resource "azurerm_role_assignment" "cert_ci_jenkins_io_operate_agent_uaid" { 63 | scope = azurerm_user_assigned_identity.cert_ci_jenkins_io_jenkins_agents.id 64 | role_definition_name = "Managed Identity Operator" 65 | principal_id = module.cert_ci_jenkins_io.controller_service_principal_id 66 | } 67 | resource "azurerm_role_assignment" "cert_ci_jenkins_io_azurevm_agents_jenkins_write_buildsreports_share" { 68 | scope = azurerm_storage_account.builds_reports_jenkins_io.id 69 | # Allow writing 70 | role_definition_name = "Storage File Data Privileged Contributor" 71 | principal_id = azurerm_user_assigned_identity.cert_ci_jenkins_io_jenkins_agents.principal_id 72 | } 73 | 74 | 75 | # Required to allow controller to check for subnets inside the virtual network 76 | resource "azurerm_role_definition" "cert_ci_jenkins_io_vnet_reader" { 77 | name = "read-cert-ci-jenkins-io-vnet" 78 | scope = data.azurerm_virtual_network.cert_ci_jenkins_io.id 79 | 80 | permissions { 81 | actions = ["Microsoft.Network/virtualNetworks/read"] 82 | } 83 | } 84 | resource "azurerm_role_assignment" "cert_ci_jenkins_io_controller_vnet_reader" { 85 | scope = data.azurerm_virtual_network.cert_ci_jenkins_io.id 86 | role_definition_id = azurerm_role_definition.cert_ci_jenkins_io_vnet_reader.role_definition_resource_id 87 | principal_id = module.cert_ci_jenkins_io.controller_service_principal_id 88 | } 89 | 90 | 91 | ## Service DNS records 92 | resource "azurerm_dns_a_record" "cert_ci_jenkins_io_controller" { 93 | name = "controller" 94 | zone_name = module.cert_ci_jenkins_io_letsencrypt.zone_name 95 | resource_group_name = module.cert_ci_jenkins_io_letsencrypt.zone_rg_name 96 | ttl = 60 97 | records = [module.cert_ci_jenkins_io.controller_private_ipv4] 98 | } 99 | resource "azurerm_dns_a_record" "cert_ci_jenkins_io" { 100 | name = "@" # Child zone: no CNAME possible! 101 | zone_name = module.cert_ci_jenkins_io_letsencrypt.zone_name 102 | resource_group_name = module.cert_ci_jenkins_io_letsencrypt.zone_rg_name 103 | ttl = 60 104 | records = [module.cert_ci_jenkins_io.controller_private_ipv4] 105 | } 106 | 107 | ## Allow access to/from ACR endpoint 108 | resource "azurerm_network_security_rule" "allow_out_https_from_cert_agents_to_acr" { 109 | count = var.terratest ? 0 : 1 110 | name = "allow-out-https-from-agents-to-acr" 111 | priority = 4050 112 | direction = "Outbound" 113 | access = "Allow" 114 | protocol = "Tcp" 115 | source_port_range = "*" 116 | destination_port_range = "443" 117 | source_address_prefixes = data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.address_prefixes 118 | destination_address_prefixes = distinct( 119 | flatten( 120 | [for rs in azurerm_private_endpoint.dockerhub_mirror["certcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] 121 | ) 122 | ) 123 | resource_group_name = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 124 | network_security_group_name = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 125 | } 126 | resource "azurerm_network_security_rule" "allow_in_https_from_cert_agents_to_acr" { 127 | count = var.terratest ? 0 : 1 128 | name = "allow-in-https-from-agents-to-acr" 129 | priority = 4050 130 | direction = "Inbound" 131 | access = "Allow" 132 | protocol = "Tcp" 133 | source_port_range = "*" 134 | destination_port_range = "443" 135 | source_address_prefixes = data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.address_prefixes 136 | destination_address_prefixes = distinct( 137 | flatten( 138 | [for rs in azurerm_private_endpoint.dockerhub_mirror["certcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] 139 | ) 140 | ) 141 | resource_group_name = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 142 | network_security_group_name = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 143 | } 144 | 145 | module "cert_ci_jenkins_io_letsencrypt" { 146 | source = "./.shared-tools/terraform/modules/azure-letsencrypt-dns" 147 | 148 | default_tags = local.default_tags 149 | zone_name = "cert.ci.jenkins.io" 150 | dns_rg_name = data.azurerm_resource_group.proddns_jenkinsio.name 151 | parent_zone_name = data.azurerm_dns_zone.jenkinsio.name 152 | principal_id = module.cert_ci_jenkins_io.controller_service_principal_id 153 | } 154 | -------------------------------------------------------------------------------- /dockerhub-mirror.tf: -------------------------------------------------------------------------------- 1 | #### ACR to use as DockerHub (and other) Registry mirror 2 | resource "azurerm_resource_group" "dockerhub_mirror" { 3 | name = "dockerhub-mirror" 4 | location = var.location 5 | } 6 | 7 | resource "azurerm_container_registry" "dockerhub_mirror" { 8 | name = "dockerhubmirror" 9 | resource_group_name = azurerm_resource_group.dockerhub_mirror.name 10 | location = azurerm_resource_group.dockerhub_mirror.location 11 | sku = "Premium" 12 | admin_enabled = false 13 | public_network_access_enabled = false # private links are used to reach the registry 14 | anonymous_pull_enabled = true # Requires "Standard" or "Premium" sku. Docker Engine cannot use auth. for pull trough cache - ref. https://github.com/moby/moby/issues/30880 15 | data_endpoint_enabled = true # Required for endpoint private link. Requires "Premium" sku. 16 | 17 | tags = local.default_tags 18 | } 19 | 20 | locals { 21 | acr_private_links = { 22 | "certcijenkinsio" = { 23 | "subnet_id" = data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.id, 24 | "vnet_id" = data.azurerm_virtual_network.cert_ci_jenkins_io.id, 25 | "rg_name" = data.azurerm_virtual_network.cert_ci_jenkins_io.resource_group_name, 26 | }, 27 | "infracijenkinsio" = { 28 | "subnet_id" = data.azurerm_subnet.infra_ci_jenkins_io_ephemeral_agents.id, 29 | "vnet_id" = data.azurerm_virtual_network.infra_ci_jenkins_io.id, 30 | "rg_name" = data.azurerm_virtual_network.infra_ci_jenkins_io.resource_group_name, 31 | }, 32 | "publick8s" = { 33 | "subnet_id" = data.azurerm_subnet.publick8s.id, 34 | "vnet_id" = data.azurerm_virtual_network.public.id, 35 | "rg_name" = data.azurerm_resource_group.public.name, 36 | }, 37 | "privatek8s" = { 38 | "subnet_id" = data.azurerm_subnet.privatek8s_tier.id, 39 | "vnet_id" = data.azurerm_virtual_network.private.id, 40 | "rg_name" = data.azurerm_resource_group.private.name, 41 | }, 42 | "trustedcijenkinsio" = { 43 | "subnet_id" = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.id, 44 | "vnet_id" = data.azurerm_virtual_network.trusted_ci_jenkins_io.id, 45 | "rg_name" = data.azurerm_virtual_network.trusted_ci_jenkins_io.resource_group_name, 46 | }, 47 | } 48 | } 49 | 50 | resource "azurerm_private_endpoint" "dockerhub_mirror" { 51 | for_each = local.acr_private_links 52 | 53 | name = "acr-${each.key}" 54 | 55 | location = azurerm_resource_group.dockerhub_mirror.location 56 | resource_group_name = azurerm_resource_group.dockerhub_mirror.name 57 | subnet_id = each.value.subnet_id 58 | 59 | custom_network_interface_name = "acr-${each.key}-nic" 60 | 61 | private_service_connection { 62 | name = "acr-${each.key}" 63 | private_connection_resource_id = azurerm_container_registry.dockerhub_mirror.id 64 | subresource_names = ["registry"] 65 | is_manual_connection = false 66 | } 67 | private_dns_zone_group { 68 | name = "privatelink.azurecr.io" 69 | private_dns_zone_ids = [ 70 | (can(each.value["private_dns_zone_id"]) ? each.value["private_dns_zone_id"] : azurerm_private_dns_zone.dockerhub_mirror[each.key].id), 71 | ] 72 | } 73 | tags = local.default_tags 74 | } 75 | 76 | resource "azurerm_private_dns_zone" "dockerhub_mirror" { 77 | for_each = local.acr_private_links 78 | # Conventional and static name required by Azure (otherwise automatic record creation does not work) 79 | name = "privatelink.azurecr.io" 80 | 81 | # Private DNS zone name is static: we can only have one per RG 82 | resource_group_name = each.value.rg_name 83 | 84 | tags = local.default_tags 85 | } 86 | 87 | resource "azurerm_private_dns_zone_virtual_network_link" "dockerhub_mirror" { 88 | for_each = local.acr_private_links 89 | 90 | name = "privatelink.azurecr.io" 91 | # Private DNS zone name is static: we can only have one per RG 92 | resource_group_name = each.value.rg_name 93 | private_dns_zone_name = azurerm_private_dns_zone.dockerhub_mirror[each.key].name 94 | virtual_network_id = each.value.vnet_id 95 | 96 | registration_enabled = true 97 | tags = local.default_tags 98 | } 99 | 100 | ## TODO: factorize and simplify RBAC policy with other keyvaults 101 | #trivy:ignore:avd-azu-0016 102 | resource "azurerm_key_vault" "dockerhub_mirror" { 103 | name = "dockerhubmirror" 104 | location = azurerm_resource_group.dockerhub_mirror.location 105 | resource_group_name = azurerm_resource_group.dockerhub_mirror.name 106 | 107 | tenant_id = data.azurerm_client_config.current.tenant_id 108 | soft_delete_retention_days = 7 109 | purge_protection_enabled = false 110 | rbac_authorization_enabled = true 111 | enabled_for_deployment = true 112 | enabled_for_disk_encryption = true 113 | enabled_for_template_deployment = true 114 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 115 | public_network_access_enabled = true 116 | # Adding a network rule with `public_network_access_enabled` set to `true` (default) selects the option "Enabled from selected virtual networks and IP addresses" 117 | network_acls { 118 | bypass = "AzureServices" 119 | default_action = "Deny" 120 | virtual_network_subnet_ids = local.app_subnets["infra.ci.jenkins.io"].agents 121 | } 122 | 123 | sku_name = "standard" 124 | 125 | tags = local.default_tags 126 | } 127 | 128 | # IMPORTANT: when bootstrapping, 2 distincts "terraform apply" are required: 129 | # 1. The first one must create the Keyvault (at least: can do the ACR and Private Endpoints/DNS/network links). 130 | # 2. Then, the 2 secrets must be manually created with the respective names "dockerhub-username" and "dockerhub-password" 131 | # 3. Finally, the set of "data.azurerm_key_vault_secret" + ACR CredentialSet + Role assignement + Registry Cache rule can be created as a 2nd terraform deployment 132 | resource "azurerm_container_registry_credential_set" "dockerhub" { 133 | name = "dockerhub" 134 | container_registry_id = azurerm_container_registry.dockerhub_mirror.id 135 | login_server = "docker.io" 136 | identity { 137 | type = "SystemAssigned" 138 | } 139 | authentication_credentials { 140 | username_secret_id = "${azurerm_key_vault.dockerhub_mirror.vault_uri}secrets/dockerhub-username" 141 | password_secret_id = "${azurerm_key_vault.dockerhub_mirror.vault_uri}secrets/dockerhub-password" 142 | } 143 | } 144 | resource "azurerm_role_assignment" "acr_read_keyvault_secrets" { 145 | scope = azurerm_key_vault.dockerhub_mirror.id 146 | role_definition_name = "Key Vault Secrets User" 147 | skip_service_principal_aad_check = true 148 | principal_id = azurerm_container_registry_credential_set.dockerhub.identity[0].principal_id 149 | } 150 | 151 | resource "azurerm_container_registry_cache_rule" "mirror_cache_rules" { 152 | for_each = { 153 | "dockerhub-library-namespace" = { 154 | source = "docker.io/library/*" 155 | target = "library/*" 156 | }, 157 | "dockerhub-jenkins-namespace" = { 158 | source = "docker.io/jenkins/*" 159 | target = "jenkins/*" 160 | } 161 | "dockerhub-moby-namespace" = { 162 | source = "docker.io/moby/*" 163 | target = "moby/*" 164 | } 165 | # Also used on AWS (ci.jenkins.io) 166 | "dockerhub-jenkinsciinfra-jau-2204" = { 167 | source = "docker.io/jenkinsciinfra/jenkins-agent-ubuntu-22.04" 168 | target = "jenkinsciinfra/jenkins-agent-ubuntu-22.04" 169 | } 170 | # Also used on AWS (ci.jenkins.io) 171 | "dockerhub-jenkinsciinfra-packaging" = { 172 | source = "docker.io/jenkinsciinfra/packaging" 173 | target = "jenkinsciinfra/packaging" 174 | } 175 | # Used by ATH image build 176 | "dockerhub-rockstorm-git-server" = { 177 | source = "docker.io/rockstorm/git-server" 178 | target = "rockstorm/git-server" 179 | } 180 | # Used by ATH image build 181 | "dockerhub-testcontainers-ryuk" = { 182 | source = "docker.io/testcontainers/ryuk" 183 | target = "testcontainers/ryuk" 184 | } 185 | } 186 | name = "mirror-${each.key}" 187 | container_registry_id = azurerm_container_registry.dockerhub_mirror.id 188 | source_repo = each.value.source 189 | target_repo = each.value.target 190 | credential_set_id = azurerm_container_registry_credential_set.dockerhub.id 191 | } 192 | 193 | #### Allow provided Principal IDs to push images to the registry 194 | resource "azurerm_role_assignment" "push_to_acr" { 195 | for_each = var.terratest ? toset([]) : toset([ 196 | azurerm_user_assigned_identity.infra_ci_jenkins_io_agents.principal_id, 197 | ]) 198 | principal_id = each.value 199 | role_definition_name = "AcrPush" 200 | scope = azurerm_container_registry.dockerhub_mirror.id 201 | skip_service_principal_aad_check = true 202 | } 203 | -------------------------------------------------------------------------------- /.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/cyrilgdn/postgresql" { 5 | version = "1.26.0" 6 | hashes = [ 7 | "h1:8bXFg6KkLzUAd44WUnqSxVY0pqXALT14h59OlYq3UTY=", 8 | "h1:EQH2pCkaR0vf3OyzyEr/SdYTg6JCrgSV1WYxkRYCUtc=", 9 | "h1:OmjWcf08BwORkQ+VpOw1qlh2LE4ZR/huj9KZt2IRHsQ=", 10 | "h1:kId855uQr126ruzaPg2KA5EZ+j2fwASTZOK4QplxyBY=", 11 | "zh:0f2ec2bb24f8bb9eb232f1650d6459a2bac732bf91bbc08b27ae5519bee89486", 12 | "zh:11dafcec9c7e6e2c8b6303d90c6061973db26f6f84adc2be02fe66e9b1b11561", 13 | "zh:13a67dc639ee053cbecc6ab28fd5bfca4780e680bd12491f1bdf0f8243fd364a", 14 | "zh:56337a42348bb9ab31837caa89d89f7a3ee0528b5a6d04a6a93a8ea155eb7f4d", 15 | "zh:590e80218e70e8081a11cf1f5df16014426d6ba8c2552713cc61f56041c7457b", 16 | "zh:5e4b12dd1874bab454720a50c600d1df359dc71f91f4a0194cf8eb335e27dfe7", 17 | "zh:6af55f892e7f463c75a62215dc74790ae9a71d7d23c74c6ddb40af258528fa46", 18 | "zh:78f6739ca865622981c28fa6628128be4651bd4629450a9ba5b1945d64b66da7", 19 | "zh:8ed469b0d9074eba59216e57794a03fe27b45586b03d24946d130949ae92093f", 20 | "zh:a261aaee5675986711cf9f963d5d9ca5ec1d62aa8c31866da54c6670d803b8a3", 21 | "zh:a64b52597738ff1bac41127141c48800f1575eaa66a67cecdc9b0b16728dae0e", 22 | "zh:ae5e821f5d5510bc2cba2aefcf6c0e62af9e28b7a25e0e8dcd039e04172594d0", 23 | "zh:cfae79ed700febe8fb29fd1c5d0a6ace0a0103bef8ec37bb653dc23afc960b33", 24 | "zh:d9a69d5475982a00d4e9e07f56987c821782595bac29e9084237285d36fa88e8", 25 | ] 26 | } 27 | 28 | provider "registry.terraform.io/hashicorp/azuread" { 29 | version = "3.7.0" 30 | hashes = [ 31 | "h1:+DHwtiYCOYEhLMot5KQ0hm+zopmUJwPJ/gJiXVfdNOs=", 32 | "h1:OsoKRyP1cmXqoSgmlZ2QcriiJDrXYHuGf+5ppydkwT4=", 33 | "h1:w4jBl2atkUWTNUspZIUpgcYuoJZ4EbULB+BdzNsZ7uk=", 34 | "h1:xUoYzyYNcneRJDN8rlJ4tT+ZjebaDqvuNm6eMxKCeX4=", 35 | "zh:01114beb07eb8415a563ca80a513e7fd11fd6f0670bdd960e9092e58464ab2b7", 36 | "zh:1460c790ca28cc11c3efd4abec076fc5cdd4ae1cc654a45c69e10ca064e0a611", 37 | "zh:161b6dd82cf279e7ac7a31c404f2ace491a5a9e7ec382ace91de2fd18f8c190b", 38 | "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", 39 | "zh:2eff0bea92c48be0221863ebf7752f491232088f4ec3cbe4da2ebf9f478fed1f", 40 | "zh:69a689198fcac054da30fea6a8abce2f12b667f6f327c9bb388f2c307ee782fd", 41 | "zh:6f5acf42d71a2f5157c54e67b2db9b080c4ebb27b812112c0c84241e66d94803", 42 | "zh:8e0a58ff2c21aff372c66c679db48578be48e99f5cf6727f082cf4b629dbda6c", 43 | "zh:9c88cc06ba7ee1f6e6b43af7e223c4d107fb77e5a984f1441fffb00c3693537f", 44 | "zh:e8656e8143baac71029857c72899f8503cede41f2559c89ce29d21c0436c96fc", 45 | "zh:e8e21913956a3f043c0e65cc95278bdc82f62ff33fd75a56b1919bd63fd14f8a", 46 | "zh:f7f213dd61d1c75b8397d97660cf329870a241e88ec6e0dc07c60725d975252e", 47 | ] 48 | } 49 | 50 | provider "registry.terraform.io/hashicorp/azurerm" { 51 | version = "4.57.0" 52 | hashes = [ 53 | "h1:Fa+0G/+9CtvcJHHHfblgOOKzyECXi62l8iq4lok3VXY=", 54 | "h1:NhgHn/RyZRDXMa7pEQlGv/9B+wjk48E+lvgq4asFKHs=", 55 | "h1:WE4WVJWxOmTqgE77tctu6STEaG+6XRpriyMZL/DrZLk=", 56 | "h1:rjN0+XlMl5wCVtwi2gGI6n3AbRwHFudCb+7szNFabm0=", 57 | "zh:05e1cc7fee7829919b772ca6ce893d9c2abb3535ebff172df38f7358cdaf8f9e", 58 | "zh:30122203abc381660582f989c9e53874bd9ff93e25476a5536ea0ae37dd51f4b", 59 | "zh:4a90f008f7707d95f8f9aca90f140a9ca0e9506b0a6d436fe516de4026cacd86", 60 | "zh:6d9e114b8aed06454b71fe91a0591cc6a16cd7acde3cb36a96e4aeaec06a315a", 61 | "zh:7145c50facd9d40615fc63561ec21962feae3fa262239f9f1f1339581226104b", 62 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 63 | "zh:95f60557f1bc4210ecc3c11e2f86fe983ed7e4af19036616b605887c1195f2ac", 64 | "zh:9722b3ab879a3457588af5f0dcd65e997263affe4b829e60ecd59dbef6239e70", 65 | "zh:b891f295b018d058e8c6f841d923d5b30ba13b8208f2c20aacc70ba48c5cc0da", 66 | "zh:cb7ff113ca0bd91ab76f9f7a492d6ce9c911d6c4deb8c8e263e38322b5ff861e", 67 | "zh:ec2950bf003d29bee3fa87ab073d57fe14a4da52e9fc646fec27798b700cc8af", 68 | "zh:f69899d9e1d570a560cfa97aebec3edc2b106f2ebc15dfdee473294dd8756deb", 69 | ] 70 | } 71 | 72 | provider "registry.terraform.io/hashicorp/kubernetes" { 73 | version = "2.37.1" 74 | hashes = [ 75 | "h1:+37jC6JlkPyPvDHudK3qaj7ZVJ0Zy9zc9+oq8h1WayA=", 76 | "h1:CtT3PtFlrxO1MocpAJxvRzLe6PHtcLSMKDSVPVV9zuw=", 77 | "h1:qo9Ue/rIEnvxOpiK9qizwRFV7rvb5gCziKVytIcZHyk=", 78 | "h1:x4cNsR4InB/AEpSRCmv6SQEY/ChylNaPceNlyzvfv7c=", 79 | "zh:0ed097413c7fc804479e325966886b405dc0b75ad2b4f54ce4df1d8e4802b397", 80 | "zh:17dcf4a685a00d2d048671124e8a1a8e836b58ecd2ef628a1c666fe0ced2e598", 81 | "zh:36891284e5bced57c438f12d0b27856b0d4b70b562bd200b01919a6a89545be9", 82 | "zh:3e49d86b508e641ba122d1b0af24cdc4d8ffa2ec1b30022436fb1d7c6ba696ea", 83 | "zh:40be623e116708bdcb0fac32989db43720f031c5fe9a4dc63395078185d24403", 84 | "zh:44fc0ac3bc39e289b67f9dde7ee9fef29eb8192197e5e68fee69098573021722", 85 | "zh:957aa451573bcde5d57f6f8338ea3139010c7f61fefe8f6a140a8c267f056511", 86 | "zh:c55fd85b7e8acaac17e30670ac3574b88b3530820dd004bcd2a5daa8624a46e9", 87 | "zh:c743f06843a1f5ecde2b8ef639f4d3db654a334ef248dee57261c719ea843f3a", 88 | "zh:c93cc71c64b838d89522ac5fb60f68e0e1e7f2fc39db6b0ead7afd78795e79ed", 89 | "zh:eda1163c2266905adc54bc78cc3e7b606a164fbc6b59be607db933b302015ccd", 90 | "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", 91 | ] 92 | } 93 | 94 | provider "registry.terraform.io/hashicorp/local" { 95 | version = "2.6.1" 96 | hashes = [ 97 | "h1:3xsXDTrMyw3QAktU4aGAja2wE0/6DQV8NaAJL3GeTeU=", 98 | "h1:DbiR/D2CPigzCGweYIyJH0N0x04oyI5xiZ9wSW/s3kQ=", 99 | "h1:LMoX85QLTgCCqRuy2aXoz47P7gZ4WRPSA00fUPC/Rho=", 100 | "h1:ixzpZFfRJjW6G1hxQDQ66Lx3gfvVxB8+g5FzwDYO0xA=", 101 | "zh:10050d08f416de42a857e4b6f76809aae63ea4ec6f5c852a126a915dede814b4", 102 | "zh:2df2a3ebe9830d4759c59b51702e209fe053f47453cb4688f43c063bac8746b7", 103 | "zh:2e759568bcc38c86ca0e43701d34cf29945736fdc8e429c5b287ddc2703c7b18", 104 | "zh:6a62a34e48500ab4aea778e355e162ebde03260b7a9eb9edc7e534c84fbca4c6", 105 | "zh:74373728ba32a1d5450a3a88ac45624579e32755b086cd4e51e88d9aca240ef6", 106 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 107 | "zh:8dddae588971a996f622e7589cd8b9da7834c744ac12bfb59c97fa77ded95255", 108 | "zh:946f82f66353bb97aefa8d95c4ca86db227f9b7c50b82415289ac47e4e74d08d", 109 | "zh:e9a5c09e6f35e510acf15b666fd0b34a30164cecdcd81ce7cda0f4b2dade8d91", 110 | "zh:eafe5b873ef42b32feb2f969c38ff8652507e695620cbaf03b9db714bee52249", 111 | "zh:ec146289fa27650c9d433bb5c7847379180c0b7a323b1b94e6e7ad5d2a7dbe71", 112 | "zh:fc882c35ce05631d76c0973b35adde26980778fc81d9da81a2fade2b9d73423b", 113 | ] 114 | } 115 | 116 | provider "registry.terraform.io/hashicorp/random" { 117 | version = "3.7.2" 118 | hashes = [ 119 | "h1:356j/3XnXEKr9nyicLUufzoF4Yr6hRy481KIxRVpK0c=", 120 | "h1:KG4NuIBl1mRWU0KD/BGfCi1YN/j3F7H4YgeeM7iSdNs=", 121 | "h1:Lmv2TxyKKm9Vt4uxcPZHw1uf0Ax/yYizJlilbLSZN8E=", 122 | "h1:hkKSY5xI4R1H4Yrg10HHbtOoxZif2dXa9HFPSbaVg5o=", 123 | "zh:14829603a32e4bc4d05062f059e545a91e27ff033756b48afbae6b3c835f508f", 124 | "zh:1527fb07d9fea400d70e9e6eb4a2b918d5060d604749b6f1c361518e7da546dc", 125 | "zh:1e86bcd7ebec85ba336b423ba1db046aeaa3c0e5f921039b3f1a6fc2f978feab", 126 | "zh:24536dec8bde66753f4b4030b8f3ef43c196d69cccbea1c382d01b222478c7a3", 127 | "zh:29f1786486759fad9b0ce4fdfbbfece9343ad47cd50119045075e05afe49d212", 128 | "zh:4d701e978c2dd8604ba1ce962b047607701e65c078cb22e97171513e9e57491f", 129 | "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", 130 | "zh:7b8434212eef0f8c83f5a90c6d76feaf850f6502b61b53c329e85b3b281cba34", 131 | "zh:ac8a23c212258b7976e1621275e3af7099e7e4a3d4478cf8d5d2a27f3bc3e967", 132 | "zh:b516ca74431f3df4c6cf90ddcdb4042c626e026317a33c53f0b445a3d93b720d", 133 | "zh:dc76e4326aec2490c1600d6871a95e78f9050f9ce427c71707ea412a2f2f1a62", 134 | "zh:eac7b63e86c749c7d48f527671c7aee5b4e26c10be6ad7232d6860167f99dbb0", 135 | ] 136 | } 137 | 138 | provider "registry.terraform.io/petoju/mysql" { 139 | version = "3.0.87" 140 | hashes = [ 141 | "h1:79KSLjDQ33LdW4qTNu18w4ZppKdyHN/xeu3rwrYGeq0=", 142 | "h1:B7vNuVKZjO7Oc1fIj66uoDmilYvAQieDNfGOA46Krgk=", 143 | "h1:NJgZ2zX8soskGgaRd6KI310/eLSTO/+1GrTELuqGvgU=", 144 | "h1:O97VXEvlZW2na5QFEOVDTcE4MNwCVKPbGFD5Yb3YZho=", 145 | "zh:0ae61cbe9aca3f3263690b311396dff6591bca09bad0ddc6e876df33b7a684c7", 146 | "zh:1583ae8d38c997b26b8a1f1554c5c47fa3fae9178447f281237bbb5ce23805b4", 147 | "zh:15bc7518bd6f74f643e24af3166fb6d479b3cfd1feaab004dec0100f230f5d5f", 148 | "zh:2f56ab512b5f7d1af572845e1ee26639105f8e7162589c2f068be313b6fc57bd", 149 | "zh:32b99a90ce6d4967d22268b99e5caa316f19eabc5222b3d2c0735cdfd5a248ae", 150 | "zh:582063ac2ba6f2d56bce288f65a7208f99076f662137fac49c926a05dbad1fee", 151 | "zh:5aaa1940bbe541b4d8df6950e8a1e2656e8e695d81ab986b404935dede0f311c", 152 | "zh:6f30b48e11c6fbc2c24b0847605926daa2ef5cd5b337b96955e5e61895f31e76", 153 | "zh:80cd8e8cca9a426a2f131322d2b8e38ba2f14be95985b8656fc0dfc7465ebf5d", 154 | "zh:89b45788e086c16184ef7262e0d426cab0208934b786c42e5b57d117c0a38df3", 155 | "zh:b16e2bff0f879217644da9f81c2b0c4bf8a98a095477669bc934aedd72022264", 156 | "zh:d648a1658ca91c7e98eb5bf7a9ac657078f733838823f18f10c664075a07d76d", 157 | "zh:d8a15e7ba2c19e229fefd0e8d1b4072dad4d58a5fe9498ba67ce212fc98b7631", 158 | "zh:f8d2085fbe9880090dbdeab8716f2bc58886b03255e472a9b804eea6b61f8910", 159 | ] 160 | } 161 | -------------------------------------------------------------------------------- /infraci.jenkins.io-agents-2.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "infracijenkinsio_agents_2" { 2 | name = local.aks_clusters["infracijenkinsio_agents_2"].name 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | #trivy:ignore:avd-azu-0040 # No need to enable oms_agent for Azure monitoring as we already have datadog 8 | resource "azurerm_kubernetes_cluster" "infracijenkinsio_agents_2" { 9 | name = local.aks_clusters["infracijenkinsio_agents_2"].name 10 | sku_tier = "Standard" 11 | ## Private cluster requires network setup to allow API access from: 12 | # - infra.ci.jenkins.io agents (for both terraform job agents and kubernetes-management agents) 13 | # - private.vpn.jenkins.io to allow admin management (either Azure UI or kube tools from admin machines) 14 | private_cluster_enabled = true 15 | private_cluster_public_fqdn_enabled = true 16 | dns_prefix = "infracijenkinsioagents2" # Avoid hyphens in this DNS host 17 | location = azurerm_resource_group.infracijenkinsio_agents_2.location 18 | resource_group_name = azurerm_resource_group.infracijenkinsio_agents_2.name 19 | kubernetes_version = local.aks_clusters["infracijenkinsio_agents_2"].kubernetes_version 20 | # default value but made explicit to please trivy 21 | role_based_access_control_enabled = true 22 | oidc_issuer_enabled = true 23 | workload_identity_enabled = true 24 | 25 | image_cleaner_interval_hours = 48 26 | 27 | network_profile { 28 | network_plugin = "azure" 29 | network_plugin_mode = "overlay" 30 | network_policy = "azure" 31 | outbound_type = "userAssignedNATGateway" 32 | load_balancer_sku = "standard" # Required to customize the outbound type 33 | pod_cidr = local.aks_clusters.infracijenkinsio_agents_2.pod_cidr 34 | } 35 | 36 | identity { 37 | type = "SystemAssigned" 38 | } 39 | 40 | default_node_pool { 41 | name = "systempool1" 42 | only_critical_addons_enabled = true # This property is the only valid way to add the "CriticalAddonsOnly=true:NoSchedule" taint to the default node pool 43 | vm_size = "Standard_D2pds_v5" 44 | temporary_name_for_rotation = "syspooltemp" 45 | upgrade_settings { 46 | max_surge = "10%" 47 | } 48 | os_sku = "AzureLinux" 49 | os_disk_type = "Ephemeral" 50 | os_disk_size_gb = 75 # Ref. Cache storage size at https://learn.microsoft.com/fr-fr/azure/virtual-machines/dasv5-dadsv5-series#dadsv5-series (depends on the instance size) 51 | orchestrator_version = local.aks_clusters["infracijenkinsio_agents_2"].kubernetes_version 52 | kubelet_disk_type = "OS" 53 | auto_scaling_enabled = true 54 | min_count = 2 # for best practices 55 | max_count = 3 # for upgrade 56 | vnet_subnet_id = data.azurerm_subnet.infracijenkinsio_agents_2.id 57 | tags = local.default_tags 58 | zones = local.aks_clusters.compute_zones.system_pool 59 | } 60 | 61 | tags = local.default_tags 62 | } 63 | 64 | # Node pool to host infra.ci.jenkins.io x86_64 agents 65 | # number of pods per node calculated with https://github.com/jenkins-infra/kubernetes-management/blob/9c14f72867170e9755f3434fb6f6dd3a8606686a/config/jenkins_infra.ci.jenkins.io.yaml#L137-L208 66 | resource "azurerm_kubernetes_cluster_node_pool" "infracijenkinsio_agents_2_linux_x86_64_agents_1" { 67 | name = "lx86n14agt1" 68 | vm_size = "Standard_D8ads_v5" # https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series Standard_D8ads_v5 8vcpu 32Go 300ssd 69 | os_sku = "AzureLinux" 70 | os_disk_type = "Ephemeral" 71 | os_disk_size_gb = 300 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dasv5-dadsv5-series (depends on the instance size) 72 | priority = "Spot" 73 | eviction_policy = "Delete" # Ref Not on priority https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool 74 | spot_max_price = -1 # Ref Not on priority https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool 75 | orchestrator_version = local.aks_clusters["infracijenkinsio_agents_2"].kubernetes_version 76 | kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_2.id 77 | auto_scaling_enabled = true 78 | min_count = 0 79 | max_count = 20 80 | zones = local.aks_clusters.compute_zones.amd64_pool 81 | vnet_subnet_id = data.azurerm_subnet.infracijenkinsio_agents_2.id 82 | 83 | node_labels = { 84 | "jenkins" = "infra.ci.jenkins.io" 85 | "role" = "jenkins-agents" 86 | "kubernetes.azure.com/scalesetpriority" = "spot" 87 | } 88 | node_taints = [ 89 | "infra.ci.jenkins.io/agents=true:NoSchedule", 90 | "kubernetes.azure.com/scalesetpriority=spot:NoSchedule" 91 | ] 92 | 93 | lifecycle { 94 | ignore_changes = [node_count] 95 | } 96 | 97 | tags = local.default_tags 98 | } 99 | 100 | # # Node pool to host infra.ci.jenkins.io arm64 agents 101 | # number of pods per node calculated with https://github.com/jenkins-infra/kubernetes-management/blob/9c14f72867170e9755f3434fb6f6dd3a8606686a/config/jenkins_infra.ci.jenkins.io.yaml#L137-L208 102 | resource "azurerm_kubernetes_cluster_node_pool" "infracijenkinsio_agents_2_linux_arm64_agents_2" { 103 | 104 | name = "la64n14agt2" 105 | vm_size = "Standard_D16pds_v5" # temporarily upgrade https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/general-purpose/dpdsv5-series?tabs=sizebasic 16vcpu 64Go 600ssd 106 | os_sku = "AzureLinux" 107 | os_disk_type = "Ephemeral" 108 | os_disk_size_gb = 600 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/sizes/general-purpose/dpdsv5-series?tabs=sizebasic (depends on the instance size) 109 | priority = "Spot" 110 | eviction_policy = "Delete" # Ref Not on priority https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool 111 | spot_max_price = -1 # Ref Not on priority https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool 112 | orchestrator_version = local.aks_clusters["infracijenkinsio_agents_2"].kubernetes_version 113 | kubernetes_cluster_id = azurerm_kubernetes_cluster.infracijenkinsio_agents_2.id 114 | auto_scaling_enabled = true 115 | min_count = 1 116 | max_count = 20 117 | zones = local.aks_clusters.compute_zones.arm64_pool 118 | vnet_subnet_id = data.azurerm_subnet.infracijenkinsio_agents_2.id 119 | 120 | node_labels = { 121 | "jenkins" = "infra.ci.jenkins.io" 122 | "role" = "jenkins-agents" 123 | "kubernetes.azure.com/scalesetpriority" = "spot" 124 | } 125 | node_taints = [ 126 | "infra.ci.jenkins.io/agents=true:NoSchedule", 127 | "kubernetes.azure.com/scalesetpriority=spot:NoSchedule" 128 | ] 129 | 130 | lifecycle { 131 | ignore_changes = [node_count] 132 | } 133 | 134 | tags = local.default_tags 135 | } 136 | resource "kubernetes_namespace" "infracijenkinsio_agents_2_infra_ci_jenkins_io_agents" { 137 | provider = kubernetes.infracijenkinsio_agents_2 138 | 139 | metadata { 140 | name = "jenkins-infra-agents" 141 | labels = { 142 | name = "jenkins-infra-agents" 143 | } 144 | } 145 | } 146 | resource "kubernetes_service_account" "infracijenkinsio_agents_2_infra_ci_jenkins_io_agents" { 147 | provider = kubernetes.infracijenkinsio_agents_2 148 | 149 | metadata { 150 | name = "jenkins-infra-agent" 151 | namespace = kubernetes_namespace.infracijenkinsio_agents_2_infra_ci_jenkins_io_agents.metadata[0].name 152 | 153 | annotations = { 154 | "azure.workload.identity/client-id" = azurerm_user_assigned_identity.infra_ci_jenkins_io_agents.client_id 155 | } 156 | } 157 | } 158 | resource "azurerm_federated_identity_credential" "infracijenkinsio_agents_2_infra_ci_jenkins_io_agents" { 159 | name = "infracijenkinsio-agents-2-${kubernetes_service_account.infracijenkinsio_agents_2_infra_ci_jenkins_io_agents.metadata[0].name}" 160 | audience = ["api://AzureADTokenExchange"] 161 | issuer = azurerm_kubernetes_cluster.infracijenkinsio_agents_2.oidc_issuer_url 162 | parent_id = azurerm_user_assigned_identity.infra_ci_jenkins_io_agents.id 163 | # RG must be the same for both the UAID and the federated ID (otherwise you get HTTP/404 during the "apply" phase) 164 | resource_group_name = azurerm_user_assigned_identity.infra_ci_jenkins_io_agents.resource_group_name 165 | subject = "system:serviceaccount:${kubernetes_namespace.infracijenkinsio_agents_2_infra_ci_jenkins_io_agents.metadata[0].name}:${kubernetes_service_account.infracijenkinsio_agents_2_infra_ci_jenkins_io_agents.metadata[0].name}" 166 | } 167 | 168 | #Configure the jenkins-infra/kubernetes-management admin service account 169 | module "infracijenkinsio_agents_2_admin_sa" { 170 | providers = { 171 | kubernetes = kubernetes.infracijenkinsio_agents_2 172 | } 173 | source = "./.shared-tools/terraform/modules/kubernetes-admin-sa" 174 | cluster_name = azurerm_kubernetes_cluster.infracijenkinsio_agents_2.name 175 | cluster_hostname = local.aks_clusters_outputs.infracijenkinsio_agents_2.cluster_hostname 176 | cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.infracijenkinsio_agents_2.kube_config.0.cluster_ca_certificate 177 | } 178 | output "kubeconfig_management_infracijenkinsio_agents_2" { 179 | sensitive = true 180 | value = module.infracijenkinsio_agents_2_admin_sa.kubeconfig 181 | } 182 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | resource "local_file" "jenkins_infra_data_report" { 2 | content = jsonencode({ 3 | "cert.ci.jenkins.io" = { 4 | "agents_azure_vms" = { 5 | "resource_group_name" = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_resource_group_name, 6 | "network_resource_group_name" = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_network_rg_name, 7 | "virtual_network_name" = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_network_name, 8 | "sub_network_name" = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_subnet_name, 9 | "storage_account_name" = module.cert_ci_jenkins_io_azurevm_agents.ephemeral_agents_storage_account_name, 10 | "user_assigned_identity" = azurerm_user_assigned_identity.cert_ci_jenkins_io_jenkins_agents.id, 11 | }, 12 | }, 13 | "infra.ci.jenkins.io" = { 14 | "controller_namespace" = kubernetes_namespace.privatek8s["infra-ci-jenkins-io"].metadata[0].name, 15 | "controller_service_account" = kubernetes_service_account.privatek8s_infra_ci_jenkins_io_controller.metadata[0].name, 16 | "controller_pvc" = kubernetes_persistent_volume_claim.privatek8s_infra_ci_jenkins_io_data.metadata[0].name, 17 | "agents_azure_vms" = { 18 | "resource_group_name" = module.infra_ci_jenkins_io_azurevm_agents.ephemeral_agents_resource_group_name, 19 | "network_resource_group_name" = module.infra_ci_jenkins_io_azurevm_agents.ephemeral_agents_network_rg_name, 20 | "virtual_network_name" = module.infra_ci_jenkins_io_azurevm_agents.ephemeral_agents_network_name, 21 | "sub_network_name" = module.infra_ci_jenkins_io_azurevm_agents.ephemeral_agents_subnet_name, 22 | "storage_account_name" = module.infra_ci_jenkins_io_azurevm_agents.ephemeral_agents_storage_account_name, 23 | "user_assigned_identity" = azurerm_user_assigned_identity.infra_ci_jenkins_io_agents.id, 24 | }, 25 | "agents_kubernetes_clusters" = { 26 | "infracijenkinsio_agents_2" = { 27 | "hostname" = local.aks_clusters_outputs.infracijenkinsio_agents_2.cluster_hostname 28 | "kubernetes_version" = local.aks_clusters["infracijenkinsio_agents_2"].kubernetes_version 29 | "agents_namespaces" = { 30 | "${kubernetes_namespace.infracijenkinsio_agents_2_infra_ci_jenkins_io_agents.metadata[0].name}" = { 31 | pods_quota = 150, 32 | }, 33 | }, 34 | "agents_service_account" = kubernetes_service_account.infracijenkinsio_agents_2_infra_ci_jenkins_io_agents.metadata[0].name, 35 | }, 36 | }, 37 | }, 38 | "release.ci.jenkins.io" = { 39 | "controller_namespace" = kubernetes_namespace.privatek8s["release-ci-jenkins-io"].metadata[0].name, 40 | "controller_service_account" = kubernetes_service_account.privatek8s_release_ci_jenkins_io_controller.metadata[0].name, 41 | "controller_pvc" = kubernetes_persistent_volume_claim.privatek8s_release_ci_jenkins_io_data.metadata[0].name, 42 | "agents_kubernetes_clusters" = { 43 | "privatek8s" = { 44 | "agents_service_account" = kubernetes_service_account.privatek8s_release_ci_jenkins_io_agents.metadata[0].name, 45 | } 46 | "persistentVolumeClaims" = { 47 | "data-storage-jenkins-io" = { 48 | "share_uri" = "/", 49 | "pvc_name" = kubernetes_persistent_volume_claim.privatek8s_release_ci_jenkins_io_agents_data_storage.metadata[0].name, 50 | } 51 | } 52 | } 53 | }, 54 | "trusted.ci.jenkins.io" = { 55 | "agents_azure_vms" = { 56 | "resource_group_name" = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_resource_group_name, 57 | "network_resource_group_name" = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_network_rg_name, 58 | "virtual_network_name" = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_network_name, 59 | "sub_network_name" = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_subnet_name, 60 | "storage_account_name" = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_storage_account_name, 61 | "user_assigned_identity" = azurerm_user_assigned_identity.trusted_ci_jenkins_io_azurevm_agents_jenkins.id, 62 | }, 63 | }, 64 | "get.jenkins.io" = { 65 | "mirrorbits" = { 66 | "share_uri" = "/get.jenkins.io/mirrorbits/", 67 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["get-jenkins-io"].metadata[0].name, 68 | }, 69 | "httpd" = { 70 | "share_uri" = "/get.jenkins.io/mirrorbits/", 71 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["get-jenkins-io"].metadata[0].name, 72 | }, 73 | "geoipdata" = { 74 | "share_uri" = "/get.jenkins.io/geoipdata/", 75 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["get-jenkins-io"].metadata[0].name, 76 | } 77 | }, 78 | "updates.jenkins.io" = { 79 | "content" = { 80 | "share_uri" = "/updates.jenkins.io/content/", 81 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["updates-jenkins-io"].metadata[0].name, 82 | }, 83 | "redirections" = { 84 | "share_uri" = "/updates.jenkins.io/redirections/", 85 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["updates-jenkins-io"].metadata[0].name, 86 | }, 87 | "geoipdata" = { 88 | "share_uri" = "/updates.jenkins.io/geoipdata/", 89 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["updates-jenkins-io"].metadata[0].name, 90 | } 91 | }, 92 | "ldap.jenkins.io" = { 93 | "data" = { 94 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_datadisks["ldap-jenkins-io"].metadata[0].name, 95 | }, 96 | "backup" = { 97 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["ldap-jenkins-io-backup"].metadata[0].name, 98 | }, 99 | }, 100 | "puppet.jenkins.io" = { 101 | "ipv4" = azurerm_public_ip.puppet_jenkins_io.ip_address, 102 | # DMZ: same in and out public IP 103 | "outbound_ips" = azurerm_public_ip.puppet_jenkins_io.ip_address, 104 | }, 105 | "javadoc.jenkins.io" = { 106 | "data" = { 107 | "share_uri" = "/javadoc.jenkins.io/", 108 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["javadoc-jenkins-io"].metadata[0].name, 109 | }, 110 | "namespace" = kubernetes_namespace.publick8s_namespaces["javadoc-jenkins-io"].metadata[0].name, 111 | }, 112 | "www.jenkins.io" = { 113 | "data" = { 114 | "share_uri" = "/www.jenkins.io/en/", 115 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["www-jenkins-io"].metadata[0].name, 116 | }, 117 | "namespace" = kubernetes_namespace.publick8s_namespaces["www-jenkins-io"].metadata[0].name, 118 | }, 119 | "reports.jenkins.io" = { 120 | "data" = { 121 | "pvc_name" = kubernetes_persistent_volume_claim.publick8s_azurefiles["reports-jenkins-io"].metadata[0].name, 122 | }, 123 | "namespace" = kubernetes_namespace.publick8s_namespaces["reports-jenkins-io"].metadata[0].name, 124 | }, 125 | "publick8s" = { 126 | hostname = azurerm_kubernetes_cluster.publick8s.fqdn, 127 | kubernetes_version = local.aks_clusters["publick8s"].kubernetes_version 128 | pod_cidrs = concat(flatten(azurerm_kubernetes_cluster.publick8s.network_profile[*].pod_cidrs)), 129 | lb_outbound_ips = { 130 | "ipv4" = [for id, pip in data.azurerm_public_ip.publick8s_lb_outbound : pip.ip_address if can(cidrnetmask("${pip.ip_address}/32"))], 131 | "ipv6" = [for id, pip in data.azurerm_public_ip.publick8s_lb_outbound : pip.ip_address if !can(cidrnetmask("${pip.ip_address}/32"))], 132 | }, 133 | }, 134 | "privatek8s" = { 135 | hostname = local.aks_clusters_outputs.privatek8s.cluster_hostname, 136 | kubernetes_version = local.aks_clusters["privatek8s"].kubernetes_version, 137 | # Outbound IPs are in azure-net (NAT gateway outbound IPs 138 | public_inbound_lb = { 139 | "public_ip_name" = azurerm_public_ip.privatek8s.name, 140 | "public_ip_rg_name" = azurerm_public_ip.privatek8s.resource_group_name, 141 | "subnet" = data.azurerm_subnet.privatek8s_tier.name, 142 | } 143 | private_inbound_ips = { 144 | "ipv4" = azurerm_dns_a_record.privatek8s_private.records, 145 | } 146 | }, 147 | "admin_public_ips" = local.admin_public_ips, 148 | }) 149 | filename = "${path.module}/jenkins-infra-data-reports/azure.json" 150 | } 151 | output "jenkins_infra_data_report" { 152 | value = local_file.jenkins_infra_data_report.content 153 | } 154 | 155 | # infra.ci Azure storage credentials 156 | output "infraci_pluginsjenkinsio_fileshare_serviceprincipal_writer_application_client_id" { 157 | value = module.infraci_pluginsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_id 158 | } 159 | output "infraci_pluginsjenkinsio_fileshare_serviceprincipal_writer_application_client_password" { 160 | sensitive = true 161 | value = module.infraci_pluginsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_password 162 | } 163 | output "infraci_contributorsjenkinsio_fileshare_serviceprincipal_writer_application_client_id" { 164 | value = module.infraci_contributorsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_id 165 | } 166 | output "infraci_contributorsjenkinsio_fileshare_serviceprincipal_writer_application_client_password" { 167 | sensitive = true 168 | value = module.infraci_contributorsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_password 169 | } 170 | output "infraci_docsjenkinsio_fileshare_serviceprincipal_writer_application_client_id" { 171 | value = module.infraci_docsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_id 172 | } 173 | output "infraci_docsjenkinsio_fileshare_serviceprincipal_writer_application_client_password" { 174 | sensitive = true 175 | value = module.infraci_docsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_password 176 | } 177 | output "infraci_statsjenkinsio_fileshare_serviceprincipal_writer_application_client_id" { 178 | value = module.infraci_statsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_id 179 | } 180 | output "infraci_statsjenkinsio_fileshare_serviceprincipal_writer_application_client_password" { 181 | sensitive = true 182 | value = module.infraci_statsjenkinsio_fileshare_serviceprincipal_writer.fileshare_serviceprincipal_writer_application_client_password 183 | } 184 | -------------------------------------------------------------------------------- /trusted.ci.jenkins.io.tf: -------------------------------------------------------------------------------- 1 | #################################################################################### 2 | ## Resources for the Controller VM 3 | #################################################################################### 4 | module "trusted_ci_jenkins_io_letsencrypt" { 5 | source = "./.shared-tools/terraform/modules/azure-letsencrypt-dns" 6 | 7 | default_tags = local.default_tags 8 | zone_name = "trusted.ci.jenkins.io" 9 | dns_rg_name = data.azurerm_resource_group.proddns_jenkinsio.name 10 | parent_zone_name = data.azurerm_dns_zone.jenkinsio.name 11 | principal_id = module.trusted_ci_jenkins_io.controller_service_principal_id 12 | } 13 | module "trusted_ci_jenkins_io" { 14 | source = "./.shared-tools/terraform/modules/azure-jenkinsinfra-controller" 15 | 16 | providers = { 17 | azurerm = azurerm 18 | azurerm.dns = azurerm 19 | azuread = azuread 20 | } 21 | 22 | service_fqdn = "trusted.ci.jenkins.io" 23 | location = data.azurerm_virtual_network.trusted_ci_jenkins_io.location 24 | admin_username = local.admin_username 25 | admin_ssh_publickey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC5K7Ro7jBl5Kc68RdzG6EXHstIBFSxO5Da8SQJSMeCbb4cHTYuBBH8jNsAFcnkN64kEu+YhmlxaWEVEIrPgfGfs13ZL7v9p+Nt76tsz6gnVdAy2zCz607pAWe7p4bBn6T9zdZcBSnvjawO+8t/5ue4ngcfAjanN5OsOgLeD6yqVyP8YTERjW78jvp2TFrIYmgWMI5ES1ln32PQmRZwc1eAOsyGJW/YIBdOxaSkZ41qUvb9b3dCorGuCovpSK2EeNphjLPpVX/NRpVY4YlDqAcTCdLdDrEeVqkiA/VDCYNhudZTDa8f1iHwBE/GEtlKmoO6dxJ5LAkRk3RIVHYrmI6XXSw5l0tHhW5D12MNwzUfDxQEzBpGK5iSfOBt5zJ5OiI9ftnsq/GV7vCXfvMVGDLUC551P5/s/wM70QmHwhlGQNLNeJxRTvd6tL11bof3K+29ivFYUmpU17iVxYOWhkNY86WyngHU6Ux0zaczF3H6H0tpg1Ca/cFO428AVPw/RTJpcAe6OVKq5zwARNApQ/p6fJKUAdXap+PpQGZlQhPLkUbwtFXGTrpX9ePTcdzryCYjgrZouvy4ZMzruJiIbFUH8mRY3xVREVaIsJakruvgw3b14oQgcB4BwYVBBqi62xIvbRzAv7Su9t2jK6OR2z3sM/hLJRqIJ5oILMORa7XqrQ==" 26 | controller_network_name = data.azurerm_virtual_network.trusted_ci_jenkins_io.name 27 | controller_network_rg_name = data.azurerm_resource_group.trusted_ci_jenkins_io.name 28 | controller_subnet_name = data.azurerm_subnet.trusted_ci_jenkins_io_controller.name 29 | controller_data_disk_size_gb = 128 30 | controller_vm_size = "Standard_B2s" 31 | default_tags = local.default_tags 32 | 33 | controller_resourcegroup_name = "jenkinsinfra-trusted-ci-controller" 34 | controller_datadisk_name = "trusted-ci-controller-data-disk" 35 | 36 | jenkins_infra_ips = { 37 | ldap_ipv4 = azurerm_public_ip.publick8s_ips["publick8s-ldap-ipv4"].ip_address, 38 | puppet_ipv4 = azurerm_public_ip.puppet_jenkins_io.ip_address, 39 | privatevpn_subnet = data.azurerm_subnet.private_vnet_data_tier.address_prefixes, 40 | } 41 | 42 | controller_service_principal_ids = [ 43 | # Commenting out to migrate to new AzureAD provider 44 | # data.azuread_service_principal.terraform_production.id, 45 | "b847a030-25e1-4791-ad04-9e8484d87bce", 46 | ] 47 | controller_packer_rg_ids = [ 48 | azurerm_resource_group.packer_images_cdf["prod"].id, 49 | ] 50 | 51 | agent_ip_prefixes = concat( 52 | data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.address_prefixes, 53 | [ 54 | azurerm_linux_virtual_machine.agent_trusted_ci_jenkins_io.private_ip_address 55 | ], 56 | ) 57 | } 58 | 59 | module "trusted_ci_jenkins_io_azurevm_agents" { 60 | source = "./.shared-tools/terraform/modules/azure-jenkinsinfra-azurevm-agents" 61 | 62 | service_fqdn = module.trusted_ci_jenkins_io.service_fqdn 63 | service_short_stripped_name = module.trusted_ci_jenkins_io.service_short_stripped_name 64 | ephemeral_agents_network_rg_name = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.resource_group_name 65 | ephemeral_agents_network_name = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.virtual_network_name 66 | ephemeral_agents_subnet_name = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.name 67 | controller_rg_name = module.trusted_ci_jenkins_io.controller_resourcegroup_name 68 | controller_ips = compact([module.trusted_ci_jenkins_io.controller_public_ipv4]) 69 | controller_service_principal_id = module.trusted_ci_jenkins_io.controller_service_principal_id 70 | default_tags = local.default_tags 71 | jenkins_infra_ips = { 72 | privatevpn_subnet = data.azurerm_subnet.private_vnet_data_tier.address_prefixes 73 | } 74 | } 75 | # Required to allow controller to check for subnets inside the agents virtual network 76 | resource "azurerm_role_definition" "trusted_ci_jenkins_io_controller_vnet_reader" { 77 | name = "Read-trusted-ci-jenkins-io-VNET" 78 | scope = data.azurerm_virtual_network.trusted_ci_jenkins_io.id 79 | 80 | permissions { 81 | actions = ["Microsoft.Network/virtualNetworks/read"] 82 | } 83 | } 84 | resource "azurerm_role_assignment" "trusted_controller_ephemeral_agents_vnet_reader" { 85 | scope = data.azurerm_virtual_network.trusted_ci_jenkins_io.id 86 | role_definition_id = azurerm_role_definition.trusted_ci_jenkins_io_controller_vnet_reader.role_definition_resource_id 87 | principal_id = module.trusted_ci_jenkins_io.controller_service_principal_id 88 | } 89 | # Allow controller to manage agents without requiring credentials (requires on the VM User Assign Identity) 90 | resource "azurerm_user_assigned_identity" "trusted_ci_jenkins_io_azurevm_agents_jenkins" { 91 | location = data.azurerm_virtual_network.trusted_ci_jenkins_io.location 92 | name = "trusted-ci-jenkins-io-agents" 93 | resource_group_name = module.trusted_ci_jenkins_io.controller_resourcegroup_name 94 | } 95 | # The Controller identity must be able to operate this identity to assign it to VM agents - https://plugins.jenkins.io/azure-vm-agents/#plugin-content-roles-required-by-feature 96 | resource "azurerm_role_assignment" "trusted_ci_jenkins_io_manage_agent_uaid" { 97 | scope = azurerm_user_assigned_identity.trusted_ci_jenkins_io_azurevm_agents_jenkins.id 98 | role_definition_name = "Managed Identity Operator" 99 | principal_id = module.trusted_ci_jenkins_io.controller_service_principal_id 100 | } 101 | resource "azurerm_role_assignment" "trusted_ci_jenkins_io_azurevm_agents_jenkins_write_buildsreports_share" { 102 | scope = azurerm_storage_account.builds_reports_jenkins_io.id 103 | # Allow writing 104 | role_definition_name = "Storage File Data Privileged Contributor" 105 | principal_id = azurerm_user_assigned_identity.trusted_ci_jenkins_io_azurevm_agents_jenkins.principal_id 106 | } 107 | resource "azurerm_role_assignment" "trusted_ci_jenkins_io_azurevm_agents_jenkins_write_reports_share" { 108 | scope = azurerm_storage_account.reports_jenkins_io.id 109 | # Allow writing 110 | role_definition_name = "Storage File Data Privileged Contributor" 111 | principal_id = azurerm_user_assigned_identity.trusted_ci_jenkins_io_azurevm_agents_jenkins.principal_id 112 | } 113 | 114 | resource "azurerm_role_assignment" "trusted_ci_jenkins_io_azurevm_agents_jenkins_write_javadoc_share" { 115 | scope = azurerm_storage_account.javadoc_jenkins_io.id 116 | # Allow writing 117 | role_definition_name = "Storage File Data Privileged Contributor" 118 | principal_id = azurerm_user_assigned_identity.trusted_ci_jenkins_io_azurevm_agents_jenkins.principal_id 119 | } 120 | 121 | #################################################################################### 122 | ## Network Security Group and rules 123 | #################################################################################### 124 | ## Outbound Rules (different set of priorities than Inbound rules) ## 125 | resource "azurerm_network_security_rule" "allow_out_from_trusted_all_to_uc" { 126 | name = "allow-out-from-trusted-all-to-uc" 127 | priority = 4050 128 | direction = "Outbound" 129 | access = "Allow" 130 | protocol = "Tcp" 131 | source_port_range = "*" 132 | destination_port_ranges = [ 133 | "3390", # mirrorbits CLI (content) 134 | ] 135 | source_address_prefixes = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.address_prefixes 136 | destination_address_prefixes = [ 137 | # Update Center (mirrorbits CLI) 138 | azurerm_private_endpoint.publick8s_updates_jenkins_io_for_trustedci.private_service_connection[0].private_ip_address, 139 | ] 140 | resource_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 141 | network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 142 | } 143 | resource "azurerm_network_security_rule" "allow_out_many_from_trusted_agents_to_pkg" { 144 | name = "allow-out-many-from-agents-to-pkg" 145 | priority = 4055 146 | direction = "Outbound" 147 | access = "Allow" 148 | protocol = "Tcp" 149 | source_port_range = "*" 150 | destination_port_ranges = [ 151 | "22", # SSH (for rsync) 152 | ] 153 | source_address_prefixes = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.address_prefixes 154 | destination_address_prefix = local.external_services["pkg.origin.jenkins.io"] 155 | resource_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 156 | network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 157 | } 158 | resource "azurerm_network_security_rule" "allow_out_many_from_trusted_agents_to_archive" { 159 | name = "allow-out-many-from-agents-to-archive" 160 | priority = 4060 161 | direction = "Outbound" 162 | access = "Allow" 163 | protocol = "Tcp" 164 | source_port_range = "*" 165 | destination_port_ranges = [ 166 | "22", # SSH (for rsync) 167 | ] 168 | source_address_prefixes = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.address_prefixes 169 | destination_address_prefix = local.external_services["archives.jenkins.io"] 170 | resource_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 171 | network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 172 | } 173 | ## Inbound Rules (different set of priorities than Outbound rules) ## 174 | resource "azurerm_network_security_rule" "allow_in_many_from_trusted_agents_to_uc" { 175 | name = "allow-in-many-from-trusted-agents-to-uc" 176 | priority = 4050 177 | direction = "Inbound" 178 | access = "Allow" 179 | protocol = "Tcp" 180 | source_port_range = "*" 181 | destination_port_ranges = [ 182 | "3390", # mirrorbits CLI (content) 183 | ] 184 | source_address_prefixes = data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.address_prefixes 185 | destination_address_prefixes = [ 186 | # Update Center (mirrorbits CLI) 187 | azurerm_private_endpoint.publick8s_updates_jenkins_io_for_trustedci.private_service_connection[0].private_ip_address, 188 | ] 189 | resource_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 190 | network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 191 | } 192 | 193 | ## Allow access to/from ACR endpoint 194 | resource "azurerm_network_security_rule" "allow_out_https_from_trusted_to_acr" { 195 | name = "allow-out-https-from-vnet-to-acr" 196 | priority = 4051 197 | direction = "Outbound" 198 | access = "Allow" 199 | protocol = "Tcp" 200 | source_port_range = "*" 201 | destination_port_range = "443" 202 | source_address_prefixes = data.azurerm_virtual_network.trusted_ci_jenkins_io.address_space 203 | destination_address_prefixes = distinct( 204 | flatten( 205 | [for rs in azurerm_private_endpoint.dockerhub_mirror["trustedcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] 206 | ) 207 | ) 208 | resource_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 209 | network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 210 | } 211 | resource "azurerm_network_security_rule" "allow_in_https_from_trusted_to_acr" { 212 | name = "allow-in-https-from-vnet-to-acr" 213 | priority = 4051 214 | direction = "Inbound" 215 | access = "Allow" 216 | protocol = "Tcp" 217 | source_port_range = "*" 218 | destination_port_range = "443" 219 | source_address_prefixes = data.azurerm_virtual_network.trusted_ci_jenkins_io.address_space 220 | destination_address_prefixes = distinct( 221 | flatten( 222 | [for rs in azurerm_private_endpoint.dockerhub_mirror["trustedcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] 223 | ) 224 | ) 225 | resource_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_rg_name 226 | network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents.ephemeral_agents_nsg_name 227 | } 228 | 229 | #################################################################################### 230 | ## Public DNS records 231 | #################################################################################### 232 | resource "azurerm_dns_a_record" "trusted_ci_controller" { 233 | name = "@" 234 | zone_name = module.trusted_ci_jenkins_io_letsencrypt.zone_name 235 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 236 | ttl = 60 237 | records = [module.trusted_ci_jenkins_io.controller_private_ipv4] 238 | } 239 | 240 | #################################################################################### 241 | ## Private network resources (endpoint, DNS, etc.) 242 | #################################################################################### 243 | resource "azurerm_private_dns_a_record" "updates_jenkins_io" { 244 | name = "updates.jenkins.io" # Full expected record name: updates.jenkins.io.privatelink.azurecr.io 245 | zone_name = azurerm_private_dns_zone.dockerhub_mirror["trustedcijenkinsio"].name 246 | resource_group_name = azurerm_private_dns_zone.dockerhub_mirror["trustedcijenkinsio"].resource_group_name 247 | ttl = 60 248 | records = [azurerm_private_endpoint.publick8s_updates_jenkins_io_for_trustedci.private_service_connection[0].private_ip_address] 249 | } 250 | 251 | ## updates.jenkins.io's mirrorbits CLI Kubernetes Service (internal LB) 252 | data "azurerm_private_link_service" "publick8s_mirrorbitscli_updates_jenkins_io" { 253 | # TODO: track with updatecli from https://github.com/jenkins-infra/kubernetes-management/config/publick8s_updates-jenkins-io.yaml 254 | name = "publick8s-updates.jenkins.io" 255 | resource_group_name = azurerm_kubernetes_cluster.publick8s.node_resource_group 256 | } 257 | resource "azurerm_private_endpoint" "publick8s_updates_jenkins_io_for_trustedci" { 258 | name = "${data.azurerm_private_link_service.publick8s_mirrorbitscli_updates_jenkins_io.name}-for-trustedci" 259 | 260 | location = var.location 261 | resource_group_name = data.azurerm_subnet.trusted_ci_jenkins_io_permanent_agents.resource_group_name 262 | subnet_id = data.azurerm_subnet.trusted_ci_jenkins_io_permanent_agents.id 263 | 264 | custom_network_interface_name = "${data.azurerm_private_link_service.publick8s_mirrorbitscli_updates_jenkins_io.name}-for-trustedci-nic" 265 | 266 | private_service_connection { 267 | name = "${data.azurerm_private_link_service.publick8s_mirrorbitscli_updates_jenkins_io.name}-for-trustedci" 268 | private_connection_resource_id = data.azurerm_private_link_service.publick8s_mirrorbitscli_updates_jenkins_io.id 269 | is_manual_connection = false 270 | } 271 | private_dns_zone_group { 272 | name = azurerm_private_dns_zone.dockerhub_mirror["trustedcijenkinsio"].name 273 | private_dns_zone_ids = [azurerm_private_dns_zone.dockerhub_mirror["trustedcijenkinsio"].id] 274 | } 275 | tags = local.default_tags 276 | } 277 | -------------------------------------------------------------------------------- /publick8s.tf: -------------------------------------------------------------------------------- 1 | resource "azurerm_resource_group" "publick8s" { 2 | name = "publick8s" 3 | location = var.location 4 | tags = local.default_tags 5 | } 6 | 7 | data "azurerm_subnet" "publick8s" { 8 | name = "publick8s" 9 | resource_group_name = data.azurerm_resource_group.public.name 10 | virtual_network_name = data.azurerm_virtual_network.public.name 11 | } 12 | 13 | resource "azurerm_dns_a_record" "public_publick8s" { 14 | name = "public.publick8s" 15 | zone_name = data.azurerm_dns_zone.jenkinsio.name 16 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 17 | ttl = 60 18 | records = [azurerm_public_ip.publick8s_ips["publick8s-public-ipv4"].ip_address] 19 | tags = local.default_tags 20 | } 21 | 22 | resource "azurerm_dns_aaaa_record" "public_publick8s" { 23 | name = "public.publick8s" 24 | zone_name = data.azurerm_dns_zone.jenkinsio.name 25 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 26 | ttl = 60 27 | records = [azurerm_public_ip.publick8s_ips["publick8s-public-ipv6"].ip_address] 28 | tags = local.default_tags 29 | } 30 | 31 | resource "azurerm_dns_a_record" "private_publick8s" { 32 | name = "private.publick8s" 33 | zone_name = data.azurerm_dns_zone.jenkinsio.name 34 | resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name 35 | ttl = 60 36 | records = ["10.245.2.12"] # External IP of the private-nginx ingress LoadBalancer, created by https://github.com/jenkins-infra/kubernetes-management/blob/54a0d4aa72b15f4236abcfbde00a080905bbb890/clusters/publick8s.yaml#L63-L69 37 | tags = local.default_tags 38 | } 39 | 40 | #trivy:ignore:azure-container-logging #trivy:ignore:azure-container-limit-authorized-ips 41 | resource "azurerm_kubernetes_cluster" "publick8s" { 42 | name = local.aks_clusters["publick8s"].name 43 | location = azurerm_resource_group.publick8s.location 44 | sku_tier = "Standard" 45 | ## Private cluster requires network setup to allow API access from: 46 | # - infra.ci.jenkins.io agents (for both terraform job agents and kubernetes-management agents) 47 | # - private.vpn.jenkins.io to allow admin management (either Azure UI or kube tools from admin machines) 48 | private_cluster_enabled = true 49 | private_cluster_public_fqdn_enabled = true 50 | 51 | resource_group_name = azurerm_resource_group.publick8s.name 52 | kubernetes_version = local.aks_clusters["publick8s"].kubernetes_version 53 | dns_prefix = local.aks_clusters["publick8s"].name 54 | 55 | # default value but made explicit to please trivy 56 | role_based_access_control_enabled = true 57 | oidc_issuer_enabled = true 58 | workload_identity_enabled = true 59 | 60 | image_cleaner_interval_hours = 48 61 | 62 | network_profile { 63 | network_plugin = "azure" 64 | network_plugin_mode = "overlay" 65 | pod_cidrs = local.aks_clusters["publick8s"].pod_cidrs # Plural form: dual stack ipv4/ipv6 66 | ip_versions = ["IPv4", "IPv6"] 67 | outbound_type = "loadBalancer" 68 | load_balancer_sku = "standard" 69 | load_balancer_profile { 70 | outbound_ports_allocated = "2560" # Max 25 Nodes, 64000 ports total per public IP 71 | idle_timeout_in_minutes = "4" 72 | managed_outbound_ip_count = "3" 73 | managed_outbound_ipv6_count = "2" 74 | } 75 | } 76 | 77 | identity { 78 | type = "SystemAssigned" 79 | } 80 | 81 | default_node_pool { 82 | name = "linuxpool" 83 | only_critical_addons_enabled = false # We run our workloads along the system workloads 84 | vm_size = "Standard_D4pds_v5" # 4 vCPU, 16 GB RAM, local disk: 150 GB and 19000 IOPS 85 | upgrade_settings { 86 | drain_timeout_in_minutes = 5 # If a pod cannot be evicted in less than 5 min, then upgrades fails 87 | max_surge = 1 # Upgrade node one by one to avoid services to go down (when only 2 replicas) 88 | } 89 | os_sku = "AzureLinux" 90 | kubelet_disk_type = "OS" 91 | os_disk_type = "Ephemeral" 92 | os_disk_size_gb = 150 # Ref. Cache storage size at https://learn.microsoft.com/en-us/azure/virtual-machines/dpsv5-dpdsv5-series#dpdsv5-series (depends on the instance size) 93 | orchestrator_version = local.aks_clusters["publick8s"].kubernetes_version 94 | auto_scaling_enabled = true 95 | min_count = 3 96 | max_count = 5 97 | vnet_subnet_id = data.azurerm_subnet.publick8s.id 98 | tags = local.default_tags 99 | zones = [1, 2, 3] 100 | # No custom node_taints 101 | } 102 | 103 | tags = local.default_tags 104 | } 105 | 106 | # Allow cluster to manage network resources in the associated subnets 107 | # It is used for managing LBs of the public and private ingress controllers 108 | resource "azurerm_role_assignment" "publick8s_subnets_networkcontributor" { 109 | for_each = toset([ 110 | data.azurerm_subnet.publick8s.id, # Node pool 111 | ]) 112 | scope = each.key 113 | role_definition_name = "Network Contributor" 114 | principal_id = azurerm_kubernetes_cluster.publick8s.identity[0].principal_id 115 | skip_service_principal_aad_check = true 116 | } 117 | 118 | # Each public load balancer used by this cluster is setup with a locked public IP. 119 | # Using a pre-determined public IP eases DNS setup and changes, but requires cluster to have the "Network Contributor" role on the IP. 120 | locals { 121 | publick8s_public_ips = { 122 | "publick8s-public-ipv4" = "IPv4" # Ingress for HTTP services 123 | "publick8s-public-ipv6" = "IPv6" # Ingress for HTTP services 124 | "publick8s-ldap-ipv4" = "IPv4" # LDAP for its own LB (cannot share public IP across LBs) 125 | } 126 | } 127 | resource "azurerm_public_ip" "publick8s_ips" { 128 | for_each = local.publick8s_public_ips 129 | 130 | name = each.key 131 | resource_group_name = azurerm_resource_group.prod_public_ips.name 132 | location = var.location 133 | ip_version = each.value 134 | allocation_method = "Static" 135 | sku = "Standard" 136 | tags = local.default_tags 137 | } 138 | resource "azurerm_management_lock" "publick8s_ips" { 139 | for_each = local.publick8s_public_ips 140 | 141 | name = each.key 142 | scope = azurerm_public_ip.publick8s_ips[each.key].id 143 | lock_level = "CanNotDelete" 144 | notes = "Locked because this is a sensitive resource that should not be removed when publick8s cluster is re-created" 145 | } 146 | resource "azurerm_role_assignment" "publick8s_ips_networkcontributor" { 147 | for_each = local.publick8s_public_ips 148 | 149 | scope = azurerm_public_ip.publick8s_ips[each.key].id 150 | role_definition_name = "Network Contributor" 151 | principal_id = azurerm_kubernetes_cluster.publick8s.identity[0].principal_id 152 | skip_service_principal_aad_check = true 153 | } 154 | 155 | ################################ 156 | ### Kubernetes Resources below 157 | ################################ 158 | resource "kubernetes_storage_class" "publick8s_statically_provisioned" { 159 | metadata { 160 | name = "statically-provisioned" 161 | } 162 | storage_provisioner = "disk.csi.azure.com" 163 | reclaim_policy = "Retain" 164 | provider = kubernetes.publick8s 165 | allow_volume_expansion = true 166 | } 167 | 168 | # Configure the jenkins-infra/kubernetes-management admin service account 169 | module "publick8s_admin_sa" { 170 | providers = { 171 | kubernetes = kubernetes.publick8s 172 | } 173 | source = "./.shared-tools/terraform/modules/kubernetes-admin-sa" 174 | cluster_name = azurerm_kubernetes_cluster.publick8s.name 175 | cluster_hostname = local.aks_clusters_outputs.publick8s.cluster_hostname 176 | cluster_ca_certificate_b64 = azurerm_kubernetes_cluster.publick8s.kube_config.0.cluster_ca_certificate 177 | } 178 | 179 | # PVCs (see below) needs their namespaces 180 | resource "kubernetes_namespace" "publick8s_namespaces" { 181 | provider = kubernetes.publick8s 182 | for_each = toset(sort(distinct(concat( 183 | [for key, value in local.aks_clusters["publick8s"].azurefile_volumes : lookup(value, "pvc_namespace", key)], 184 | [for key, value in local.aks_clusters["publick8s"].azuredisk_volumes : lookup(value, "pvc_namespace", key)], 185 | ["data-storage-jenkins-io"], 186 | )))) 187 | 188 | metadata { 189 | name = each.key 190 | labels = { 191 | name = each.key 192 | } 193 | } 194 | } 195 | 196 | # PVs (see below) need storage secret keys when using CSI Azure file (as workload identity cannot be used with AKS CSI driver) 197 | resource "kubernetes_secret" "publick8s_azurefiles" { 198 | provider = kubernetes.publick8s 199 | for_each = toset(sort(distinct(concat( 200 | [for key, value in local.aks_clusters["publick8s"].azurefile_volumes : key if can(value["secret_name"])], 201 | )))) 202 | 203 | metadata { 204 | name = local.aks_clusters["publick8s"].azurefile_volumes[each.key].secret_name 205 | namespace = local.aks_clusters["publick8s"].azurefile_volumes[each.key].secret_namespace 206 | } 207 | 208 | data = { 209 | # Convention: secret name and storage account name are the same (it is namespaced and it makes no sense to duplicate on a given NS for many PVCs: we reuse) 210 | azurestorageaccountname = local.aks_clusters["publick8s"].azurefile_volumes[each.key].secret_name 211 | azurestorageaccountkey = local.aks_clusters["publick8s"].azurefile_volumes[each.key].storage_account_key 212 | } 213 | 214 | type = "Opaque" 215 | } 216 | resource "kubernetes_secret" "publick8s_azurefile_jenkins_io_storage_account" { 217 | provider = kubernetes.publick8s 218 | 219 | metadata { 220 | name = "data-storage-jenkins-io-storage-account" 221 | namespace = kubernetes_namespace.publick8s_namespaces["data-storage-jenkins-io"].metadata[0].name 222 | } 223 | 224 | data = { 225 | azurestorageaccountname = azurerm_storage_account.data_storage_jenkins_io.name 226 | azurestorageaccountkey = azurerm_storage_account.data_storage_jenkins_io.primary_access_key 227 | } 228 | 229 | type = "Opaque" 230 | } 231 | 232 | # We assume usage of the "big" NFS data storage as default (unless the local specifies other values for edge cases) 233 | # Note: when deleting a PV, you have to remove the 'metadata.finalizers' key (usually when deletion is stuck) 234 | resource "kubernetes_persistent_volume" "publick8s_azurefiles" { 235 | provider = kubernetes.publick8s 236 | for_each = local.aks_clusters["publick8s"].azurefile_volumes 237 | 238 | metadata { 239 | # Same name as the namespace (easier to map PVs which are NOT namespaced) 240 | name = each.key 241 | } 242 | spec { 243 | capacity = { 244 | storage = "${lookup(each.value, "capacity", azurerm_storage_share.data_storage_jenkins_io.quota)}Gi" 245 | } 246 | access_modes = lookup(each.value, "access_modes", ["ReadOnlyMany"]) 247 | persistent_volume_reclaim_policy = "Retain" 248 | storage_class_name = kubernetes_storage_class.publick8s_statically_provisioned.id 249 | # Ensure that only the designated PVC can claim this PV (to avoid injection as PV are not namespaced) 250 | claim_ref { 251 | # Default: PV name and NS names are the same (easier to map PVs which are NOT namespaced) 252 | # But we allow using a custom PVC namespace when the key (e.?g. the PV name) differs 253 | namespace = lookup(each.value, "pvc_namespace", each.key) 254 | name = each.key 255 | } 256 | mount_options = lookup(each.value, "mount_options", [ 257 | "nconnect=4", # Mandatory value (4) for Premium Azure File Share NFS 4.1. Increasing require using NetApp NFS instead ($$$) 258 | "noresvport", # ref. https://linux.die.net/man/5/nfs 259 | "actimeo=10", # Data is changed quite often 260 | "cto", # Ensure data consistency at the cost of slower I/O 261 | ]) 262 | persistent_volume_source { 263 | csi { 264 | driver = "file.csi.azure.com" 265 | fs_type = "ext4" 266 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 267 | volume_handle = lookup(each.value, "volume_handle", "${azurerm_storage_account.data_storage_jenkins_io.resource_group_name}#${azurerm_storage_account.data_storage_jenkins_io.name}#${azurerm_storage_share.data_storage_jenkins_io.name}") 268 | read_only = lookup(each.value, "read_only", true) 269 | volume_attributes = lookup(each.value, "volume_attributes", { 270 | protocol = "nfs" 271 | resourceGroup = azurerm_storage_account.data_storage_jenkins_io.resource_group_name 272 | shareName = azurerm_storage_share.data_storage_jenkins_io.name 273 | }) 274 | node_stage_secret_ref { 275 | name = lookup(each.value, "secret_name", kubernetes_secret.publick8s_azurefile_jenkins_io_storage_account.metadata[0].name) 276 | namespace = lookup(each.value, "secret_namespace", kubernetes_secret.publick8s_azurefile_jenkins_io_storage_account.metadata[0].namespace) 277 | } 278 | } 279 | } 280 | } 281 | } 282 | resource "kubernetes_persistent_volume_claim" "publick8s_azurefiles" { 283 | provider = kubernetes.publick8s 284 | for_each = local.aks_clusters["publick8s"].azurefile_volumes 285 | 286 | metadata { 287 | # Mapping 1:1 with PV and PVC using names (to allow claim_ref to work on PV) 288 | name = kubernetes_persistent_volume.publick8s_azurefiles[each.key].metadata[0].name 289 | # Default: PV name and NS names are the same (easier to map PVs which are NOT namespaced) 290 | # But we allow using a custom PVC namespace when the key (e.?g. the PV name) differs 291 | namespace = lookup(each.value, "pvc_namespace", each.key) 292 | } 293 | spec { 294 | access_modes = kubernetes_persistent_volume.publick8s_azurefiles[each.key].spec[0].access_modes 295 | volume_name = kubernetes_persistent_volume.publick8s_azurefiles[each.key].metadata[0].name 296 | storage_class_name = kubernetes_persistent_volume.publick8s_azurefiles[each.key].spec[0].storage_class_name 297 | resources { 298 | requests = { 299 | storage = kubernetes_persistent_volume.publick8s_azurefiles[each.key].spec[0].capacity.storage 300 | } 301 | } 302 | } 303 | } 304 | 305 | # Note: when deleting a PV, you have to remove the 'metadata.finalizers' key (usually when deletion is stuck) 306 | resource "kubernetes_persistent_volume" "publick8s_datadisks" { 307 | provider = kubernetes.publick8s 308 | for_each = local.aks_clusters["publick8s"].azuredisk_volumes 309 | 310 | metadata { 311 | # Disk name is the last element from the Azure ID string 312 | name = element(split("/", each.value.disk_id), "-1") 313 | } 314 | spec { 315 | capacity = { 316 | storage = "${each.value.disk_size}Gi" 317 | } 318 | access_modes = ["ReadWriteOnce"] 319 | persistent_volume_reclaim_policy = "Retain" 320 | storage_class_name = kubernetes_storage_class.publick8s_statically_provisioned.id 321 | persistent_volume_source { 322 | csi { 323 | driver = "disk.csi.azure.com" 324 | volume_handle = each.value.disk_id 325 | } 326 | } 327 | } 328 | } 329 | resource "kubernetes_persistent_volume_claim" "publick8s_datadisks" { 330 | provider = kubernetes.publick8s 331 | for_each = local.aks_clusters["publick8s"].azuredisk_volumes 332 | 333 | metadata { 334 | # Disk name is the last element from the Azure ID string 335 | name = element(split("/", each.value.disk_id), "-1") 336 | # Default: PV name and NS names are the same (easier to map PVs which are NOT namespaced) 337 | # But we allow using a custom PVC namespace when the key (e.?g. the PV name) differs 338 | namespace = lookup(each.value, "pvc_namespace", each.key) 339 | } 340 | spec { 341 | access_modes = kubernetes_persistent_volume.publick8s_datadisks[each.key].spec[0].access_modes 342 | volume_name = kubernetes_persistent_volume.publick8s_datadisks[each.key].metadata.0.name 343 | storage_class_name = kubernetes_persistent_volume.publick8s_datadisks[each.key].spec[0].storage_class_name 344 | resources { 345 | requests = { 346 | storage = kubernetes_persistent_volume.publick8s_datadisks[each.key].spec[0].capacity.storage 347 | } 348 | } 349 | } 350 | } 351 | # Permissions/Role required to allow AKS CSI driver to access the Azure disk 352 | resource "azurerm_role_definition" "publick8s_datadisks" { 353 | for_each = local.aks_clusters["publick8s"].azuredisk_volumes 354 | 355 | name = "publick8s-read-disk-${each.key}" 356 | scope = each.value.disk_rg_id 357 | 358 | permissions { 359 | actions = [ 360 | "Microsoft.Compute/disks/read", 361 | "Microsoft.Compute/disks/write", 362 | ] 363 | } 364 | } 365 | resource "azurerm_role_assignment" "publick8s_datadisks" { 366 | for_each = local.aks_clusters["publick8s"].azuredisk_volumes 367 | 368 | scope = each.value.disk_rg_id 369 | role_definition_id = azurerm_role_definition.publick8s_datadisks[each.key].role_definition_resource_id 370 | principal_id = azurerm_kubernetes_cluster.publick8s.identity[0].principal_id 371 | } 372 | 373 | # Retrieve effective outbound IPs 374 | data "azurerm_public_ip" "publick8s_lb_outbound" { 375 | ## Disable this resource when running in terratest 376 | # to avoid the error "The "for_each" set includes values derived from resource attributes that cannot be determined until apply" 377 | for_each = var.terratest ? toset([]) : toset(concat(flatten(azurerm_kubernetes_cluster.publick8s.network_profile[*].load_balancer_profile[*].effective_outbound_ips))) 378 | 379 | name = element(split("/", each.key), "-1") 380 | resource_group_name = azurerm_kubernetes_cluster.publick8s.node_resource_group 381 | } 382 | -------------------------------------------------------------------------------- /locals.tf: -------------------------------------------------------------------------------- 1 | # Retrieving end dates from updatecli values, easier location to track and update them 2 | data "local_file" "locals_yaml" { 3 | filename = "updatecli/values.yaml" 4 | } 5 | 6 | locals { 7 | public_db_pgsql_admin_login = "psqladmin${random_password.public_db_pgsql_admin_login.result}" 8 | public_db_mysql_admin_login = "mysqladmin${random_password.public_db_mysql_admin_login.result}" 9 | 10 | shared_galleries = { 11 | "dev" = { 12 | description = "Shared images built by pull requests in jenkins-infra/packer-images (consider it untrusted)." 13 | images = ["ubuntu-22.04-amd64", "ubuntu-22.04-arm64", "windows-2019-amd64", "windows-2022-amd64", "windows-2025-amd64"] 14 | } 15 | "staging" = { 16 | description = "Shared images built by the principal code branch in jenkins-infra/packer-images (ready to be tested)." 17 | images = ["ubuntu-22.04-amd64", "ubuntu-22.04-arm64", "windows-2019-amd64", "windows-2022-amd64", "windows-2025-amd64"] 18 | } 19 | "prod" = { 20 | description = "Shared images built by the releases in jenkins-infra/packer-images (⚠️ Used in production.)." 21 | images = ["ubuntu-22.04-amd64", "ubuntu-22.04-arm64", "windows-2019-amd64", "windows-2022-amd64", "windows-2025-amd64"] 22 | } 23 | } 24 | 25 | # Tracked by 'updatecli' from the following source: https://reports.jenkins.io/jenkins-infra-data-reports/azure-net.json 26 | outbound_ips_trusted_ci_jenkins_io = "104.209.128.236" 27 | # Tracked by 'updatecli' from the following source: https://reports.jenkins.io/jenkins-infra-data-reports/azure-net.json 28 | outbound_ips_infra_ci_jenkins_io = "20.57.120.46 52.179.141.53 172.210.200.59 20.10.193.4" 29 | # Tracked by 'updatecli' from the following source: https://reports.jenkins.io/jenkins-infra-data-reports/azure-net.json 30 | outbound_ips_private_vpn_jenkins_io = "52.232.183.117" 31 | 32 | admin_public_ips = { 33 | dduportal = ["82.67.112.167"], 34 | smerle33 = ["86.207.165.174"], 35 | mwaite = ["162.142.59.220"], 36 | } 37 | 38 | # TODO: track with updatecli 39 | external_services = { 40 | "pkg.origin.jenkins.io" = "52.202.51.185", 41 | "archives.jenkins.io" = "46.101.121.132", 42 | } 43 | 44 | # Ref. https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-githubs-ip-addresses 45 | # Only IPv4 46 | github_ips = { 47 | webhooks = ["140.82.112.0/20", "143.55.64.0/20", "185.199.108.0/22", "192.30.252.0/22"] 48 | } 49 | gpg_keyserver_ips = { 50 | "keyserver.ubuntu.com" = ["162.213.33.8", "162.213.33.9"] 51 | } 52 | 53 | default_tags = { 54 | scope = "terraform-managed" 55 | repository = "jenkins-infra/azure" 56 | } 57 | 58 | admin_username = "jenkins-infra-team" 59 | 60 | aks_clusters = { 61 | "infracijenkinsio_agents_2" = { 62 | name = "infracijenkinsio-agents-2", 63 | kubernetes_version = "1.33.5", 64 | # https://learn.microsoft.com/en-us/azure/aks/concepts-network-azure-cni-overlay#pods 65 | pod_cidr = "10.100.0.0/14", # 10.100.0.1 - 10.103.255.255 66 | }, 67 | "privatek8s" = { 68 | name = "privatek8s", 69 | kubernetes_version = "1.33.5", 70 | # https://learn.microsoft.com/en-us/azure/aks/concepts-network-azure-cni-overlay#pods 71 | pod_cidr = "10.100.0.0/14", # 10.100.0.1 - 10.103.255.255 72 | }, 73 | "publick8s" = { 74 | name = "publick8s", 75 | kubernetes_version = "1.33.5", 76 | # https://learn.microsoft.com/en-us/azure/aks/concepts-network-azure-cni-overlay#pods 77 | pod_cidrs = [ 78 | "10.100.0.0/14", # 10.100.0.1 - 10.103.255.255 79 | "fd12:3456:789a::/64", # Dual stack is required to provide public IPv6 LBs 80 | ], 81 | azurefile_volumes = { 82 | "get-jenkins-io" = {}, 83 | "updates-jenkins-io" = {}, 84 | "www-jenkins-io" = {}, 85 | "staging-pkg-origin-jenkins-io" = {}, 86 | "staging-get-jenkins-io" = {}, 87 | "pkg-origin-jenkins-io" = {}, 88 | "builds-reports-jenkins-io" = { 89 | capacity = azurerm_storage_share.builds_reports_jenkins_io.quota, 90 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 91 | volume_handle = "${azurerm_storage_account.builds_reports_jenkins_io.resource_group_name}#${azurerm_storage_account.builds_reports_jenkins_io.name}#${azurerm_storage_share.builds_reports_jenkins_io.name}" 92 | mount_options = [ 93 | "dir_mode=0777", 94 | "file_mode=0777", 95 | "uid=0", 96 | "gid=0", 97 | "mfsymlinks", 98 | "cache=strict", # Default on usual kernels but worth setting it explicitly 99 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 100 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 101 | ], 102 | volume_attributes = { 103 | resourceGroup = azurerm_storage_account.builds_reports_jenkins_io.resource_group_name, 104 | shareName = azurerm_storage_share.builds_reports_jenkins_io.name, 105 | }, 106 | secret_name = azurerm_storage_account.builds_reports_jenkins_io.name, 107 | secret_namespace = "builds-reports-jenkins-io", 108 | storage_account_key = azurerm_storage_account.builds_reports_jenkins_io.primary_access_key, 109 | }, 110 | "contributors-jenkins-io" = { 111 | capacity = azurerm_storage_share.contributors_jenkins_io.quota, 112 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 113 | volume_handle = "${azurerm_storage_account.contributors_jenkins_io.resource_group_name}#${azurerm_storage_account.contributors_jenkins_io.name}#${azurerm_storage_share.contributors_jenkins_io.name}" 114 | mount_options = [ 115 | "dir_mode=0777", 116 | "file_mode=0777", 117 | "uid=0", 118 | "gid=0", 119 | "mfsymlinks", 120 | "cache=strict", # Default on usual kernels but worth setting it explicitly 121 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 122 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 123 | ], 124 | volume_attributes = { 125 | resourceGroup = azurerm_storage_account.contributors_jenkins_io.resource_group_name, 126 | shareName = azurerm_storage_share.contributors_jenkins_io.name, 127 | }, 128 | secret_name = azurerm_storage_account.contributors_jenkins_io.name, 129 | secret_namespace = "contributors-jenkins-io", 130 | storage_account_key = azurerm_storage_account.contributors_jenkins_io.primary_access_key, 131 | }, 132 | "docs-jenkins-io" = { 133 | capacity = azurerm_storage_share.docs_jenkins_io.quota, 134 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 135 | volume_handle = "${azurerm_storage_account.docs_jenkins_io.resource_group_name}#${azurerm_storage_account.docs_jenkins_io.name}#${azurerm_storage_share.docs_jenkins_io.name}" 136 | mount_options = [ 137 | "dir_mode=0777", 138 | "file_mode=0777", 139 | "uid=0", 140 | "gid=0", 141 | "mfsymlinks", 142 | "cache=strict", # Default on usual kernels but worth setting it explicitly 143 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 144 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 145 | ], 146 | volume_attributes = { 147 | resourceGroup = azurerm_storage_account.docs_jenkins_io.resource_group_name, 148 | shareName = azurerm_storage_share.docs_jenkins_io.name, 149 | }, 150 | secret_name = azurerm_storage_account.docs_jenkins_io.name, 151 | secret_namespace = "docs-jenkins-io", 152 | storage_account_key = azurerm_storage_account.docs_jenkins_io.primary_access_key, 153 | }, 154 | "javadoc-jenkins-io" = { 155 | capacity = azurerm_storage_share.javadoc_jenkins_io.quota, 156 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 157 | volume_handle = "${azurerm_storage_account.javadoc_jenkins_io.resource_group_name}#${azurerm_storage_account.javadoc_jenkins_io.name}#${azurerm_storage_share.javadoc_jenkins_io.name}" 158 | mount_options = [ 159 | "dir_mode=0777", 160 | "file_mode=0777", 161 | "uid=0", 162 | "gid=0", 163 | "mfsymlinks", 164 | "cache=strict", # Default on usual kernels but worth setting it explicitly 165 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 166 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 167 | ], 168 | volume_attributes = { 169 | resourceGroup = azurerm_storage_account.javadoc_jenkins_io.resource_group_name, 170 | shareName = azurerm_storage_share.javadoc_jenkins_io.name, 171 | }, 172 | secret_name = azurerm_storage_account.javadoc_jenkins_io.name, 173 | secret_namespace = "javadoc-jenkins-io", 174 | storage_account_key = azurerm_storage_account.javadoc_jenkins_io.primary_access_key, 175 | }, 176 | # LDAP needs a read/write PVC to store its backups 177 | "ldap-jenkins-io-backup" = { 178 | pvc_namespace = "ldap-jenkins-io", 179 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 180 | volume_handle = "${azurerm_storage_account.ldap_jenkins_io.resource_group_name}#${azurerm_storage_account.ldap_jenkins_io.name}#${azurerm_storage_share.ldap_jenkins_io_backups.name}" 181 | # between 3 to 8 years of LDAP ldif backups 182 | # TODO: We should purge backups older than 1 year (username, email and password data) 183 | capacity = "10", 184 | access_modes = ["ReadWriteMany"], 185 | read_only = false, 186 | mount_options = [ 187 | "dir_mode=0777", 188 | "file_mode=0777", 189 | "uid=0", 190 | "gid=0", 191 | "mfsymlinks", 192 | "cache=strict", # Default on usual kernels but worth setting it explicitly 193 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 194 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 195 | ] 196 | volume_attributes = { 197 | resourceGroup = azurerm_storage_account.ldap_jenkins_io.resource_group_name, 198 | shareName = azurerm_storage_share.ldap_jenkins_io_backups.name, 199 | }, 200 | secret_name = azurerm_storage_account.ldap_jenkins_io.name, 201 | secret_namespace = "ldap-jenkins-io", 202 | storage_account_key = azurerm_storage_account.ldap_jenkins_io.primary_access_key, 203 | }, 204 | "plugins-jenkins-io" = { 205 | capacity = azurerm_storage_share.plugins_jenkins_io.quota, 206 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 207 | volume_handle = "${azurerm_storage_account.plugins_jenkins_io.resource_group_name}#${azurerm_storage_account.plugins_jenkins_io.name}#${azurerm_storage_share.plugins_jenkins_io.name}" 208 | mount_options = [ 209 | "dir_mode=0777", 210 | "file_mode=0777", 211 | "uid=0", 212 | "gid=0", 213 | "mfsymlinks", 214 | "cache=strict", # Default on usual kernels but worth setting it explicitly 215 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 216 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 217 | ], 218 | volume_attributes = { 219 | resourceGroup = azurerm_storage_account.plugins_jenkins_io.resource_group_name, 220 | shareName = azurerm_storage_share.plugins_jenkins_io.name, 221 | }, 222 | secret_name = azurerm_storage_account.plugins_jenkins_io.name, 223 | secret_namespace = "plugins-jenkins-io", 224 | storage_account_key = azurerm_storage_account.plugins_jenkins_io.primary_access_key, 225 | }, 226 | "reports-jenkins-io" = { 227 | capacity = azurerm_storage_share.reports_jenkins_io.quota, 228 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 229 | volume_handle = "${azurerm_storage_account.reports_jenkins_io.resource_group_name}#${azurerm_storage_account.reports_jenkins_io.name}#${azurerm_storage_share.reports_jenkins_io.name}" 230 | mount_options = [ 231 | "dir_mode=0777", 232 | "file_mode=0777", 233 | "uid=0", 234 | "gid=0", 235 | "mfsymlinks", 236 | "cache=strict", # Default on usual kernels but worth setting it explicitly 237 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 238 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 239 | ], 240 | volume_attributes = { 241 | resourceGroup = azurerm_storage_account.reports_jenkins_io.resource_group_name, 242 | shareName = azurerm_storage_share.reports_jenkins_io.name, 243 | }, 244 | secret_name = azurerm_storage_account.reports_jenkins_io.name, 245 | secret_namespace = "reports-jenkins-io", 246 | storage_account_key = azurerm_storage_account.reports_jenkins_io.primary_access_key, 247 | }, 248 | "stats-jenkins-io" = { 249 | capacity = azurerm_storage_share.stats_jenkins_io.quota, 250 | # `volumeHandle` must be unique on the cluster for this volume and must looks like: "{resource-group-name}#{account-name}#{file-share-name}" 251 | volume_handle = "${azurerm_storage_account.stats_jenkins_io.resource_group_name}#${azurerm_storage_account.stats_jenkins_io.name}#${azurerm_storage_share.stats_jenkins_io.name}" 252 | mount_options = [ 253 | "dir_mode=0777", 254 | "file_mode=0777", 255 | "uid=0", 256 | "gid=0", 257 | "mfsymlinks", 258 | "cache=strict", # Default on usual kernels but worth setting it explicitly 259 | "nosharesock", # Use new TCP connection for each CIFS mount (need more memory but avoid lost packets to create mount timeouts) 260 | "nobrl", # disable sending byte range lock requests to the server and for applications which have challenges with posix locks 261 | ], 262 | volume_attributes = { 263 | resourceGroup = azurerm_storage_account.stats_jenkins_io.resource_group_name, 264 | shareName = azurerm_storage_share.stats_jenkins_io.name, 265 | }, 266 | secret_name = azurerm_storage_account.stats_jenkins_io.name, 267 | secret_namespace = "stats-jenkins-io", 268 | storage_account_key = azurerm_storage_account.stats_jenkins_io.primary_access_key, 269 | }, 270 | } 271 | azuredisk_volumes = { 272 | "ldap-jenkins-io" = { 273 | disk_id = "${azurerm_managed_disk.ldap_jenkins_io_data.id}", 274 | disk_size = "${azurerm_managed_disk.ldap_jenkins_io_data.disk_size_gb}", 275 | disk_rg_id = "${azurerm_resource_group.ldap_jenkins_io.id}", 276 | } 277 | "weekly-ci-jenkins-io" = { 278 | disk_id = "${azurerm_managed_disk.weekly_ci_jenkins_io.id}", 279 | disk_size = "${azurerm_managed_disk.weekly_ci_jenkins_io.disk_size_gb}", 280 | disk_rg_id = "${azurerm_resource_group.weekly_ci_jenkins_io.id}", 281 | } 282 | } 283 | }, 284 | "compute_zones" = { 285 | system_pool = [1, 2], # Note: Zone 3 is not allowed for system pool. 286 | arm64_pool = [2, 3], 287 | amd64_pool = [1, 2], 288 | } 289 | } 290 | 291 | # These cluster_hostname cannot be on the 'local.aks_cluster' to avoid cyclic dependencies (when expanding the map) 292 | aks_clusters_outputs = { 293 | "infracijenkinsio_agents_2" = { 294 | cluster_hostname = "https://${azurerm_kubernetes_cluster.infracijenkinsio_agents_2.fqdn}:443", # Cannot use the kubeconfig host as it provides a private DNS name 295 | }, 296 | "privatek8s" = { 297 | cluster_hostname = "https://${azurerm_kubernetes_cluster.privatek8s.fqdn}:443", # Cannot use the kubeconfig host as it provides a private DNS name 298 | }, 299 | "publick8s" = { 300 | cluster_hostname = "https://${azurerm_kubernetes_cluster.publick8s.fqdn}:443", # Cannot use the kubeconfig host as it provides a private DNS name 301 | }, 302 | } 303 | 304 | end_dates = yamldecode(data.local_file.locals_yaml.content).end_dates 305 | 306 | app_subnets = { 307 | "release.ci.jenkins.io" = { 308 | "controller" = [data.azurerm_subnet.privatek8s_release_ci_controller_tier.id], 309 | "agents" = [ 310 | # Container agents 311 | data.azurerm_subnet.privatek8s_release_tier.id, 312 | ], 313 | }, 314 | "infra.ci.jenkins.io" = { 315 | "controller" = [data.azurerm_subnet.privatek8s_infra_ci_controller_tier.id], 316 | "agents" = [ 317 | # VM agents (CDF subscription) 318 | data.azurerm_subnet.infra_ci_jenkins_io_ephemeral_agents.id, 319 | # Container agents (CDF subscription) 320 | data.azurerm_subnet.infracijenkinsio_agents_2.id, 321 | ], 322 | }, 323 | "trusted.ci.jenkins.io" = { 324 | "controller" = [data.azurerm_subnet.trusted_ci_jenkins_io_controller.id], 325 | "agents" = [ 326 | # Permanent agents (Update Center generation) 327 | data.azurerm_subnet.trusted_ci_jenkins_io_permanent_agents.id, 328 | # VM agents (CDF subscription) 329 | data.azurerm_subnet.trusted_ci_jenkins_io_ephemeral_agents.id, 330 | ], 331 | }, 332 | "cert.ci.jenkins.io" = { 333 | "controller" = [data.azurerm_subnet.cert_ci_jenkins_io_controller.id], 334 | "agents" = [ 335 | # VM agents (CDF subscription) 336 | data.azurerm_subnet.cert_ci_jenkins_io_ephemeral_agents.id, 337 | ], 338 | }, 339 | } 340 | 341 | infra_ci_jenkins_io_fqdn = "infra.ci.jenkins.io" 342 | infra_ci_jenkins_io_service_short_name = trimprefix(trimprefix(local.infra_ci_jenkins_io_fqdn, "jenkins.io"), ".") 343 | infra_ci_jenkins_io_service_short_stripped_name = replace(local.infra_ci_jenkins_io_service_short_name, ".", "-") 344 | } 345 | --------------------------------------------------------------------------------