├── modules ├── helm-base-roles │ ├── helm │ │ └── base-roles │ │ │ ├── values.yaml │ │ │ ├── templates │ │ │ ├── crb:kubelet-api.yaml │ │ │ ├── crb:bootstrappers.yaml │ │ │ └── _helpers.tpl │ │ │ ├── .helmignore │ │ │ └── Chart.yaml │ ├── locals.tf │ ├── main.tf │ └── variables.tf ├── charlotte │ ├── TODO │ ├── variables.tf │ ├── providers.tf │ ├── README │ ├── resources.tf │ └── locals.tf ├── helm-machine-controller-manager │ ├── helm │ │ └── values.yaml.tftpl │ ├── locals.tf │ ├── main.tf │ └── variables.tf ├── k8s-config-vars │ ├── default │ │ ├── kube-scheduler.yaml │ │ ├── kube-controller-manager.yaml │ │ ├── kube-apiserver.yaml │ │ └── kubelet.yaml │ ├── variables.tf │ ├── locals.vault.instances.tf │ ├── terraform.tf │ ├── locals.cloud.tf │ ├── local.base.tf │ ├── locals.kube-apiserver-flags.tf │ ├── locals.kube-scheduler.tf │ ├── locals.vault.tf │ ├── locals.kube-flags.tf │ ├── locals.kubernetes.ports.tf │ ├── locals.kube-controller-manager-flags.tf │ ├── locals.kubelet-config-flags.tf │ ├── outputs.tf │ ├── locals.path.tf │ ├── locals.addresses.tf │ ├── locals.instances.tf │ └── local.component-versions.tf ├── keycloak-base │ ├── locals.tf │ ├── providers.tf │ └── keycloak.tf ├── helm-vault-issuer │ ├── variables.tf │ ├── helm │ │ └── base-vault-node-csr │ │ │ ├── values.yaml │ │ │ ├── .helmignore │ │ │ ├── templates │ │ │ ├── gatekeeper.mutation.yaml │ │ │ ├── cluster:issuer:vault.yaml │ │ │ └── _helpers.tpl │ │ │ └── Chart.yaml │ └── main.tf ├── k8s-templates │ ├── bashrc │ │ ├── output.tf │ │ ├── variables.tf │ │ ├── locals.tf │ │ └── templates │ │ │ └── bashrc.tftpl │ ├── services │ │ ├── kubelet │ │ │ ├── templates │ │ │ │ └── config.yaml.tftpl │ │ │ ├── output.tf │ │ │ ├── variables.tf │ │ │ └── locals.tf │ │ └── key-keeper │ │ │ ├── output.tf │ │ │ ├── templates │ │ │ └── service.yaml.tftpl │ │ │ ├── variables.tf │ │ │ └── locals.tf │ ├── kubeconfig │ │ ├── output.tf │ │ ├── main.tf │ │ ├── templates │ │ │ └── kubeconfig.tftpl │ │ └── variables.tf │ ├── static-pods │ │ ├── etcd │ │ │ ├── output.tf │ │ │ ├── variables.tf │ │ │ ├── locals.tf │ │ │ └── templates │ │ │ │ └── etcd.yaml.tftpl │ │ └── kubeadm-config │ │ │ ├── output.tf │ │ │ ├── modules.tf │ │ │ ├── variables.tf │ │ │ ├── locals.tf │ │ │ └── templates │ │ │ └── kubeadm-config.yaml.tftpl │ └── cloud-init-master │ │ ├── output.tf │ │ ├── variables.tf │ │ ├── modules.tf │ │ ├── module.static-pod.tf │ │ ├── locals.etcd.tf │ │ ├── modules.kubeconfig.tf │ │ ├── locals.kubeconfig.tf │ │ └── locals.template.tf ├── helm-temlate │ ├── helm │ │ └── values.yaml.tftpl │ ├── variables.tf │ └── main.tf ├── k8s-addons │ ├── variables.tf │ └── main.tf ├── k8s-vault │ ├── variables.tf │ ├── vault.core.approle.tf │ ├── vault.kv.tf │ ├── vault.core.mount.pki.tf │ ├── README.md │ ├── vault.certificate.roles.tf │ └── vault.core.create.ca.tf ├── k8s-vault-master │ ├── variables.tf │ ├── output.tf │ ├── templates │ │ └── vault │ │ │ ├── vault-cluster-policies.tftpl │ │ │ └── vault-bootstarp-approle-all.tftpl │ ├── main.tf │ └── README.md ├── helm-yandex-csi-controller │ ├── helm │ │ └── values.yaml.tftpl │ ├── data.tf │ ├── providers.tf │ ├── locals.tf │ ├── main.tf │ ├── variables.tf │ └── yandex.tf ├── k8s-yandex-master-infra │ ├── modules.tf │ ├── terraform.tf │ ├── yandex.cluster.subnets.tf │ ├── yandex.cluster.instance.master.disk.tf │ ├── output.tf │ ├── data.tf │ ├── yandex.cluster.dns-internal.tf │ ├── variables.tf │ ├── yandex.lock.box.all.tf │ ├── yandex.cluster.lb.tf │ ├── yandex.cluster.instance.masters.tf │ └── locals.tf ├── helm-certmanager │ ├── locals.tf │ ├── main.tf │ ├── variables.tf │ └── helm │ │ └── values.yaml.tftpl ├── helm-gatekeeper │ ├── locals.tf │ ├── main.tf │ ├── variables.tf │ └── helm │ │ └── values.yaml.tftpl ├── helm-yandex-machine-instance │ ├── data.tf │ ├── terraform.tf │ ├── main.tf │ ├── variables.tf │ ├── locals.tf │ ├── helm │ │ └── values.yaml.tftpl │ └── yandex.tf ├── keycloak-vault │ ├── providers.tf │ ├── variables.tf │ ├── vault.keycloak.policies.tf │ ├── locals.tf │ ├── vault.keycloak.approle.tf │ ├── templates │ │ └── keycloak-policy.tftpl │ ├── vault.keycloak.certificate.tf │ └── vault.keycloak.role.tf ├── k8s-yandex-base │ ├── providers.tf │ ├── data.tf │ ├── main.tf │ └── variables.tf ├── vault-root-pki │ ├── providers.tf │ ├── vault.core.pki.tf │ └── variables.tf ├── yandex-network-base │ ├── providers.tf │ ├── data.tf │ ├── main.tf │ └── variables.tf ├── helm-yandex-cloud-controller │ ├── providers.tf │ ├── data.tf │ ├── helm │ │ └── values.yaml.tftpl │ ├── main.tf │ ├── locals.tf │ ├── variables.tf │ └── yandex.tf ├── k8s-master-infra-firewall │ ├── terraform.tf │ ├── local.tf │ ├── variables.tf │ └── charlotte.firewall.tf ├── k8s-yandex-cluster-infra │ ├── terraform.tf │ ├── output.tf │ ├── main.tf │ └── variables.tf ├── helm-coredns │ ├── locals.tf │ ├── helm │ │ └── values.yaml.tftpl │ ├── main.tf │ └── variables.tf ├── k8s-ready-status │ ├── variables.tf │ └── main.tf ├── helm-victoria-metrics-stack-operator │ ├── locals.tf │ ├── main.tf │ ├── variables.tf │ └── helm │ │ ├── vm-operator.yaml │ │ └── common.yaml └── helm-yandex-cilium │ ├── main.tf │ ├── variables.tf │ ├── locals.tf │ └── helm │ └── values.yaml.tftpl ├── README.md └── .gitignore /modules/helm-base-roles/helm/base-roles/values.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/charlotte/TODO: -------------------------------------------------------------------------------- 1 | [] - Проверить работу SG-1 to SG-1 -------------------------------------------------------------------------------- /modules/helm-machine-controller-manager/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/default/kube-scheduler.yaml: -------------------------------------------------------------------------------- 1 | leader-elect: "true" -------------------------------------------------------------------------------- /modules/keycloak-base/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | idp_provider_realm = "master" 3 | } -------------------------------------------------------------------------------- /modules/helm-vault-issuer/variables.tf: -------------------------------------------------------------------------------- 1 | variable "global_vars" { 2 | type = any 3 | } 4 | -------------------------------------------------------------------------------- /modules/k8s-templates/bashrc/output.tf: -------------------------------------------------------------------------------- 1 | output "k8s" { 2 | value = local.bashrc-k8s 3 | } -------------------------------------------------------------------------------- /modules/k8s-templates/services/kubelet/templates/config.yaml.tftpl: -------------------------------------------------------------------------------- 1 | ${kubelet_config_flags} -------------------------------------------------------------------------------- /modules/k8s-config-vars/variables.tf: -------------------------------------------------------------------------------- 1 | variable "extra_args" { 2 | type = any 3 | default = {} 4 | } 5 | -------------------------------------------------------------------------------- /modules/k8s-templates/kubeconfig/output.tf: -------------------------------------------------------------------------------- 1 | output "kubeconfig" { 2 | value = local.kube-apiserver-manifest 3 | } -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/etcd/output.tf: -------------------------------------------------------------------------------- 1 | output "manifest" { 2 | value = local.manifest-map 3 | 4 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #### About 2 | 3 | Данный репозиторий для агрегации переиспользуемых модулей терраформ 4 | для проекта Fraima -------------------------------------------------------------------------------- /modules/charlotte/variables.tf: -------------------------------------------------------------------------------- 1 | variable "security_groups" { 2 | # default = [] 3 | type = any 4 | } 5 | 6 | -------------------------------------------------------------------------------- /modules/helm-temlate/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | %{ if extra_values != {} } 2 | ${yamlencode(extra_values)} 3 | %{ endif ~} 4 | -------------------------------------------------------------------------------- /modules/k8s-templates/services/kubelet/output.tf: -------------------------------------------------------------------------------- 1 | output "kubelet-config" { 2 | value = local.kubelet-config 3 | } 4 | -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/kubeadm-config/output.tf: -------------------------------------------------------------------------------- 1 | output "manifest" { 2 | value = local.manifest-map 3 | 4 | } -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/output.tf: -------------------------------------------------------------------------------- 1 | output "cloud-init-render" { 2 | value = local.cloud-init-master-map 3 | } 4 | -------------------------------------------------------------------------------- /modules/k8s-addons/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | type = any 3 | } 4 | 5 | variable "extra_values" { 6 | type = any 7 | } 8 | -------------------------------------------------------------------------------- /modules/k8s-templates/bashrc/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "K8S: ?" 3 | type = any 4 | default = null 5 | } 6 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.vault.instances.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | vault_instances = { 3 | k8s_vault_master_secret_id = { 4 | enabled = true 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /modules/k8s-vault/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "Module:k8s-config-vars: base vars" 3 | type = any 4 | default = {} 5 | } 6 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | utils = { 5 | source = "cloudposse/utils" 6 | } 7 | 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/kubeadm-config/modules.tf: -------------------------------------------------------------------------------- 1 | # module "kubelet" { 2 | # source = "../../services/kubelet" 3 | # k8s_global_vars = var.k8s_global_vars 4 | # } -------------------------------------------------------------------------------- /modules/k8s-vault-master/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "Module:k8s-config-vars: base vars" 3 | type = any 4 | default = {} 5 | } 6 | -------------------------------------------------------------------------------- /modules/k8s-vault/vault.core.approle.tf: -------------------------------------------------------------------------------- 1 | resource "vault_auth_backend" "approle" { 2 | type = "approle" 3 | path = var.k8s_global_vars.global_path.base_vault_path_approle 4 | } 5 | -------------------------------------------------------------------------------- /modules/charlotte/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | sgroups = { 4 | version = "1.1.2" 5 | source = "fraima/charlotte" 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /modules/k8s-templates/services/key-keeper/output.tf: -------------------------------------------------------------------------------- 1 | output "key-keeper-config" { 2 | value = local.key-keeper-config-map 3 | } 4 | 5 | output "key-keeper-service" { 6 | value = local.key-keeper-service 7 | } -------------------------------------------------------------------------------- /modules/k8s-templates/services/kubelet/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "k8s_global_vars" { 3 | description = "module:K8S-CERTIFICATE-VARS: base certificate vars" 4 | type = any 5 | default = {} 6 | } 7 | -------------------------------------------------------------------------------- /modules/helm-base-roles/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_values = try(yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 3 | })), {} ) 4 | 5 | merge_values = merge(local.base_values, var.extra_values) 6 | } -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | serviceAccountJSON: ${ base64encode(jsonencode(yandex_cloud_controller_sa.service_account_json)) } 2 | folderID: ${ base64encode(yandex_cloud_controller_sa.folder_id) } 3 | -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | type = any 3 | default = null 4 | } 5 | 6 | variable "actual-release" { 7 | type = string 8 | default = "v0_1" 9 | } 10 | 11 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/modules.tf: -------------------------------------------------------------------------------- 1 | module "k8s-cloud-init-master" { 2 | source = "../k8s-templates/cloud-init-master" 3 | k8s_global_vars = var.k8s_global_vars 4 | } 5 | -------------------------------------------------------------------------------- /modules/helm-certmanager/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 3 | })) 4 | 5 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 6 | } -------------------------------------------------------------------------------- /modules/helm-gatekeeper/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 3 | })) 4 | 5 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 6 | } -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/data.tf: -------------------------------------------------------------------------------- 1 | 2 | data "yandex_iam_service_account" "yandex-k8s-controllers" { 3 | name = try(var.extra_values.module_values.yandex_cloud_controller_sa_name ,var.yandex_cloud_controller_sa_name) 4 | } 5 | -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/data.tf: -------------------------------------------------------------------------------- 1 | 2 | data "yandex_iam_service_account" "yandex-k8s-controllers" { 3 | name = try(var.extra_values.module_values.yandex_cloud_controller_sa_name ,var.yandex_cloud_controller_sa_name) 4 | } 5 | -------------------------------------------------------------------------------- /modules/keycloak-vault/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | vault = { 5 | source = "hashicorp/vault" 6 | version = "3.12.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | } 11 | -------------------------------------------------------------------------------- /modules/k8s-yandex-base/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | } 11 | -------------------------------------------------------------------------------- /modules/k8s-templates/services/kubelet/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | kubelet-config = templatefile("${path.module}/templates/config.yaml.tftpl", { 3 | kubelet_config_flags = var.k8s_global_vars.kube_flags.kubelet_config_flags 4 | }) 5 | } -------------------------------------------------------------------------------- /modules/keycloak-base/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | keycloak = { 5 | source = "mrparkers/keycloak" 6 | version = "3.6.0" 7 | } 8 | 9 | } 10 | required_version = ">= 0.13" 11 | } 12 | -------------------------------------------------------------------------------- /modules/keycloak-vault/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | variable "external_keycloak_url" { 4 | type = string 5 | default = "auth.dobry-kot.ru" 6 | } 7 | 8 | variable "root_ca_path" { 9 | type = string 10 | default = "pki-root" 11 | } 12 | -------------------------------------------------------------------------------- /modules/vault-root-pki/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | 5 | vault = { 6 | source = "hashicorp/vault" 7 | version = "3.12.0" 8 | } 9 | 10 | } 11 | required_version = ">= 1.3.4" 12 | } 13 | -------------------------------------------------------------------------------- /modules/yandex-network-base/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | 11 | } 12 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | } 11 | -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | } 11 | -------------------------------------------------------------------------------- /modules/k8s-master-infra-firewall/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | sgroups = { 5 | version = "1.1.2" 6 | source = "fraima/charlotte" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | 11 | } 12 | -------------------------------------------------------------------------------- /modules/k8s-yandex-cluster-infra/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | 11 | } 12 | -------------------------------------------------------------------------------- /modules/helm-machine-controller-manager/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_values = try(yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 3 | })), {} ) 4 | 5 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 6 | } -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | } 9 | required_version = ">= 0.13" 10 | 11 | } 12 | -------------------------------------------------------------------------------- /modules/charlotte/README: -------------------------------------------------------------------------------- 1 | mkdir -p ~/.terraform.d/plugins/registry.terraform.io/fraima/charlotte/1.0.0/linux_amd64 2 | 3 | cp bin/terraform-provider-sgroups ~/.terraform.d/plugins/registry.terraform.io/fraima/charlotte/1.1.0/linux_amd64/terraform-provider-charlotte_v1.1.0 4 | 5 | -------------------------------------------------------------------------------- /modules/k8s-yandex-cluster-infra/output.tf: -------------------------------------------------------------------------------- 1 | output "k8s_global_vars" { 2 | value = module.k8s-global-vars 3 | } 4 | 5 | output "kube-apiserver-lb" { 6 | value = module.k8s-masters.kube-apiserver-lb 7 | } 8 | 9 | output "k8s-vault" { 10 | value = module.k8s-vault 11 | } -------------------------------------------------------------------------------- /modules/helm-vault-issuer/helm/base-vault-node-csr/values.yaml: -------------------------------------------------------------------------------- 1 | 2 | vault: 3 | approle: 4 | secretId: ${vaut_approle_secretid} 5 | roleId: ${vaut_approle_roleid} 6 | path: ${vaut_approle_path} 7 | sign: 8 | path: ${vaut_sign_path} 9 | server: ${vaut_server} 10 | -------------------------------------------------------------------------------- /modules/helm-coredns/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 3 | clusterIP = var.global_vars.k8s-addresses.dns_address 4 | })) 5 | 6 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 7 | } -------------------------------------------------------------------------------- /modules/k8s-yandex-base/data.tf: -------------------------------------------------------------------------------- 1 | data "yandex_resourcemanager_cloud" "current" { 2 | name = var.cloud.extra-args.name 3 | } 4 | 5 | data "yandex_resourcemanager_folder" "current" { 6 | name = var.cloud.extra-args.folder_name 7 | cloud_id = data.yandex_resourcemanager_cloud.current.id 8 | } 9 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.cloud.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | k8s_provider = { 3 | service_account_name = try(var.extra_args.serviceaccount_k8s_controllers_name, "k8s-controllers") 4 | namespace = try(var.extra_args.namespace, "kube-fraime-controllers") 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /modules/yandex-network-base/data.tf: -------------------------------------------------------------------------------- 1 | 2 | data "yandex_resourcemanager_cloud" "current" { 3 | name = var.cloud.extra-args.name 4 | } 5 | 6 | data "yandex_resourcemanager_folder" "current" { 7 | name = var.cloud.extra-args.folder_name 8 | cloud_id = data.yandex_resourcemanager_cloud.current.id 9 | } 10 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/local.base.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base = { 3 | ssh_username = try(var.extra_args.ssh_username, null) == null ? "dk" : var.extra_args.ssh_username 4 | ssh_rsa_path = try(var.extra_args.ssh_rsa_path, null) == null ? "~/.ssh/id_rsa.pub" : var.extra_args.ssh_rsa_path 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /modules/k8s-ready-status/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "module:K8S-CERTIFICATE-VARS: base certificate vars" 3 | type = any 4 | default = {} 5 | } 6 | 7 | variable "cluster_instances" { 8 | type = any 9 | } 10 | 11 | variable "kube_apiserver_lb" { 12 | type = any 13 | } 14 | 15 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/terraform.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | 3 | required_providers { 4 | yandex = { 5 | source = "yandex-cloud/yandex" 6 | version = "0.81.0" 7 | } 8 | sgroups = { 9 | version = "1.1.2" 10 | source = "fraima/charlotte" 11 | } 12 | } 13 | required_version = ">= 0.13" 14 | 15 | } 16 | -------------------------------------------------------------------------------- /modules/keycloak-vault/vault.keycloak.policies.tf: -------------------------------------------------------------------------------- 1 | resource "vault_policy" "auth" { 2 | for_each = local.certificates 3 | name = "${var.root_ca_path}/${each.key}" 4 | 5 | policy = templatefile("${path.module}/templates/keycloak-policy.tftpl", { 6 | pki_path = var.root_ca_path 7 | certificate_role = each.key 8 | }) 9 | } 10 | -------------------------------------------------------------------------------- /modules/k8s-vault-master/output.tf: -------------------------------------------------------------------------------- 1 | output "secret_id_all" { 2 | description = "the secret_id list was generated from the masters" 3 | value = vault_approle_auth_backend_role_secret_id.all_masters 4 | } 5 | 6 | output "role_id_all" { 7 | description = "the role_id list was generated from the masters" 8 | value = vault_approle_auth_backend_role.all_masters 9 | } 10 | -------------------------------------------------------------------------------- /modules/keycloak-vault/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | 3 | policy_names = flatten([ 4 | for policy_key, policy_value in vault_policy.auth : 5 | {"${policy_value.name}": ""} 6 | ] 7 | ) 8 | policy_names_map = { for item in local.policy_names : 9 | keys(item)[0] => values(item)[0] 10 | } 11 | 12 | policy_name_list = keys(local.policy_names_map) 13 | } -------------------------------------------------------------------------------- /modules/k8s-master-infra-firewall/local.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | internal = flatten([ 3 | for network in sgroups_network.masters_internal: 4 | network.name 5 | 6 | ]) 7 | external = flatten([ 8 | for network in sgroups_network.masters_external: 9 | network.name 10 | 11 | ]) 12 | 13 | networks_master = concat(local.internal, local.external) 14 | } 15 | -------------------------------------------------------------------------------- /modules/k8s-templates/bashrc/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | bashrc-k8s = templatefile("${path.module}/templates/bashrc.tftpl", { 3 | ssl = var.k8s_global_vars.ssl 4 | base_path = var.k8s_global_vars.global_path 5 | main_path = var.k8s_global_vars.main_path 6 | etcd_server_port = var.k8s_global_vars.kubernetes-ports.etcd-server-port 7 | }) 8 | } 9 | -------------------------------------------------------------------------------- /modules/k8s-templates/services/key-keeper/templates/service.yaml.tftpl: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=key-keeper-agent 3 | 4 | Wants=network-online.target 5 | After=network-online.target 6 | 7 | [Service] 8 | ExecStart=/usr/bin/key-keeper -config-dir ${base_local_path_certs} -config-regexp .*vault-config 9 | 10 | Restart=always 11 | StartLimitInterval=0 12 | RestartSec=10 13 | 14 | [Install] 15 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /modules/k8s-master-infra-firewall/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "module:K8S-CERTIFICATE-VARS: base certificate vars" 3 | type = any 4 | default = {} 5 | } 6 | 7 | variable "cluster_instances_internal" { 8 | type = any 9 | } 10 | 11 | variable "cluster_instances_external" { 12 | type = any 13 | } 14 | 15 | variable "cluster_api_ip" { 16 | type = any 17 | } 18 | -------------------------------------------------------------------------------- /modules/helm-base-roles/helm/base-roles/templates/crb:kubelet-api.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: custom:kube-apiserver-kubelet-client 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: system:kubelet-api-admin 10 | subjects: 11 | - kind: User 12 | apiGroup: rbac.authorization.k8s.io 13 | name: custom:kube-apiserver-kubelet-client 14 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/data.tf: -------------------------------------------------------------------------------- 1 | 2 | data "yandex_iam_service_account" "yandex-k8s-controllers" { 3 | name = try(var.extra_values.module_values.yandex_cloud_controller_sa_name ,var.yandex_cloud_controller_sa_name) 4 | } 5 | 6 | data "yandex_vpc_network" "cluster-vpc" { 7 | name = var.yandex_default_vpc_name 8 | } 9 | 10 | data "yandex_vpc_route_table" "cluster-vpc-route-table" { 11 | name = var.yandex_default_route_table_name 12 | } 13 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.kube-apiserver-flags.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | default_kube_apiserver_flags = { 3 | secure-port = "${local.kubernetes-ports.kube-apiserver-port}" 4 | } 5 | } 6 | 7 | data "utils_deep_merge_yaml" "kube_apiserver_flags" { 8 | input = [ 9 | yamlencode(local.default_kube_apiserver_flags), 10 | file("${path.module}/default/kube-apiserver.yaml"), 11 | yamlencode(try(var.extra_args.kube_apiserver_flags, {})) 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.kube-scheduler.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | default_kube_scheduler_flags = { 3 | secure-port = "${local.kubernetes-ports.kube-scheduler-port}" 4 | } 5 | } 6 | 7 | data "utils_deep_merge_yaml" "kube_scheduler_flags" { 8 | input = [ 9 | yamlencode(local.default_kube_scheduler_flags), 10 | file("${path.module}/default/kube-scheduler.yaml"), 11 | yamlencode(try(var.extra_args.kube_scheduler_flags, {})) 12 | ] 13 | } 14 | 15 | -------------------------------------------------------------------------------- /modules/helm-base-roles/main.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "base-roles" { 2 | 3 | name = var.release_name 4 | chart = "${path.module}/helm/base-roles" 5 | namespace = var.namespace 6 | 7 | values = [ 8 | try(templatefile("${path.module}/helm/base-roles/values.yaml", { 9 | extra_values = local.merge_values 10 | })) 11 | 12 | ] 13 | 14 | timeout = 6000 15 | wait = true 16 | atomic = true 17 | } 18 | -------------------------------------------------------------------------------- /modules/keycloak-vault/vault.keycloak.approle.tf: -------------------------------------------------------------------------------- 1 | resource "vault_auth_backend" "auth" { 2 | type = "approle" 3 | path = "${var.root_ca_path}/keycloak/approle" 4 | } 5 | 6 | resource "vault_approle_auth_backend_role" "auth" { 7 | backend = "${vault_auth_backend.auth.path}" 8 | role_name = "keycloak" 9 | token_policies = local.policy_name_list 10 | secret_id_bound_cidrs = [] 11 | token_bound_cidrs = [] 12 | } 13 | -------------------------------------------------------------------------------- /modules/helm-base-roles/helm/base-roles/templates/crb:bootstrappers.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # enable bootstrapping nodes to create CSR 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRoleBinding 5 | metadata: 6 | name: create-csrs-for-bootstrapping 7 | subjects: 8 | - kind: Group 9 | name: system:bootstrappers 10 | apiGroup: rbac.authorization.k8s.io 11 | roleRef: 12 | kind: ClusterRole 13 | name: system:node-bootstrapper 14 | apiGroup: rbac.authorization.k8s.io -------------------------------------------------------------------------------- /modules/helm-victoria-metrics-stack-operator/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | default_values = yamldecode(templatefile("${path.module}/helm/common.yaml", { 3 | })) 4 | 5 | base_values = yamldecode(templatefile("${path.module}/helm/vm-operator.yaml", { 6 | release_name = try(var.extra_values.release_name ,var.release_name) 7 | })) 8 | 9 | merge_default = merge(local.default_values, local.base_values) 10 | merge_values = merge(local.merge_default, var.extra_values) 11 | } -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.vault.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | vault-config = { 3 | vault_server = try(var.extra_args.vault_server, null) == null ? "vault.ru" : var.extra_args.vault_server 4 | vault_server_insecure = try(var.extra_args.vault_server_insecure, null) == null ? true : var.extra_args.vault_server_insecure 5 | caBundle = try(var.extra_args.ca_bundle_path, null) == null ? "" : var.extra_args.ca_bundle_path 6 | } 7 | } -------------------------------------------------------------------------------- /modules/k8s-templates/services/key-keeper/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | variable "instance_type" { 4 | description = "K8S: node type" 5 | type = string 6 | default = null 7 | } 8 | 9 | variable "k8s_global_vars" { 10 | description = "module:K8S-CERTIFICATE-VARS: base certificate vars" 11 | type = any 12 | default = {} 13 | } 14 | 15 | variable "instance_list_map" { 16 | description = "K8S: node type" 17 | type = any 18 | default = {} 19 | } 20 | -------------------------------------------------------------------------------- /modules/helm-base-roles/helm/base-roles/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /modules/helm-temlate/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | variable "release_name" { 4 | type = string 5 | 6 | } 7 | 8 | variable "chart_repo" { 9 | type = string 10 | 11 | } 12 | 13 | variable "chart_name" { 14 | type = string 15 | 16 | } 17 | 18 | variable "chart_version" { 19 | type = string 20 | 21 | } 22 | 23 | variable "namespace" { 24 | type = string 25 | 26 | } 27 | 28 | variable "global_vars" { 29 | type = any 30 | } 31 | 32 | variable "extra_values" { 33 | type = any 34 | 35 | } 36 | -------------------------------------------------------------------------------- /modules/helm-vault-issuer/helm/base-vault-node-csr/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | serviceAccountJSON: ${ base64encode(jsonencode(yandex_cloud_controller_sa.service_account_json)) } 2 | folderID: ${ base64encode(yandex_cloud_controller_sa.folder_id) } 3 | 4 | vpcID: ${ base64encode(yandex_cloud_controller_sa.vpc_id) } 5 | routeTableID: ${ base64encode(yandex_cloud_controller_sa.route_table_id) } 6 | 7 | clusterName: ${cluster_name} 8 | podCIDR: ${pod_cidr} 9 | k8sApiServer: ${k8s_api_server} 10 | k8sApiServerPort: ${k8s_api_server_port} 11 | -------------------------------------------------------------------------------- /modules/k8s-templates/kubeconfig/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | kube-apiserver-manifest = templatefile("${path.module}/templates/kubeconfig.tftpl", { 3 | component-name = var.component-name 4 | certificate-authority = var.certificate-authority 5 | kube-apiserver = var.kube-apiserver 6 | kube-apiserver-port = var.kube-apiserver-port 7 | client-certificate = var.client-certificate 8 | client-key = var.client-key 9 | }) 10 | 11 | } 12 | -------------------------------------------------------------------------------- /modules/k8s-yandex-base/main.tf: -------------------------------------------------------------------------------- 1 | resource "yandex_iam_service_account" "yandex-k8s-controllers" { 2 | name = var.k8s-service-account.extra-args.name 3 | description = var.k8s-service-account.extra-args.description 4 | } 5 | 6 | resource "yandex_resourcemanager_folder_iam_binding" "yandex-k8s-controllers" { 7 | folder_id = data.yandex_resourcemanager_folder.current.id 8 | role = "admin" 9 | 10 | members = [ 11 | "serviceAccount:${yandex_iam_service_account.yandex-k8s-controllers.id}", 12 | ] 13 | } -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/yandex.cluster.subnets.tf: -------------------------------------------------------------------------------- 1 | resource "yandex_vpc_subnet" "master-subnets" { 2 | for_each = local.subnets_set_map 3 | 4 | name = join("-", [local.cluster_name, substr(sha256(split(":", each.key)[0]), 0, 8)]) 5 | 6 | v4_cidr_blocks = [split(":", each.key)[0]] 7 | zone = split(":", each.key)[1] 8 | 9 | network_id = data.yandex_vpc_network.cluster-vpc.id 10 | route_table_id = data.yandex_vpc_route_table.cluster-vpc-route-table.id 11 | } 12 | -------------------------------------------------------------------------------- /modules/helm-temlate/main.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "template" { 2 | name = var.release_name 3 | 4 | repository = var.chart_repo 5 | chart = var.chart_name 6 | version = var.chart_version 7 | 8 | namespace = var.namespace 9 | create_namespace = true 10 | 11 | values = [ 12 | templatefile("${path.module}/helm/values.yaml.tftpl", { 13 | extra_values = var.extra_values 14 | }) 15 | 16 | ] 17 | 18 | timeout = 6000 19 | wait = true 20 | atomic = true 21 | } 22 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.kube-flags.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | kube_flags = { 3 | kube_apiserver_flags = yamlencode(yamldecode(data.utils_deep_merge_yaml.kube_apiserver_flags.output)) 4 | kube_controller_manager_flags = yamlencode(yamldecode(data.utils_deep_merge_yaml.kube_controller_manager_flags.output)) 5 | kube_scheduler_flags = yamlencode(yamldecode(data.utils_deep_merge_yaml.kube_scheduler_flags.output)) 6 | kubelet_config_flags = yamlencode(yamldecode(data.utils_deep_merge_yaml.kubelet_config_flags.output)) 7 | } 8 | } -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/modules.tf: -------------------------------------------------------------------------------- 1 | module "kubelet-service-master" { 2 | source = "../services/kubelet" 3 | k8s_global_vars = var.k8s_global_vars 4 | } 5 | 6 | module "key-keeper-service-master" { 7 | source = "../services/key-keeper" 8 | instance_type = "master" 9 | k8s_global_vars = var.k8s_global_vars 10 | instance_list_map = var.k8s_global_vars.master_vars.master_instance_extra_list_map 11 | } 12 | 13 | module "bashrc" { 14 | source = "../bashrc" 15 | k8s_global_vars = var.k8s_global_vars 16 | 17 | } 18 | -------------------------------------------------------------------------------- /modules/helm-coredns/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | tolerations: 2 | - key: node-role.kubernetes.io/master 3 | effect: NoSchedule 4 | - key: "CriticalAddonsOnly" 5 | operator: "Exists" 6 | - key: node.cloudprovider.kubernetes.io/uninitialized 7 | value: "true" 8 | effect: NoSchedule 9 | 10 | fullnameOverride: coredns 11 | 12 | replicaCount: 3 13 | service: 14 | # TODO вывести в переменную под указанную подсеть 15 | clusterIP: ${clusterIP} 16 | 17 | readinessProbe: 18 | initialDelaySeconds: 0 19 | failureThreshold: 15 20 | periodSeconds: 5 -------------------------------------------------------------------------------- /modules/keycloak-vault/templates/keycloak-policy.tftpl: -------------------------------------------------------------------------------- 1 | 2 | path "${pki_path}/sign/${certificate_role}" { 3 | capabilities = ["update"] 4 | } 5 | 6 | path "${pki_path}/${certificate_role}/certs" { 7 | capabilities = ["list"] 8 | } 9 | 10 | path "auth/${pki_path}/approle/login" { 11 | capabilities = [ "update" ] 12 | } 13 | 14 | path "auth/token/renew" { 15 | capabilities = ["update"] 16 | } 17 | 18 | path "auth/token/renew-self" { 19 | capabilities = ["update"] 20 | } 21 | 22 | path "${pki_path}/cert/ca" { 23 | capabilities = ["read"] 24 | } -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/kubeadm-config/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "K8S: ?" 3 | type = any 4 | default = null 5 | } 6 | 7 | 8 | variable "kubernetes_version" { 9 | description = "K8S: ?" 10 | type = string 11 | default = null 12 | } 13 | 14 | variable "instance_list_map" { 15 | description = "K8S: instances" 16 | type = any 17 | default = {} 18 | } 19 | 20 | variable "etcd_list_servers" { 21 | description = "K8S: etcd_list_servers" 22 | type = any 23 | default = {} 24 | } 25 | -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | k8s_api_server = var.global_vars.k8s-addresses.kube_apiserver_lb_fqdn 3 | k8s_api_server_port = var.global_vars.kubernetes-ports.kube-apiserver-port-lb 4 | 5 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 6 | yandex_cloud_controller_sa = local.yandex_k8s_csi_controller_sa_payload 7 | kubeApiServerIP = "https://${local.k8s_api_server}:${local.k8s_api_server_port}" 8 | })) 9 | 10 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 11 | } -------------------------------------------------------------------------------- /modules/helm-vault-issuer/helm/base-vault-node-csr/templates/gatekeeper.mutation.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: mutations.gatekeeper.sh/v1beta1 2 | kind: Assign 3 | metadata: 4 | name: node-csr-clusterissuer-mutation 5 | spec: 6 | applyTo: 7 | - groups: ["certificates.k8s.io"] 8 | kinds: ["CertificateSigningRequest"] 9 | versions: ["v1"] 10 | match: 11 | scope: Cluster 12 | kinds: 13 | - apiGroups: ["*"] 14 | kinds: ["CertificateSigningRequest"] 15 | 16 | location: "spec.signerName" 17 | parameters: 18 | assign: 19 | value: "clusterissuers.cert-manager.io/vault-issuer" -------------------------------------------------------------------------------- /modules/helm-victoria-metrics-stack-operator/main.tf: -------------------------------------------------------------------------------- 1 | module "victoria-metrics" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/helm-coredns/main.tf: -------------------------------------------------------------------------------- 1 | module "coredns" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/helm-yandex-cilium/main.tf: -------------------------------------------------------------------------------- 1 | module "cilium" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/helm-certmanager/main.tf: -------------------------------------------------------------------------------- 1 | module "certmanager" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/helm-gatekeeper/main.tf: -------------------------------------------------------------------------------- 1 | module "gatekeeper" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.kubernetes.ports.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | kubernetes-ports = { 3 | etcd-server-port = "2383" 4 | etcd-peer-port = "2380" 5 | etcd-metrics-port = "2381" 6 | etcd-server-port-target-lb = "2382" 7 | kube-apiserver-port = "6443" 8 | kube-apiserver-port-lb = "443" 9 | kube-controller-manager-port = "10257" 10 | kube-scheduler-port = "10259" 11 | kubelet-healthz-port = 10248 12 | kubelet-server-port = 10250 13 | yandex-controller-manager = 11258 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /modules/k8s-vault-master/templates/vault/vault-cluster-policies.tftpl: -------------------------------------------------------------------------------- 1 | path "auth/${k8s_global_vars.global_path.base_vault_path_approle}/login" { 2 | capabilities = [ "update" ] 3 | } 4 | 5 | path "auth/token/renew" { 6 | capabilities = ["update"] 7 | } 8 | 9 | path "auth/token/renew-self" { 10 | capabilities = ["update"] 11 | } 12 | 13 | path "${k8s_global_vars.ssl.intermediate.kubernetes-ca.default.path}/sign/${issuer_name}" { 14 | capabilities = ["update"] 15 | } 16 | 17 | path "${k8s_global_vars.ssl.intermediate.kubernetes-ca.default.path}/${issuer_name}/certs" { 18 | capabilities = ["list"] 19 | } 20 | -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/main.tf: -------------------------------------------------------------------------------- 1 | module "yandex-csi-controller" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/helm-base-roles/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "base-roles" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://helm.fraima.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "base-roles" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "0.0.1" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-system" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/helm-yandex-cilium/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "cilium" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://helm.cilium.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "cilium" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "1.12.0" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-sdn" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/main.tf: -------------------------------------------------------------------------------- 1 | module "yandex-cloud-controller" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/yandex.cluster.instance.master.disk.tf: -------------------------------------------------------------------------------- 1 | resource "yandex_compute_disk" "etcd" { 2 | for_each = local.instances_disk_map 3 | 4 | name = "${replace(each.key, "_", "-")}-${local.cluster_name}" 5 | 6 | size = local.master_secondary_disk["${split("_", each.key)[0]}"].size 7 | type = local.master_secondary_disk["${split("_", each.key)[0]}"].type 8 | 9 | zone = try( 10 | var.k8s_global_vars.master_vars.master_group.resources_override[split("_", each.key)[1]].network_interface.zone, 11 | var.k8s_global_vars.master_vars.master_group.default_zone 12 | ) 13 | 14 | labels = {} 15 | 16 | } 17 | -------------------------------------------------------------------------------- /modules/helm-coredns/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "coredns" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://coredns.github.io/helm" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "coredns" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "1.19.4" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-dns" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/helm-machine-controller-manager/main.tf: -------------------------------------------------------------------------------- 1 | module "machine-controller-manager" { 2 | 3 | source = "../helm-temlate" 4 | 5 | release_name = try(var.extra_values.release.release_name ,var.release_name) 6 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 7 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 8 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 9 | namespace = try(var.extra_values.release.namespace ,var.namespace) 10 | 11 | global_vars = var.global_vars 12 | extra_values = local.merge_values 13 | 14 | } 15 | -------------------------------------------------------------------------------- /modules/k8s-templates/kubeconfig/templates/kubeconfig.tftpl: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | clusters: 4 | - cluster: 5 | certificate-authority: ${certificate-authority} 6 | server: https://${kube-apiserver}:${kube-apiserver-port} 7 | name: kubernetes 8 | contexts: 9 | - context: 10 | cluster: kubernetes 11 | namespace: default 12 | user: ${component-name} 13 | name: ${component-name}@kubernetes 14 | current-context: ${component-name}@kubernetes 15 | kind: Config 16 | preferences: {} 17 | users: 18 | - name: ${component-name} 19 | user: 20 | client-certificate: ${client-certificate} 21 | client-key: ${client-key} 22 | -------------------------------------------------------------------------------- /modules/helm-yandex-cilium/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | k8s_api_server = var.global_vars.k8s-addresses.kube_apiserver_lb_fqdn 3 | k8s_api_server_port = var.global_vars.kubernetes-ports.kube-apiserver-port-lb 4 | cluster_name = var.global_vars.cluster_metadata.cluster_name 5 | 6 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 7 | k8s_api_server_fqdn = local.k8s_api_server 8 | k8s_api_server_port = local.k8s_api_server_port 9 | cluster_name = local.cluster_name 10 | })) 11 | 12 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 13 | } -------------------------------------------------------------------------------- /modules/keycloak-vault/vault.keycloak.certificate.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | idp_provider_realm = "master" 3 | 4 | certificates = { 5 | keycloak-server = { 6 | allow_bare_domains = true 7 | allow_glob_domains = true 8 | allowed_domains_template = true 9 | allow_ip_sans = true 10 | allow_localhost = true 11 | server_flag = true 12 | use_csr_common_name = true 13 | allowed_domains = [ 14 | var.external_keycloak_url, 15 | "custom:keycloak-server", 16 | "localhost", 17 | ] 18 | } 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /modules/k8s-yandex-base/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s-service-account" { 2 | type = map(object({ 3 | name = string 4 | description = string 5 | })) 6 | default = { 7 | "extra-args" = { 8 | name = "yandex-k8s-controllers" 9 | description = "service account to manage all in k8s clusters" 10 | } 11 | } 12 | } 13 | 14 | variable "cloud" { 15 | type = map(object({ 16 | name = optional(string, "") 17 | folder_name = optional(string, "") 18 | })) 19 | default = { 20 | "extra-args" = { 21 | folder_name = "example" 22 | name = "cloud-uid-vf465ie7" 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /modules/helm-certmanager/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "cert-manager" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://charts.jetstack.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "cert-manager" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "v1.9.1" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-certmanager" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/module.static-pod.tf: -------------------------------------------------------------------------------- 1 | module "static-pod-etcd" { 2 | source = "../static-pods/etcd" 3 | k8s_global_vars = var.k8s_global_vars 4 | instance_list_map = var.k8s_global_vars.master_vars.master_instance_extra_list_map 5 | etcd_initial_cluster = local.etcd_initial_cluster 6 | } 7 | 8 | module "static-pod-kubeadm-config" { 9 | source = "../static-pods/kubeadm-config" 10 | k8s_global_vars = var.k8s_global_vars 11 | instance_list_map = var.k8s_global_vars.master_vars.master_instance_extra_list_map 12 | etcd_list_servers = local.etcd_list_servers 13 | } 14 | -------------------------------------------------------------------------------- /modules/helm-gatekeeper/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "gatekeeper" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://open-policy-agent.github.io/gatekeeper/charts" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "gatekeeper" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "3.11.0" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-opa" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/k8s-vault/vault.kv.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "tls_private_key" "kube_apiserver_sa_key" { 3 | algorithm = "RSA" 4 | rsa_bits = 2048 5 | } 6 | 7 | resource "vault_kv_secret_v2" "kube_apiserver_sa" { 8 | depends_on = [ 9 | vault_mount.kubernetes-secrets 10 | ] 11 | mount = "${vault_mount.kubernetes-secrets.path}" 12 | name = "kube-apiserver-sa" 13 | cas = 1 14 | delete_all_versions = true 15 | data_json = jsonencode( 16 | { 17 | private = tls_private_key.kube_apiserver_sa_key.private_key_pem 18 | public = tls_private_key.kube_apiserver_sa_key.public_key_pem 19 | } 20 | ) 21 | } 22 | -------------------------------------------------------------------------------- /modules/yandex-network-base/main.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "yandex_vpc_network" "cluster-vpc" { 3 | name = var.vpc.extra-args.name 4 | } 5 | 6 | resource "yandex_vpc_gateway" "cluster-vpc-gateway" { 7 | name = var.gateway.extra-args.name 8 | shared_egress_gateway {} 9 | } 10 | 11 | resource "yandex_vpc_route_table" "cluster-vpc-route-table" { 12 | name = "${var.route-table.extra-args.name}" 13 | network_id = yandex_vpc_network.cluster-vpc.id 14 | static_route { 15 | destination_prefix = "0.0.0.0/0" 16 | gateway_id = yandex_vpc_gateway.cluster-vpc-gateway.id 17 | } 18 | lifecycle { 19 | ignore_changes = [ 20 | static_route 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/output.tf: -------------------------------------------------------------------------------- 1 | output "kube-apiserver-lb" { 2 | value = tolist(tolist(yandex_lb_network_load_balancer.api-external.listener)[0].external_address_spec)[0].address 3 | } 4 | 5 | output "master_cidr_list" { 6 | value = flatten([ 7 | for compute in yandex_compute_instance.master: 8 | "${compute.network_interface[0].ip_address}/32" 9 | ]) 10 | } 11 | 12 | output "yandex_compute_instance_master" { 13 | value = yandex_compute_instance.master 14 | } 15 | 16 | output "cluster_external_instances_map" { 17 | value = local.cluster_external_instances_map 18 | } 19 | 20 | output "cluster_internal_instances_map" { 21 | value = local.cluster_internal_instances_map 22 | } -------------------------------------------------------------------------------- /modules/helm-machine-controller-manager/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "machine-controller-manager" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://helm.fraima.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "cluster-machine-controller" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "0.0.3" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-machine-controller-manager" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/helm-victoria-metrics-stack-operator/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "victoria-metrics-k8s-stack" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://victoriametrics.github.io/helm-charts/" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "victoria-metrics-k8s-stack" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "0.14.17" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-monitoring" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.kube-controller-manager-flags.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | default_kube_controller_manager_flags = { 3 | secure-port = "${local.kubernetes-ports.kube-controller-manager-port}" 4 | cluster-cidr = "${local.k8s_network.pod_cidr}" 5 | node-cidr-mask-size = "${local.k8s_network.node_cidr_mask}" 6 | cluster-name = "${local.cluster_metadata.cluster_name}" 7 | } 8 | } 9 | 10 | data "utils_deep_merge_yaml" "kube_controller_manager_flags" { 11 | input = [ 12 | yamlencode(local.default_kube_controller_manager_flags), 13 | file("${path.module}/default/kube-controller-manager.yaml"), 14 | yamlencode(try(var.extra_args.kube_controller_manager_flags, {})) 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/data.tf: -------------------------------------------------------------------------------- 1 | data "yandex_iam_service_account" "yandex-k8s-controllers" { 2 | name = var.k8s_global_vars.k8s_provider.service_account_name 3 | } 4 | 5 | data "yandex_resourcemanager_cloud" "current" { 6 | name = var.cloud_metadata.cloud_name 7 | } 8 | 9 | data "yandex_resourcemanager_folder" "current" { 10 | name = var.cloud_metadata.folder_name 11 | cloud_id = data.yandex_resourcemanager_cloud.current.id 12 | } 13 | 14 | data "yandex_vpc_network" "cluster-vpc" { 15 | name = var.k8s_global_vars.master_vars.master_group.vpc_name 16 | } 17 | 18 | data "yandex_vpc_route_table" "cluster-vpc-route-table" { 19 | name = var.k8s_global_vars.master_vars.master_group.route_table_name 20 | } 21 | -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/main.tf: -------------------------------------------------------------------------------- 1 | module "yandex-machine-instance" { 2 | 3 | depends_on = [ 4 | kubernetes_secret.cloud-secret, 5 | yandex_iam_service_account_key.yandex-k8s-controllers-key 6 | ] 7 | source = "../helm-temlate" 8 | 9 | release_name = try(var.extra_values.release.release_name ,var.release_name) 10 | chart_repo = try(var.extra_values.release.chart_repo ,var.chart_repo) 11 | chart_name = try(var.extra_values.release.chart_name ,var.chart_name) 12 | chart_version = try(var.extra_values.release.chart_version ,var.chart_version) 13 | namespace = try(var.extra_values.release.namespace ,var.namespace) 14 | 15 | global_vars = var.global_vars 16 | extra_values = local.merge_values 17 | 18 | } 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docs/build/* 2 | k8s/.terraform/* 3 | k8s/.terraform* 4 | k8s/states/* 5 | base/.terraform/* 6 | base/.terraform* 7 | base/states/* 8 | 9 | k8s/providers-dev.tf 10 | base/providers-dev.tf 11 | 1.yaml 12 | test.yaml 13 | terragrunt/* 14 | k8s/terraform.tfstate.d/* 15 | 16 | 17 | k8s-yandex-cluster-naked/* 18 | 19 | charlotte-example/* 20 | charlotte-demo/.terraform.lock.hcl 21 | charlotte-demo/export.sh 22 | charlotte-demo/hbf-server.tf 23 | charlotte-demo/main.tf 24 | charlotte-demo/providers.tf 25 | charlotte-demo/README.md 26 | charlotte-demo/terraform.tfstate 27 | charlotte-demo/terraform.tfstate.backup 28 | charlotte-demo/.terraform/terraform.tfstate 29 | charlotte-demo/.terraform/providers/registry.terraform.io/fraima/charlotte/*/linux_amd64 30 | 1.sh 31 | 2.sh 32 | 2.md 33 | -------------------------------------------------------------------------------- /modules/helm-vault-issuer/helm/base-vault-node-csr/templates/cluster:issuer:vault.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Secret 4 | type: Opaque 5 | metadata: 6 | name: cert-manager-vault-approle 7 | namespace: kube-fraima-certmanager 8 | data: 9 | secretId: {{ .Values.vault.approle.secretId }} 10 | 11 | --- 12 | apiVersion: cert-manager.io/v1 13 | kind: ClusterIssuer 14 | metadata: 15 | name: vault-issuer 16 | spec: 17 | vault: 18 | path: {{ .Values.vault.sign.path }} 19 | server: {{ .Values.vault.server }} 20 | caBundle: 21 | auth: 22 | appRole: 23 | path: {{ .Values.vault.approle.path }} 24 | roleId: {{ .Values.vault.approle.roleId }} 25 | secretRef: 26 | name: cert-manager-vault-approle 27 | key: secretId 28 | 29 | -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "yandex-csi-controller" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://helm.fraima.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "yandex-csi-controller" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "0.0.4" 20 | } 21 | 22 | variable "yandex_cloud_controller_sa_name" { 23 | type = string 24 | default = "k8s-csi-controller" 25 | } 26 | 27 | variable "namespace" { 28 | type = string 29 | default = "kube-fraima-csi" 30 | } 31 | 32 | variable "global_vars" { 33 | type = any 34 | } 35 | 36 | variable "extra_values" { 37 | type = any 38 | default = {} 39 | } 40 | -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/etcd/variables.tf: -------------------------------------------------------------------------------- 1 | variable "k8s_global_vars" { 2 | description = "K8S: ?" 3 | type = any 4 | default = null 5 | } 6 | 7 | variable "data_dir" { 8 | description = "K8S: ?" 9 | type = string 10 | default = "/var/lib/etcd" 11 | } 12 | 13 | variable "etcd_version" { 14 | description = "K8S: ?" 15 | type = string 16 | default = null 17 | } 18 | 19 | variable "etcd_image" { 20 | description = "K8S: ?" 21 | type = string 22 | default = null 23 | } 24 | 25 | variable "instance_list_map" { 26 | description = "K8S: node type" 27 | type = any 28 | default = {} 29 | } 30 | 31 | 32 | variable "etcd_initial_cluster" { 33 | description = "K8S: ?" 34 | type = string 35 | default = null 36 | } 37 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | k8s_api_server = var.global_vars.k8s-addresses.kube_apiserver_lb_fqdn 3 | k8s_api_server_port = var.global_vars.kubernetes-ports.kube-apiserver-port-lb 4 | pod_cidr = var.global_vars.k8s_network.pod_cidr 5 | cluster_name = var.global_vars.cluster_metadata.cluster_name 6 | 7 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 8 | yandex_cloud_controller_sa = local.yandex_k8s_controllers_sa_payload 9 | cluster_name = local.cluster_name 10 | pod_cidr = local.pod_cidr 11 | k8s_api_server = local.k8s_api_server 12 | k8s_api_server_port = local.k8s_api_server_port 13 | })) 14 | 15 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 16 | } -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "yandex-compute-instance" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://helm.fraima.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "machine-group" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "0.1.13" 20 | } 21 | 22 | variable "namespace" { 23 | type = string 24 | default = "kube-fraima-machine-controller-manager" 25 | } 26 | 27 | variable "global_vars" { 28 | type = any 29 | } 30 | 31 | variable "extra_values" { 32 | type = any 33 | default = {} 34 | } 35 | 36 | variable "custom_values" { 37 | type = any 38 | default = {} 39 | } 40 | 41 | variable "yandex_cloud_controller_sa_name" { 42 | type = string 43 | default = "k8s-cloud-controller" 44 | } -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_values = yamldecode(templatefile("${path.module}/helm/values.yaml.tftpl", { 3 | k8s_api_server_fqdn = var.global_vars.k8s-addresses.kube_apiserver_lb_fqdn 4 | k8s_api_server_port = var.global_vars.kubernetes-ports.kube-apiserver-port-lb 5 | resolved = true 6 | subnet_id = try(var.custom_values.subnet_id, "") 7 | zone = try(var.custom_values.zone, "") 8 | image_id = try(var.custom_values.image_id, "") 9 | replicas = try(var.custom_values.replicas, 0) 10 | provider_secret_name = var.release_name 11 | ssh_username = var.global_vars.base.ssh_username 12 | ssh_key = file(var.global_vars.base.ssh_rsa_path) 13 | })) 14 | 15 | merge_values = merge(local.base_values, try(var.extra_values.extra_values, {})) 16 | } 17 | -------------------------------------------------------------------------------- /modules/k8s-templates/kubeconfig/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "component-name" { 3 | description = "K8S: component name" 4 | type = string 5 | default = "default" 6 | } 7 | 8 | variable "certificate-authority" { 9 | description = "K8S: cluster ca bundle path" 10 | type = string 11 | default = null 12 | } 13 | 14 | variable "kube-apiserver" { 15 | description = "K8S: cluster api" 16 | type = string 17 | default = "127.0.0.1" 18 | } 19 | 20 | variable "kube-apiserver-port" { 21 | description = "K8S: cluster api port" 22 | type = string 23 | default = "443" 24 | } 25 | 26 | variable "client-certificate" { 27 | description = "K8S: cluster client cert path" 28 | type = string 29 | default = null 30 | } 31 | 32 | variable "client-key" { 33 | description = "K8S: cluster client cert key path " 34 | type = string 35 | default = null 36 | } 37 | -------------------------------------------------------------------------------- /modules/yandex-network-base/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cloud" { 2 | type = map(object({ 3 | name = optional(string, "") 4 | folder_name = optional(string, "") 5 | })) 6 | default = { 7 | "extra-args" = { 8 | folder_name = "example" 9 | name = "example" 10 | } 11 | } 12 | } 13 | 14 | variable "vpc" { 15 | type = map(object({ 16 | name = optional(string, "") 17 | })) 18 | default = { 19 | "extra-args" = { 20 | name = "vpc-example" 21 | } 22 | } 23 | } 24 | 25 | variable "gateway" { 26 | type = map(object({ 27 | name = optional(string, "") 28 | })) 29 | default = { 30 | "extra-args" = { 31 | name = "gw-example" 32 | } 33 | } 34 | } 35 | 36 | variable "route-table" { 37 | type = map(object({ 38 | name = optional(string, "") 39 | })) 40 | default = { 41 | "extra-args" = { 42 | name = "route-table-example" 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | variable "release_name" { 3 | type = string 4 | default = "yandex-cloud-controller" 5 | } 6 | 7 | variable "chart_repo" { 8 | type = string 9 | default = "https://helm.fraima.io" 10 | } 11 | 12 | variable "chart_name" { 13 | type = string 14 | default = "yandex-cloud-controller" 15 | } 16 | 17 | variable "chart_version" { 18 | type = string 19 | default = "0.0.3" 20 | } 21 | 22 | variable "yandex_cloud_controller_sa_name" { 23 | type = string 24 | default = "k8s-cloud-controller" 25 | } 26 | 27 | variable "yandex_default_vpc_name" { 28 | type = string 29 | } 30 | 31 | variable "yandex_default_route_table_name" { 32 | type = string 33 | } 34 | 35 | variable "namespace" { 36 | type = string 37 | default = "kube-fraima-ccm" 38 | } 39 | 40 | variable "global_vars" { 41 | type = any 42 | } 43 | 44 | variable "extra_values" { 45 | type = any 46 | default = {} 47 | } 48 | -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/locals.etcd.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | 4 | etcd_list_servers = formatlist("https://%s.${var.k8s_global_vars.cluster_metadata.cluster_name}.${var.k8s_global_vars.cluster_metadata.base_domain}:${var.k8s_global_vars.kubernetes-ports.etcd-server-port}", 5 | var.k8s_global_vars.master_vars.master_instance_extra_list) 6 | 7 | 8 | etcd_initial_cluster_list = flatten([ 9 | for instance_name in var.k8s_global_vars.master_vars.master_instance_extra_list : [ 10 | "${instance_name}.${var.k8s_global_vars.cluster_metadata.cluster_name}.${var.k8s_global_vars.cluster_metadata.base_domain}=https://${instance_name}.${var.k8s_global_vars.cluster_metadata.cluster_name}.${var.k8s_global_vars.cluster_metadata.base_domain}:${var.k8s_global_vars.kubernetes-ports.etcd-peer-port}" 11 | ] 12 | ]) 13 | 14 | etcd_initial_cluster = join(",", local.etcd_initial_cluster_list) 15 | 16 | } 17 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/default/kube-controller-manager.yaml: -------------------------------------------------------------------------------- 1 | v: "2" 2 | allocate-node-cidrs: "true" 3 | concurrent-deployment-syncs: "5" 4 | concurrent-endpoint-syncs: "5" 5 | concurrent-namespace-syncs: "10" 6 | concurrent-replicaset-syncs: "20" 7 | concurrent-resource-quota-syncs: "5" 8 | horizontal-pod-autoscaler-sync-period: "30s" 9 | kube-api-burst: "120" 10 | kube-api-qps: "100" 11 | leader-elect: "true" 12 | leader-elect-lease-duration: "15s" 13 | leader-elect-renew-deadline: "10s" 14 | leader-elect-retry-period: "2s" 15 | namespace-sync-period: "2m0s" 16 | node-monitor-grace-period: "40s" 17 | node-monitor-period: "5s" 18 | node-startup-grace-period: "10s" 19 | # 1.27.1 DEPRECATED 20 | # TODO 21 | pod-eviction-timeout: "30s" 22 | profiling: "false" 23 | resource-quota-sync-period: "5m0s" 24 | terminated-pod-gc-threshold: "0" 25 | cluster-signing-duration: "1440m" 26 | use-service-account-credentials: "true" 27 | authorization-always-allow-paths: "/healthz,/metrics" 28 | feature-gates: "RotateKubeletServerCertificate=true" 29 | cloud-provider: "external" 30 | controllers: "*,bootstrapsigner,tokencleaner" 31 | -------------------------------------------------------------------------------- /modules/vault-root-pki/vault.core.pki.tf: -------------------------------------------------------------------------------- 1 | resource "vault_mount" "core_root_ca" { 2 | path = var.root-pki.extra-args.path 3 | type = "pki" 4 | description = var.root-pki.extra-args.description 5 | default_lease_ttl_seconds = var.root-pki.extra-args.default_lease_ttl_seconds 6 | max_lease_ttl_seconds = var.root-pki.extra-args.max_lease_ttl_seconds 7 | } 8 | 9 | resource "vault_pki_secret_backend_root_cert" "core_root" { 10 | backend = vault_mount.core_root_ca.path 11 | type = var.root-pki.extra-args.type 12 | common_name = var.root-pki.extra-args.common_name 13 | ttl = var.root-pki.extra-args.ttl 14 | format = var.root-pki.extra-args.format 15 | private_key_format = var.root-pki.extra-args.private_key_format 16 | key_type = var.root-pki.extra-args.key_type 17 | key_bits = var.root-pki.extra-args.key_bits 18 | exclude_cn_from_sans = var.root-pki.extra-args.exclude_cn_from_sans 19 | province = var.root-pki.extra-args.province 20 | } 21 | 22 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.kubelet-config-flags.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | default_kubelet_config_flags = { 3 | staticPodPath = "${local.global_path.base_static_pod_path}" 4 | tlsCertFile = "${local.ssl.intermediate["kubernetes-ca"].issuers["kubelet-server"].certificates["kubelet-server"].key-keeper-args.host_path}/kubelet-server.pem" 5 | tlsPrivateKeyFile = "${local.ssl.intermediate["kubernetes-ca"].issuers["kubelet-server"].certificates["kubelet-server"].key-keeper-args.host_path}/kubelet-server-key.pem" 6 | healthzPort = local.kubernetes-ports.kubelet-healthz-port 7 | authentication = { 8 | x509 = { 9 | clientCAFile = "${local.ssl.intermediate["kubernetes-ca"].default.host_path}/kubernetes-ca.pem" 10 | } 11 | } 12 | 13 | clusterDNS = [ 14 | "${local.k8s-addresses.dns_address}" 15 | ] 16 | } 17 | } 18 | 19 | data "utils_deep_merge_yaml" "kubelet_config_flags" { 20 | input = [ 21 | yamlencode(local.default_kubelet_config_flags), 22 | file("${path.module}/default/kubelet.yaml"), 23 | yamlencode(try(var.extra_args.kubelet_config_flags, {})) 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | 2 | machineContoller: 3 | namespace: kube-fraima-machine-controller-manager 4 | 5 | clusterUrl: ${k8s_api_server_fqdn} 6 | clusterPort: ${k8s_api_server_port} 7 | 8 | replicas: ${replicas} 9 | minReadySeconds: 300 10 | 11 | strategy: 12 | rollingUpdate: 13 | maxSurge: 1 14 | maxUnavailable: 1 15 | type: RollingUpdate 16 | 17 | selector: 18 | matchLabels: 19 | instance-group: example 20 | 21 | metadata: 22 | labels: 23 | instance-group: example 24 | nodeLabels: 25 | node-role.kubernetes.io/example: "" 26 | provider: yandex 27 | cloudLabels: 28 | tair: critical 29 | 30 | bootDisk: 31 | imageID: ${image_id} 32 | typeID: network-hdd 33 | autoDelete: true 34 | size: 20 35 | 36 | resources: 37 | cpu: 8 38 | coreFraction: 20 39 | memory: 8 40 | 41 | networkInterfaces: 42 | nat: true 43 | regionID: ru-central1 44 | subnetID: "${subnet_id}" 45 | zoneID: "${zone}" 46 | networkType: STANDARD 47 | 48 | provider: 49 | type: YandexCloud 50 | name: ${provider_secret_name} 51 | 52 | ssh: 53 | username: ${ssh_username} 54 | key: ${ssh_key} 55 | -------------------------------------------------------------------------------- /modules/helm-certmanager/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | tolerations: 2 | - key: node-role.kubernetes.io/master 3 | effect: NoSchedule 4 | - key: "CriticalAddonsOnly" 5 | operator: "Exists" 6 | - key: node.cloudprovider.kubernetes.io/uninitialized 7 | value: "true" 8 | effect: NoSchedule 9 | 10 | webhook: 11 | tolerations: 12 | - key: node-role.kubernetes.io/master 13 | effect: NoSchedule 14 | - key: "CriticalAddonsOnly" 15 | operator: "Exists" 16 | - key: node.cloudprovider.kubernetes.io/uninitialized 17 | value: "true" 18 | effect: NoSchedule 19 | 20 | cainjector: 21 | tolerations: 22 | - key: node-role.kubernetes.io/master 23 | effect: NoSchedule 24 | - key: "CriticalAddonsOnly" 25 | operator: "Exists" 26 | - key: node.cloudprovider.kubernetes.io/uninitialized 27 | value: "true" 28 | effect: NoSchedule 29 | 30 | 31 | startupapicheck: 32 | tolerations: 33 | - key: node-role.kubernetes.io/master 34 | effect: NoSchedule 35 | - key: "CriticalAddonsOnly" 36 | operator: "Exists" 37 | - key: node.cloudprovider.kubernetes.io/uninitialized 38 | value: "true" 39 | effect: NoSchedule 40 | 41 | featureGates: "ExperimentalCertificateSigningRequestControllers=true" 42 | installCRDs: true 43 | -------------------------------------------------------------------------------- /modules/helm-base-roles/helm/base-roles/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: base-roles 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | -------------------------------------------------------------------------------- /modules/helm-vault-issuer/helm/base-vault-node-csr/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: base 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | # It is recommended to use it with quotes. 24 | appVersion: "1.16.0" 25 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/outputs.tf: -------------------------------------------------------------------------------- 1 | output "ssl" { 2 | value = local.ssl 3 | } 4 | 5 | output "secrets" { 6 | value = local.secrets 7 | } 8 | 9 | output "global_path" { 10 | value = local.global_path 11 | } 12 | 13 | output "main_path" { 14 | value = local.main_path 15 | } 16 | 17 | output "ssl_for_each_map" { 18 | value = local.ssl_for_each_map 19 | } 20 | 21 | output "k8s-addresses" { 22 | value = local.k8s-addresses 23 | } 24 | 25 | output "vault-config" { 26 | value = local.vault-config 27 | } 28 | 29 | output "kubernetes-ports" { 30 | value = local.kubernetes-ports 31 | } 32 | 33 | output "base" { 34 | value = local.base 35 | } 36 | 37 | output "k8s_network" { 38 | value = local.k8s_network 39 | } 40 | 41 | output "cluster_metadata" { 42 | value = local.cluster_metadata 43 | } 44 | 45 | output "kube_flags" { 46 | value = local.kube_flags 47 | } 48 | 49 | output "k8s_provider" { 50 | value = local.k8s_provider 51 | } 52 | 53 | output "master_vars" { 54 | value = local.master_vars 55 | } 56 | 57 | output "component_versions" { 58 | value = yamldecode(data.utils_deep_merge_yaml.component_versions.output) 59 | } 60 | 61 | output "vault_instances" { 62 | value = local.vault_instances 63 | } 64 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/yandex.cluster.dns-internal.tf: -------------------------------------------------------------------------------- 1 | #### INTERNAL DNS ZONE ###### 2 | ##--> 3 | resource "yandex_dns_zone" "cluster-external" { 4 | name = local.cluster_name 5 | zone = local.base_cluster_dns_zone 6 | public = false 7 | private_networks = [data.yandex_vpc_network.cluster-vpc.id] 8 | } 9 | 10 | #### INTERNAL DNS FOR KUBE-APISERVER ###### 11 | ##--> 12 | resource "yandex_dns_recordset" "api-external" { 13 | zone_id = yandex_dns_zone.cluster-external.id 14 | name = local.kube_apiserver_lb_fqdn 15 | type = "A" 16 | ttl = 60 17 | data = local.kube_apiserver_lb_ip 18 | } 19 | 20 | #### INTERNAL DNS FRO ETCD DISCOVERY ###### 21 | ##--> 22 | # Включить, если требуется дискавери ETCD через SRV 23 | # resource "yandex_dns_recordset" "etcd-srv-server" { 24 | # zone_id = yandex_dns_zone.cluster-external.id 25 | # name = local.etcd_srv_server_record 26 | # type = "SRV" 27 | # ttl = 60 28 | # data = local.etcd_member_servers_srv 29 | # } 30 | 31 | # resource "yandex_dns_recordset" "etcd-srv-client" { 32 | # zone_id = yandex_dns_zone.cluster-external.id 33 | # name = local.etcd_srv_client_record 34 | # type = "SRV" 35 | # ttl = 60 36 | # data = local.etcd_member_clients_srv 37 | # } 38 | 39 | -------------------------------------------------------------------------------- /modules/helm-yandex-csi-controller/yandex.tf: -------------------------------------------------------------------------------- 1 | resource "yandex_iam_service_account_key" "yandex-k8s-controllers-key" { 2 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 3 | description = "key for cluster <${var.global_vars.cluster_metadata.cluster_name}>" 4 | key_algorithm = "RSA_4096" 5 | 6 | } 7 | 8 | locals { 9 | yandex_k8s_csi_controller_sa_payload = { 10 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 11 | created_at = data.yandex_iam_service_account.yandex-k8s-controllers.created_at 12 | folder_id = data.yandex_iam_service_account.yandex-k8s-controllers.folder_id 13 | service_account_json = { 14 | id = yandex_iam_service_account_key.yandex-k8s-controllers-key.id 15 | service_account_id = yandex_iam_service_account_key.yandex-k8s-controllers-key.service_account_id 16 | created_at = yandex_iam_service_account_key.yandex-k8s-controllers-key.created_at 17 | key_algorithm = yandex_iam_service_account_key.yandex-k8s-controllers-key.key_algorithm 18 | public_key = yandex_iam_service_account_key.yandex-k8s-controllers-key.public_key 19 | private_key = yandex_iam_service_account_key.yandex-k8s-controllers-key.private_key 20 | } 21 | 22 | } 23 | } -------------------------------------------------------------------------------- /modules/k8s-config-vars/default/kube-apiserver.yaml: -------------------------------------------------------------------------------- 1 | v: "4" 2 | event-ttl: "1h0m0s" 3 | kubernetes-service-node-port: "0" 4 | # 1.27.1 DEPRECATED 5 | # TODO 6 | master-service-namespace: "default" 7 | max-connection-bytes-per-sec: "0" 8 | max-requests-inflight: "400" 9 | min-request-timeout: "1800" 10 | profiling: "false" 11 | feature-gates: "RotateKubeletServerCertificate=true" 12 | anonymous-auth: "true" 13 | audit-log-maxage: "30" 14 | audit-log-maxbackup: "10" 15 | audit-log-maxsize: "1000" 16 | audit-log-mode: "batch" 17 | enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,AlwaysPullImages,NodeRestriction,PodSecurity" 18 | enable-bootstrap-token-auth: "true" 19 | runtime-config: "api/all=true" 20 | enable-aggregator-routing: "true" 21 | api-audiences: "konnectivity-server" 22 | requestheader-allowed-names: "front-proxy-client" 23 | requestheader-extra-headers-prefix: "X-Remote-Extra-" 24 | requestheader-group-headers: "X-Remote-Group" 25 | requestheader-username-headers: "X-Remote-User" 26 | allow-privileged: "true" 27 | authorization-mode: "Node,RBAC" 28 | service-account-issuer: "https://kubernetes.default.svc.cluster.local" 29 | kubelet-preferred-address-types: "InternalIP,ExternalIP,Hostname" 30 | kubelet-timeout: "5s" 31 | cloud-provider: "external" 32 | -------------------------------------------------------------------------------- /modules/keycloak-base/keycloak.tf: -------------------------------------------------------------------------------- 1 | 2 | # create groups openid client scope 3 | resource "keycloak_openid_client_scope" "groups" { 4 | realm_id = local.idp_provider_realm 5 | name = "groups" 6 | include_in_token_scope = true 7 | gui_order = 1 8 | } 9 | 10 | resource "keycloak_openid_group_membership_protocol_mapper" "groups" { 11 | realm_id = local.idp_provider_realm 12 | client_scope_id = keycloak_openid_client_scope.groups.id 13 | name = "groups" 14 | claim_name = "groups" 15 | full_path = false 16 | } 17 | 18 | resource "keycloak_openid_client" "kube" { 19 | realm_id = local.idp_provider_realm 20 | client_id = "kubernetes-clusters" 21 | name = "kubernetes-clusters" 22 | enabled = true 23 | access_type = "CONFIDENTIAL" 24 | client_secret = "kube-client-secret" 25 | standard_flow_enabled = false 26 | implicit_flow_enabled = false 27 | direct_access_grants_enabled = true 28 | } 29 | 30 | # configure kube openid client default scopes 31 | resource "keycloak_openid_client_default_scopes" "kube" { 32 | realm_id = local.idp_provider_realm 33 | client_id = keycloak_openid_client.kube.id 34 | default_scopes = [ 35 | "email", 36 | "groups", 37 | ] 38 | } -------------------------------------------------------------------------------- /modules/k8s-ready-status/main.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "null_resource" "cluster" { 3 | for_each = var.cluster_instances 4 | 5 | triggers = { 6 | cluster_instance_ids = join(",", keys(var.cluster_instances)) 7 | } 8 | 9 | connection { 10 | host = each.value 11 | user = var.k8s_global_vars.base.ssh_username 12 | type = "ssh" 13 | private_key = file(split(".pub", var.k8s_global_vars.base.ssh_rsa_path)[0]) 14 | agent = "false" 15 | } 16 | # TODO поправить команду так, что бы не падала сборка 17 | provisioner "remote-exec" { 18 | inline = [ 19 | "until cloud-init status | grep -i done; do sleep 1s; done", 20 | "sudo kubectl --request-timeout=5m cluster-info --kubeconfig=/etc/kubernetes/admin.conf" 21 | ] 22 | } 23 | } 24 | 25 | # data "http" "kubernetes-livez" { 26 | # insecure =true 27 | # request_timeout_ms = 60000 28 | 29 | # retry { 30 | # attempts =1000 31 | # min_delay_ms = 1000 32 | # } 33 | 34 | # url = "https://${try(var.kube_apiserver_lb, "")}:${var.k8s_global_vars.kubernetes-ports.kube-apiserver-port-lb}/livez?verbose" 35 | 36 | # request_headers = { 37 | # Accept = "application/json" 38 | # } 39 | # lifecycle { 40 | # postcondition { 41 | # condition = contains([200], self.status_code) 42 | # error_message = "Cluster is not ready" 43 | # } 44 | # } 45 | # } 46 | -------------------------------------------------------------------------------- /modules/k8s-master-infra-firewall/charlotte.firewall.tf: -------------------------------------------------------------------------------- 1 | module "firewall" { 2 | depends_on = [ 3 | sgroups_group.masters, 4 | sgroups_group.api, 5 | ] 6 | source = "../charlotte" 7 | security_groups = local.charlotte_payload 8 | } 9 | 10 | resource "sgroups_network" "masters_internal" { 11 | 12 | for_each = var.cluster_instances_internal 13 | 14 | name = substr(sha256("${each.value}/32"), 0, 10) 15 | cidr = "${each.value}/32" 16 | } 17 | 18 | resource "sgroups_network" "masters_external" { 19 | 20 | for_each = var.cluster_instances_external 21 | 22 | name = substr(sha256("${each.value}/32"), 0, 10) 23 | cidr = "${each.value}/32" 24 | } 25 | 26 | resource "sgroups_group" "masters" { 27 | depends_on = [ 28 | sgroups_network.masters_external, 29 | sgroups_network.masters_internal, 30 | ] 31 | name = "kubernetes/${var.k8s_global_vars.cluster_metadata.cluster_name}/masters" 32 | networks = join(",", local.networks_master) 33 | } 34 | 35 | resource "sgroups_network" "api" { 36 | 37 | name = substr(sha256("${var.cluster_api_ip}/32"), 0, 10) 38 | cidr = "${var.cluster_api_ip}/32" 39 | } 40 | 41 | resource "sgroups_group" "api" { 42 | depends_on = [ 43 | sgroups_network.api 44 | ] 45 | name = "kubernetes/${var.k8s_global_vars.cluster_metadata.cluster_name}/api" 46 | networks = substr(sha256("${var.cluster_api_ip}/32"), 0, 10) 47 | } 48 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/variables.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | variable "k8s_global_vars" { 4 | description = "module:K8S-CERTIFICATE-VARS: base certificate vars" 5 | type = any 6 | default = {} 7 | } 8 | 9 | variable "k8s_vault_master" { 10 | type = object({ 11 | secret_id_all = any 12 | role_id_all = any 13 | }) 14 | default = { 15 | secret_id_all = null 16 | role_id_all = null 17 | } 18 | } 19 | 20 | variable "cloud_metadata" { 21 | type = object({ 22 | cloud_name = string 23 | folder_name = string 24 | }) 25 | default = { 26 | cloud_name = null 27 | folder_name = null 28 | } 29 | } 30 | 31 | # variable "master_group"{ 32 | # type = object({ 33 | # name = string 34 | # count = number 35 | # vpc_name = string 36 | # route_table_name = string 37 | # # subnets = any 38 | # default_subnet = optional(string, "") 39 | # default_zone = optional(string, "") 40 | # resources_override = any 41 | # resources = any 42 | # metadata = any 43 | # }) 44 | # default = { 45 | # name = "master" 46 | # count = 0 47 | # vpc_name = null 48 | # # subnets = null 49 | # default_subnet = null 50 | # route_table_name = "" 51 | # default_zone = "ru-central1-a" 52 | # resources_override = {} 53 | # metadata = {} 54 | # resources = { 55 | # core = 4 56 | # memory = 8 57 | # core_fraction = 100 58 | # disk = {} 59 | # } 60 | # } 61 | # } 62 | -------------------------------------------------------------------------------- /modules/helm-yandex-cloud-controller/yandex.tf: -------------------------------------------------------------------------------- 1 | resource "yandex_iam_service_account_key" "yandex-k8s-controllers-key" { 2 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 3 | description = "key for cluster <${var.global_vars.cluster_metadata.cluster_name}>" 4 | key_algorithm = "RSA_4096" 5 | 6 | } 7 | 8 | locals { 9 | yandex_k8s_controllers_sa_payload = { 10 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 11 | created_at = data.yandex_iam_service_account.yandex-k8s-controllers.created_at 12 | folder_id = data.yandex_iam_service_account.yandex-k8s-controllers.folder_id 13 | vpc_id = data.yandex_vpc_network.cluster-vpc.id 14 | route_table_id = data.yandex_vpc_route_table.cluster-vpc-route-table.id 15 | service_account_json = { 16 | id = yandex_iam_service_account_key.yandex-k8s-controllers-key.id 17 | service_account_id = yandex_iam_service_account_key.yandex-k8s-controllers-key.service_account_id 18 | created_at = yandex_iam_service_account_key.yandex-k8s-controllers-key.created_at 19 | key_algorithm = yandex_iam_service_account_key.yandex-k8s-controllers-key.key_algorithm 20 | public_key = yandex_iam_service_account_key.yandex-k8s-controllers-key.public_key 21 | private_key = yandex_iam_service_account_key.yandex-k8s-controllers-key.private_key 22 | } 23 | 24 | } 25 | } -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/kubeadm-config/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | manifest = flatten([ 3 | for node_name, node_content in var.instance_list_map: 4 | {"${node_name}" = templatefile("${path.module}/templates/kubeadm-config.yaml.tftpl", { 5 | 6 | etcd_list_servers = var.etcd_list_servers 7 | secrets = var.k8s_global_vars.secrets 8 | service_cidr = var.k8s_global_vars.k8s_network.service_cidr 9 | ssl = var.k8s_global_vars.ssl 10 | component_versions = var.k8s_global_vars.component_versions 11 | base_path = var.k8s_global_vars.global_path 12 | main_path = var.k8s_global_vars.main_path 13 | kube_apiserver_port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port 14 | kube_api_fqdn = var.k8s_global_vars.k8s-addresses.kube_apiserver_lb_fqdn 15 | cluster_name = var.k8s_global_vars.cluster_metadata.cluster_name 16 | pod_cidr = var.k8s_global_vars.k8s_network.pod_cidr 17 | node_cidr_mask = var.k8s_global_vars.k8s_network.node_cidr_mask 18 | kube_flags = var.k8s_global_vars.kube_flags 19 | # kubelet_config = module.kubelet.kubelet-config 20 | })} 21 | ]) 22 | 23 | manifest-map = { for item in local.manifest : 24 | keys(item)[0] => values(item)[0] 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /modules/helm-vault-issuer/main.tf: -------------------------------------------------------------------------------- 1 | resource "vault_approle_auth_backend_role" "k8s-vault-role" { 2 | 3 | backend = var.global_vars.global_path.base_vault_path_approle 4 | role_name = "kubelet-peer-k8s-certmanager" 5 | token_policies = ["${var.global_vars.main_path.base_vault_path}/certificates/vault-cluster-policy"] 6 | token_ttl = 60 7 | secret_id_bound_cidrs = [] 8 | token_bound_cidrs = [] 9 | 10 | } 11 | 12 | resource "vault_approle_auth_backend_role_secret_id" "k8s-vault-secret" { 13 | backend = var.global_vars.global_path.base_vault_path_approle 14 | role_name = vault_approle_auth_backend_role.k8s-vault-role.role_name 15 | } 16 | 17 | resource "helm_release" "base-vault-node-csr" { 18 | 19 | name = "base-vault-node-csr" 20 | chart = "${path.module}/helm/base-vault-node-csr" 21 | namespace = "kube-system" 22 | atomic = true 23 | values = [ 24 | templatefile("${path.module}/helm/base-vault-node-csr/values.yaml", { 25 | vaut_approle_secretid : base64encode(vault_approle_auth_backend_role_secret_id.k8s-vault-secret.secret_id) 26 | vaut_approle_roleid : vault_approle_auth_backend_role.k8s-vault-role.role_id 27 | vaut_sign_path : "${var.global_vars.ssl.intermediate.kubernetes-ca.default.path}/sign/kubelet-peer-k8s-certmanager" 28 | vaut_server : var.global_vars.vault-config.vault_server 29 | vaut_approle_path : var.global_vars.global_path.base_vault_path_approle 30 | }) 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/etcd/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | manifest = flatten([ 3 | for node_name, node_content in var.instance_list_map: 4 | {"${node_name}" = templatefile("${path.module}/templates/etcd.yaml.tftpl", { 5 | base_local_path_certs = var.k8s_global_vars.global_path.base_local_path_certs 6 | ssl = var.k8s_global_vars.ssl 7 | cluster_name = var.k8s_global_vars.cluster_metadata.cluster_name 8 | base_domain = var.k8s_global_vars.cluster_metadata.base_domain 9 | component_versions = var.k8s_global_vars.component_versions 10 | discovery_srv = var.k8s_global_vars.k8s-addresses.base_cluster_fqdn 11 | full_instance_name = format("${node_name}.${var.k8s_global_vars.k8s-addresses.base_cluster_fqdn}") 12 | etcd_peer_port = var.k8s_global_vars.kubernetes-ports.etcd-peer-port 13 | etcd_server_port = var.k8s_global_vars.kubernetes-ports.etcd-server-port 14 | etcd_metrics_port = var.k8s_global_vars.kubernetes-ports.etcd-metrics-port 15 | etcd_server_port_target_lb = var.k8s_global_vars.kubernetes-ports.etcd-server-port-target-lb 16 | data_dir = var.data_dir 17 | etcd_initial_cluster = var.etcd_initial_cluster 18 | })} 19 | ]) 20 | 21 | manifest-map = { for item in local.manifest : 22 | keys(item)[0] => values(item)[0] 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/yandex.lock.box.all.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "yandex_lockbox_secret" "master_key_keeper_approles_role_id_all" { 3 | for_each = var.k8s_global_vars.master_vars.master_instance_extra_list_map 4 | name = "${each.key}-all-role-id" 5 | } 6 | 7 | resource "yandex_lockbox_secret_version" "master_key_keeper_approles_role_id_all" { 8 | for_each = var.k8s_global_vars.master_vars.master_instance_extra_list_map 9 | 10 | secret_id = yandex_lockbox_secret.master_key_keeper_approles_role_id_all[each.key].id 11 | 12 | dynamic "entries" { 13 | for_each = local.map_role_id_all 14 | content { 15 | key = replace(entries.key, "-", "-${local.extra_cluster_name}-") 16 | text_value = local.role_id_all[entries.key].role_id 17 | } 18 | } 19 | } 20 | 21 | resource "yandex_lockbox_secret" "master_key_keeper_approles_secret_id_all" { 22 | for_each = var.k8s_global_vars.master_vars.master_instance_extra_list_map 23 | name = "${each.key}-all-secret-id" 24 | } 25 | 26 | resource "yandex_lockbox_secret_version" "master_key_keeper_approles_secret_id_all" { 27 | for_each = var.k8s_global_vars.master_vars.master_instance_extra_list_map 28 | 29 | secret_id = yandex_lockbox_secret.master_key_keeper_approles_secret_id_all[each.key].id 30 | 31 | dynamic "entries" { 32 | for_each = local.map_secret_id_alls 33 | content { 34 | key = replace(entries.key, "-", "-${local.extra_cluster_name}-") 35 | text_value = local.secret_id_all[entries.key].secret_id 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /modules/k8s-templates/services/key-keeper/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | key-keeper-config = flatten([ 3 | for node_name, node_content in var.instance_list_map: 4 | {"${node_name}" = templatefile("${path.module}/templates/config-all-approle-all.tftpl", { 5 | intermediates = var.k8s_global_vars.ssl.intermediate 6 | external_intermediates = var.k8s_global_vars.ssl.external_intermediate 7 | secrets = var.k8s_global_vars.secrets 8 | base_path = var.k8s_global_vars.global_path 9 | base_certificate_atrs = var.k8s_global_vars.ssl.global-args.key-keeper-args 10 | vault_config = var.k8s_global_vars.vault-config 11 | availability_zone = node_name 12 | full_instance_name = "${node_name}.${var.k8s_global_vars.k8s-addresses.base_cluster_fqdn}" 13 | base_cluster_fqdn = "${var.k8s_global_vars.k8s-addresses.base_cluster_fqdn}" 14 | external_instance_name = "${node_name}-${var.k8s_global_vars.cluster_metadata.cluster_name}" 15 | instance_type = var.instance_type 16 | })} 17 | ]) 18 | key-keeper-config-map = { for item in local.key-keeper-config : 19 | keys(item)[0] => values(item)[0] 20 | } 21 | 22 | key-keeper-service = templatefile("${path.module}/templates/service.yaml.tftpl", { 23 | base_local_path_certs = var.k8s_global_vars.global_path.base_local_path_certs 24 | }) 25 | 26 | } 27 | -------------------------------------------------------------------------------- /modules/vault-root-pki/variables.tf: -------------------------------------------------------------------------------- 1 | variable "root-pki" { 2 | description = "Map of vault with their parameters" 3 | type = map(object({ 4 | path = optional(string, "pki-root") 5 | description = optional(string, "Root CA") 6 | default_lease_ttl_seconds = optional(number, 321408000) 7 | max_lease_ttl_seconds = optional(number, 321408000) 8 | type = optional(string, "internal") 9 | common_name = optional(string, "Root CA") 10 | ttl = optional(number, 321408000) 11 | format = optional(string, "pem") 12 | private_key_format = optional(string, "der") 13 | key_type = optional(string, "rsa") 14 | key_bits = optional(number, 4096) 15 | exclude_cn_from_sans = optional(bool, true) 16 | province = optional(string, "CA") 17 | 18 | 19 | })) 20 | default = { 21 | "extra-args" = { 22 | description = "Root CA" 23 | path = "pki-root" 24 | type = "internal" 25 | common_name = "Root CA" 26 | format = "pem" 27 | key_type = "rsa" 28 | private_key_format = "der" 29 | province = "CA" 30 | exclude_cn_from_sans = true 31 | default_lease_ttl_seconds = 321408000 32 | max_lease_ttl_seconds = 321408000 33 | ttl = 321408000 34 | key_bits = 4096 35 | 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/yandex.cluster.lb.tf: -------------------------------------------------------------------------------- 1 | #### LB ###### 2 | ##--> 3 | resource "yandex_lb_target_group" "master-tg" { 4 | name = local.yandex_lb_target_group_master 5 | region_id = "ru-central1" 6 | 7 | dynamic "target" { 8 | for_each = var.k8s_global_vars.master_vars.master_instance_list_map 9 | 10 | content { 11 | subnet_id = yandex_vpc_subnet.master-subnets[ 12 | "${try( 13 | var.k8s_global_vars.master_vars.master_group.resources_override[target.key].network_interface.subnet, 14 | var.k8s_global_vars.master_vars.master_group.default_subnet 15 | )}:${try( 16 | var.k8s_global_vars.master_vars.master_group.resources_override[target.key].network_interface.zone, 17 | var.k8s_global_vars.master_vars.master_group.default_zone 18 | )}" 19 | ].id 20 | address = yandex_compute_instance.master[target.key].network_interface.0.ip_address 21 | } 22 | } 23 | } 24 | 25 | resource "yandex_lb_network_load_balancer" "api-external" { 26 | name = local.kube_apiserver_lb_name 27 | type = "external" 28 | region_id = "ru-central1" 29 | 30 | listener { 31 | name = local.kube_apiserver_listener_name 32 | port = local.kube_apiserver_port_lb 33 | target_port = local.kube_apiserver_port 34 | 35 | external_address_spec { 36 | ip_version = "ipv4" 37 | } 38 | } 39 | attached_target_group { 40 | target_group_id = yandex_lb_target_group.master-tg.id 41 | 42 | healthcheck { 43 | name = "tcp" 44 | tcp_options { 45 | port = local.kube_apiserver_port 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /modules/helm-yandex-machine-instance/yandex.tf: -------------------------------------------------------------------------------- 1 | resource "yandex_iam_service_account_key" "yandex-k8s-controllers-key" { 2 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 3 | description = "key for cluster <${var.global_vars.cluster_metadata.cluster_name}>" 4 | key_algorithm = "RSA_4096" 5 | 6 | } 7 | 8 | locals { 9 | yandex_k8s_mi_controller_sa_payload = { 10 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 11 | created_at = data.yandex_iam_service_account.yandex-k8s-controllers.created_at 12 | folder_id = data.yandex_iam_service_account.yandex-k8s-controllers.folder_id 13 | service_account_json = { 14 | id = yandex_iam_service_account_key.yandex-k8s-controllers-key.id 15 | service_account_id = yandex_iam_service_account_key.yandex-k8s-controllers-key.service_account_id 16 | created_at = yandex_iam_service_account_key.yandex-k8s-controllers-key.created_at 17 | key_algorithm = yandex_iam_service_account_key.yandex-k8s-controllers-key.key_algorithm 18 | public_key = yandex_iam_service_account_key.yandex-k8s-controllers-key.public_key 19 | private_key = yandex_iam_service_account_key.yandex-k8s-controllers-key.private_key 20 | } 21 | } 22 | } 23 | 24 | resource "kubernetes_secret" "cloud-secret" { 25 | metadata { 26 | name = var.release_name 27 | namespace = var.namespace 28 | } 29 | 30 | data = { 31 | folderID = local.yandex_k8s_mi_controller_sa_payload.folder_id 32 | serviceAccountJSON = jsonencode(local.yandex_k8s_mi_controller_sa_payload.service_account_json) 33 | } 34 | 35 | type = "Opaque" 36 | } -------------------------------------------------------------------------------- /modules/k8s-templates/bashrc/templates/bashrc.tftpl: -------------------------------------------------------------------------------- 1 | # .bashrc 2 | 3 | # User specific aliases and functions 4 | 5 | PROMPT_COMMAND='history -a' 6 | export HISTCONTROL="ignoreboth" 7 | export HISTIGNORE="" 8 | shopt -s cmdhist 9 | 10 | alias rm='rm -i' 11 | alias cp='cp -i' 12 | alias mv='mv -i' 13 | 14 | alias kg='kubectl get' 15 | alias ka='kubectl apply' 16 | alias kd='kubectl delete' 17 | alias ki='kubectl describe' 18 | alias ke='kubectl edit' 19 | alias k='kubectl ' 20 | alias kl='kubectl logs ' 21 | 22 | new_kubeconfig () { 23 | SRC_SSH_PORT=$(env | grep SSH_CONNECTION | awk '{print $2}') 24 | cp -uf ${main_path.base_kubernetes_path}/admin.conf /tmp/kubeconfig 25 | export KUBECONFIG=/tmp/kubeconfig 26 | } 27 | new_kubeconfig 28 | 29 | kns () { 30 | kubectl config set-context --current --namespace=$1 31 | } 32 | 33 | 34 | export ETCDCTL_CERT=${ssl.intermediate["etcd-ca"].issuers["etcd-client"].certificates["kube-apiserver-etcd-client"].key-keeper-args.host_path}/kube-apiserver-etcd-client.pem \ 35 | export ETCDCTL_KEY=${ssl.intermediate["etcd-ca"].issuers["etcd-client"].certificates["kube-apiserver-etcd-client"].key-keeper-args.host_path}/kube-apiserver-etcd-client-key.pem \ 36 | export ETCDCTL_CACERT=${ssl.intermediate["etcd-ca"].default.host_path}/etcd-ca.pem \ 37 | 38 | etcd_endpoints () { 39 | export ENDPOINTS=$(echo $(ENDPOINTS=127.0.0.1:${etcd_server_port} 40 | etcdctl \ 41 | --endpoints=$ENDPOINTS \ 42 | member list | 43 | awk '{print $5}' | 44 | sed "s/,//") | sed "s/ /,/g") 45 | } 46 | 47 | etcd_endpoints 48 | 49 | estat () { 50 | etcdctl \ 51 | --write-out=table \ 52 | --endpoints=$ENDPOINTS \ 53 | endpoint status 54 | } 55 | 56 | # Source global definitions 57 | if [ -f /etc/bashrc ]; then 58 | . /etc/bashrc 59 | fi 60 | 61 | -------------------------------------------------------------------------------- /modules/k8s-vault-master/main.tf: -------------------------------------------------------------------------------- 1 | resource "vault_policy" "kubernetes-certmanager" { 2 | 3 | name = "${var.k8s_global_vars.main_path.base_vault_path}/certificates/vault-cluster-policy" 4 | 5 | policy = templatefile("${path.module}/templates/vault/vault-cluster-policies.tftpl", { 6 | k8s_global_vars = var.k8s_global_vars 7 | issuer_name = "kubelet-peer-k8s-certmanager" 8 | } 9 | ) 10 | } 11 | 12 | resource "vault_policy" "kubernetes-bootstrap-master" { 13 | 14 | name = "${var.k8s_global_vars.main_path.base_vault_path}/bootstrap-master" 15 | 16 | policy = templatefile("${path.module}/templates/vault/vault-bootstarp-approle-all.tftpl", { 17 | k8s_global_vars = var.k8s_global_vars 18 | instance_type = "master" 19 | } 20 | ) 21 | } 22 | 23 | resource "vault_approle_auth_backend_role" "all_masters" { 24 | for_each = var.k8s_global_vars.master_vars.master_instance_list_map 25 | 26 | backend = var.k8s_global_vars.global_path.base_vault_path_approle 27 | role_name = each.key 28 | token_ttl = 60 29 | token_policies = [vault_policy.kubernetes-bootstrap-master.name] 30 | secret_id_bound_cidrs = [] 31 | token_bound_cidrs = [] 32 | 33 | } 34 | 35 | resource "vault_approle_auth_backend_role_secret_id" "all_masters" { 36 | 37 | for_each = { 38 | for k, v in var.k8s_global_vars.master_vars.master_instance_list_map: 39 | k => v 40 | if try(var.k8s_global_vars.vault_instances.k8s_vault_master_secret_id.enabled, false) == true 41 | } 42 | 43 | backend = var.k8s_global_vars.global_path.base_vault_path_approle 44 | role_name = vault_approle_auth_backend_role.all_masters[each.key].role_name 45 | } 46 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.path.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | main_path = { 3 | base_vault_path = try(var.extra_args.base_vault_path, null) == null ? "clusters/${local.cluster_metadata.cluster_name}" : var.extra_args.base_vault_path 4 | base_kubernetes_path = try(var.extra_args.base_kubernetes_path, null) == null ? "/etc/kubernetes" : var.extra_args.base_kubernetes_path 5 | } 6 | 7 | global_path = { 8 | base_local_path_certs = try(var.extra_args.base_local_path_certs, null) == null ? "${local.main_path.base_kubernetes_path}/pki" : var.extra_args.base_local_path_certs 9 | base_static_pod_path = try(var.extra_args.base_static_pod_path, null) == null ? "${local.main_path.base_kubernetes_path}/manifests" : var.extra_args.base_static_pod_path 10 | base_local_path_vault = try(var.extra_args.base_local_path_vault, null) == null ? "/var/lib/key-keeper/vault" : var.extra_args.base_local_path_vault 11 | base_vault_path_pki = try(var.extra_args.base_vault_path, null) == null ? "${local.main_path.base_vault_path}/pki" : var.extra_args.base_vault_path 12 | base_vault_path_kv = try(var.extra_args.base_vault_path_kv, null) == null ? "${local.main_path.base_vault_path}/kv" : var.extra_args.base_vault_path_kv 13 | base_vault_path_approle = try(var.extra_args.base_vault_path_approle, null) == null ? "${local.main_path.base_vault_path}/approle" : var.extra_args.base_vault_path_approle 14 | root_vault_path_pki = try(var.extra_args.root_vault_path_pki, null) == null ? "pki-root" : var.extra_args.root_vault_path_pki 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /modules/helm-gatekeeper/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | 2 | postUpgrade: 3 | tolerations: 4 | - key: node-role.kubernetes.io/master 5 | effect: NoSchedule 6 | - key: "CriticalAddonsOnly" 7 | operator: "Exists" 8 | - key: node.cloudprovider.kubernetes.io/uninitialized 9 | value: "true" 10 | effect: NoSchedule 11 | 12 | postInstall: 13 | probeWebhook: 14 | enabled: false 15 | tolerations: 16 | - key: node-role.kubernetes.io/master 17 | effect: NoSchedule 18 | - key: "CriticalAddonsOnly" 19 | operator: "Exists" 20 | - key: node.cloudprovider.kubernetes.io/uninitialized 21 | value: "true" 22 | effect: NoSchedule 23 | 24 | preUninstall: 25 | tolerations: 26 | - key: node-role.kubernetes.io/master 27 | effect: NoSchedule 28 | - key: "CriticalAddonsOnly" 29 | operator: "Exists" 30 | - key: node.cloudprovider.kubernetes.io/uninitialized 31 | value: "true" 32 | effect: NoSchedule 33 | 34 | controllerManager: 35 | tolerations: 36 | - key: node-role.kubernetes.io/master 37 | effect: NoSchedule 38 | - key: "CriticalAddonsOnly" 39 | operator: "Exists" 40 | - key: node.cloudprovider.kubernetes.io/uninitialized 41 | value: "true" 42 | effect: NoSchedule 43 | 44 | audit: 45 | tolerations: 46 | - key: node-role.kubernetes.io/master 47 | effect: NoSchedule 48 | - key: "CriticalAddonsOnly" 49 | operator: "Exists" 50 | - key: node.cloudprovider.kubernetes.io/uninitialized 51 | value: "true" 52 | effect: NoSchedule 53 | 54 | crds: 55 | tolerations: 56 | - key: node-role.kubernetes.io/master 57 | effect: NoSchedule 58 | - key: "CriticalAddonsOnly" 59 | operator: "Exists" 60 | - key: node.cloudprovider.kubernetes.io/uninitialized 61 | value: "true" 62 | effect: NoSchedule 63 | -------------------------------------------------------------------------------- /modules/charlotte/resources.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "sgroups_networks" "networks" { 3 | 4 | dynamic "items" { 5 | for_each = local.networks_map 6 | 7 | content { 8 | name = items.key 9 | cidr = items.value 10 | } 11 | } 12 | 13 | } 14 | 15 | # resource "sgroups_groups" "groups" { 16 | # depends_on = [ 17 | # sgroups_networks.networks 18 | # ] 19 | 20 | # dynamic "items" { 21 | # for_each = local.security_groups_network__name__map 22 | 23 | # content { 24 | # name = items.key 25 | # networks = items.value 26 | # } 27 | # } 28 | # } 29 | 30 | resource "sgroups_group" "groups" { 31 | depends_on = [ 32 | sgroups_networks.networks 33 | ] 34 | 35 | for_each = local.security_groups_network__name__map 36 | 37 | lifecycle { 38 | ignore_changes = [ 39 | networks 40 | ] 41 | } 42 | 43 | name = each.key 44 | networks = each.value 45 | 46 | } 47 | 48 | 49 | resource "sgroups_rules" "rules" { 50 | depends_on = [ 51 | sgroups_group.groups, 52 | ] 53 | 54 | for_each = local.rules_map 55 | 56 | dynamic "items" { 57 | for_each = { 58 | for k, v in each.value.access: 59 | k => v 60 | } 61 | content { 62 | proto = items.key 63 | sg_from = each.value.sg_from 64 | sg_to = each.value.sg_to 65 | 66 | dynamic "ports" { 67 | for_each = { 68 | for access_item in items.value: 69 | "${each.value.sg_from}:${each.value.sg_to}:${substr(sha256(try(join(",", access_item.ports_from), "")), 0, 10)}" => access_item 70 | } 71 | content { 72 | s = try(join(",", ports.value.ports_from), null) 73 | d = try(join(",", ports.value.ports_to), null) 74 | } 75 | 76 | } 77 | 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /modules/k8s-yandex-cluster-infra/main.tf: -------------------------------------------------------------------------------- 1 | 2 | module "k8s-global-vars" { 3 | source = "../k8s-config-vars" 4 | extra_args = var.global_vars 5 | } 6 | 7 | module "k8s-vault" { 8 | depends_on = [ 9 | module.k8s-global-vars 10 | ] 11 | source = "../k8s-vault" 12 | k8s_global_vars = module.k8s-global-vars 13 | } 14 | 15 | module "k8s-vault-master" { 16 | depends_on = [ 17 | module.k8s-vault 18 | ] 19 | source = "../k8s-vault-master" 20 | k8s_global_vars = module.k8s-global-vars 21 | } 22 | 23 | module "k8s-masters" { 24 | depends_on = [ 25 | module.k8s-vault-master, 26 | ] 27 | source = "../k8s-yandex-master-infra" 28 | 29 | k8s_global_vars = module.k8s-global-vars 30 | cloud_metadata = var.cloud_metadata 31 | k8s_vault_master = module.k8s-vault-master 32 | } 33 | 34 | module "k8s-ready-status" { 35 | depends_on = [ 36 | module.k8s-masters 37 | ] 38 | source = "../k8s-ready-status" 39 | cluster_instances = module.k8s-masters.cluster_external_instances_map 40 | k8s_global_vars = module.k8s-global-vars 41 | kube_apiserver_lb = module.k8s-masters.kube-apiserver-lb 42 | } 43 | 44 | 45 | module "addons" { 46 | 47 | source = "../k8s-addons" 48 | 49 | depends_on = [ 50 | module.k8s-ready-status 51 | ] 52 | 53 | k8s_global_vars = module.k8s-global-vars 54 | extra_values = var.global_vars 55 | 56 | } 57 | 58 | module "k8s-masters-firewall" { 59 | count = try(var.global_vars.firewall.enabled, false) == true ? 1 : 0 60 | 61 | source = "../k8s-master-infra-firewall" 62 | cluster_instances_internal = module.k8s-masters.cluster_internal_instances_map 63 | cluster_instances_external = module.k8s-masters.cluster_external_instances_map 64 | cluster_api_ip = try(module.k8s-masters.kube-apiserver-lb, "") 65 | k8s_global_vars = module.k8s-global-vars 66 | } 67 | -------------------------------------------------------------------------------- /modules/k8s-vault-master/templates/vault/vault-bootstarp-approle-all.tftpl: -------------------------------------------------------------------------------- 1 | path "auth/${k8s_global_vars.global_path.base_vault_path_approle}/login" { 2 | capabilities = [ "update" ] 3 | } 4 | 5 | path "auth/token/renew" { 6 | capabilities = ["update"] 7 | } 8 | 9 | path "auth/token/renew-self" { 10 | capabilities = ["update"] 11 | } 12 | 13 | %{~for external_intermediate_name, external_intermediate_content in k8s_global_vars.ssl.external_intermediate ~} 14 | %{~ if try(external_intermediate_content.labels["instance-${instance_type}"], false) == true ~} 15 | 16 | path "${ external_intermediate_content.default.path }/cert/ca" { 17 | capabilities = ["read"] 18 | } 19 | 20 | path "${ external_intermediate_content.default.path }/cert/ca_chain" { 21 | capabilities = ["read"] 22 | } 23 | 24 | %{~ endif ~} 25 | %{~endfor~} 26 | 27 | %{~for intermediate_name, intermediate_content in k8s_global_vars.ssl.intermediate ~} 28 | %{~ if try(intermediate_content.labels["instance-${instance_type}"], false) == true ~} 29 | 30 | path "${ intermediate_content.default.path }/cert/ca" { 31 | capabilities = ["read"] 32 | } 33 | 34 | path "${ intermediate_content.default.path }/cert/ca_chain" { 35 | capabilities = ["read"] 36 | } 37 | 38 | 39 | 40 | %{~ endif ~} 41 | %{~for issuer_name, issuer_content in intermediate_content.issuers ~} 42 | %{~ if try(issuer_content.labels["instance-${instance_type}"], false) == true ~} 43 | 44 | 45 | path "${intermediate_content.default.path}/sign/${issuer_name}" { 46 | capabilities = ["update"] 47 | } 48 | 49 | path "${intermediate_content.default.path}/${issuer_name}/certs" { 50 | capabilities = ["list"] 51 | } 52 | 53 | %{~ endif ~} 54 | %{~endfor~} 55 | %{~endfor~} 56 | 57 | %{for secret_name, secret_content in k8s_global_vars.secrets } 58 | %{~ if try(secret_content.labels["instance-${instance_type}"], false) == true ~} 59 | 60 | path "${secret_content.path}/data/${secret_name}" { 61 | capabilities = ["read"] 62 | } 63 | 64 | %{~ endif ~} 65 | %{~endfor~} 66 | -------------------------------------------------------------------------------- /modules/helm-base-roles/helm/base-roles/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "base.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "base.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "base.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "base.labels" -}} 37 | helm.sh/chart: {{ include "base.chart" . }} 38 | {{ include "base.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "base.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "base.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "base.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "base.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /modules/helm-vault-issuer/helm/base-vault-node-csr/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "base.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "base.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "base.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "base.labels" -}} 37 | helm.sh/chart: {{ include "base.chart" . }} 38 | {{ include "base.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "base.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "base.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "base.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "base.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /modules/k8s-yandex-cluster-infra/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | default = "" 4 | } 5 | 6 | variable "base_domain" { 7 | type = string 8 | default = "" 9 | } 10 | 11 | variable "vault_server" { 12 | type = string 13 | default = "" 14 | } 15 | 16 | variable "service_cidr" { 17 | type = string 18 | default = "172.16.0.0/16" 19 | } 20 | 21 | variable "pod_cidr" { 22 | type = string 23 | default = "10.200.0.0/16" 24 | } 25 | 26 | variable "node_cidr_mask" { 27 | type = string 28 | default = "24" 29 | } 30 | 31 | variable "cloud_metadata" { 32 | type = object({ 33 | cloud_name = string 34 | folder_name = string 35 | }) 36 | default = { 37 | cloud_name = null 38 | folder_name = null 39 | } 40 | } 41 | 42 | variable "global_vars" { 43 | type = any 44 | } 45 | 46 | # variable "master_group"{ 47 | # type = object({ 48 | # name = string 49 | # count = number 50 | # vpc_name = string 51 | # route_table_name = string 52 | # # subnets = any 53 | # default_subnet = optional(string, "") 54 | # default_zone = optional(string, "") 55 | # resources_override = any 56 | # resources = any 57 | # metadata = any 58 | # # ssh_username = string 59 | # # ssh_rsa_path = string 60 | 61 | # }) 62 | # validation { 63 | # condition = ( 64 | # contains([1,3,5], var.master_group.count) == true 65 | # ) 66 | # error_message = "var.master_group.count is not correct. Number of master can be 1,3,5" 67 | # } 68 | # default = { 69 | # name = "master" 70 | # count = 0 71 | # vpc_name = null 72 | # # subnets = null 73 | # route_table_name = "" 74 | # default_subnet = "10.1.0.0/16" 75 | # default_zone = "ru-central1-a" 76 | # resources_override = {} 77 | # metadata = {} 78 | # resources = { 79 | # core = 4 80 | # memory = 8 81 | # core_fraction = 100 82 | # disk = {} 83 | # first_disk = 20 84 | # etcd_disk = 60 85 | # } 86 | # # ssh_username = "dkot" 87 | # # ssh_rsa_path = "~/.ssh/id_rsa" 88 | 89 | # } 90 | # } 91 | -------------------------------------------------------------------------------- /modules/k8s-vault/vault.core.mount.pki.tf: -------------------------------------------------------------------------------- 1 | 2 | resource "vault_mount" "root_ca" { 3 | for_each = var.k8s_global_vars.ssl_for_each_map.root_ca_default_map_only 4 | 5 | allowed_managed_keys = each.value.mount.allowed_managed_keys 6 | audit_non_hmac_request_keys = each.value.mount.audit_non_hmac_request_keys 7 | audit_non_hmac_response_keys = each.value.mount.audit_non_hmac_response_keys 8 | options = each.value.mount.options 9 | external_entropy_access = each.value.mount.external_entropy_access 10 | seal_wrap = each.value.mount.seal_wrap 11 | local = each.value.mount.local 12 | max_lease_ttl_seconds = each.value.mount.max_lease_ttl_seconds 13 | default_lease_ttl_seconds = each.value.mount.default_lease_ttl_seconds 14 | description = each.value.mount.description 15 | type = each.value.mount.mount_type 16 | path = each.value.default.path 17 | } 18 | 19 | resource "vault_mount" "intermediate" { 20 | for_each = var.k8s_global_vars.ssl_for_each_map.intermediate_ca_default_map_only 21 | 22 | allowed_managed_keys = each.value.mount.allowed_managed_keys 23 | audit_non_hmac_request_keys = each.value.mount.audit_non_hmac_request_keys 24 | audit_non_hmac_response_keys = each.value.mount.audit_non_hmac_response_keys 25 | options = each.value.mount.options 26 | external_entropy_access = each.value.mount.external_entropy_access 27 | seal_wrap = each.value.mount.seal_wrap 28 | local = each.value.mount.local 29 | max_lease_ttl_seconds = each.value.mount.max_lease_ttl_seconds 30 | default_lease_ttl_seconds = each.value.mount.default_lease_ttl_seconds 31 | description = each.value.mount.description 32 | type = each.value.mount.mount_type 33 | path = each.value.path 34 | } 35 | 36 | resource "vault_mount" "kubernetes-secrets" { 37 | path = var.k8s_global_vars.global_path.base_vault_path_kv 38 | type = "kv-v2" 39 | description = "KV Version 2 for K8S CP secrets" 40 | options = {} 41 | } 42 | 43 | -------------------------------------------------------------------------------- /modules/k8s-vault-master/README.md: -------------------------------------------------------------------------------- 1 | # Module K8S-vault-master 2 | 3 | This module is designed to create an environment in Vault for master nodes. 4 | 5 | ### Resources created: 6 | 7 | | имя | тип | описание | 8 | | --------------------------------------------------| -------- | ------------------------------------------------------------------ | 9 | | **`vault_policy `** | resource | Defines the permissions for the master node to interact with Vault | 10 | | **`vault_approle_auth_backend_role `** | resource | Creates an Approle under which access will be initialized | 11 | | **`vault_approle_auth_backend_role_secret_id `** | resource | Generates a secret-id, which is a confidential part of the access | 12 | 13 | ### Input variables: 14 | 15 | | имя | тип | описание | 16 | | --------------------------------------------------| -------- | ------------------------------------------------------------------ | 17 | | **`k8s_global_vars `** | any | Module:k8s-config-vars: base vars | 18 | 19 | ### Output variables: 20 | 21 | | имя | тип | описание | 22 | | --------------------------------------------------| -------- | ------------------------------------------------------------------ | 23 | | **`secret_id_all `** | any | The secret_id list was generated from the masters | 24 | | **`role_id_all `** | any | Map the role_id list was generated from the masters | 25 | 26 | Example: 27 | --------- 28 | ```terraform 29 | 30 | module "k8s-global-vars" { 31 | source = "../k8s-config-vars" 32 | extra_args = var.global_vars 33 | } 34 | 35 | module "k8s-vault" { 36 | depends_on = [ 37 | module.k8s-global-vars 38 | ] 39 | source = "../k8s-vault" 40 | k8s_global_vars = module.k8s-global-vars 41 | } 42 | 43 | module "k8s-vault-master" { 44 | depends_on = [ 45 | module.k8s-vault 46 | ] 47 | source = "../k8s-vault-master" 48 | k8s_global_vars = module.k8s-global-vars 49 | } 50 | 51 | ``` -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.addresses.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | cluster_metadata = { 3 | cluster_name = try(var.extra_args.cluster_name, null) == null ? "default" : var.extra_args.cluster_name 4 | base_domain = try(var.extra_args.base_domain, null) == null ? "example.ru" : var.extra_args.base_domain 5 | } 6 | 7 | k8s_network = { 8 | pod_cidr = try(var.extra_args.pod_cidr, null) == null ? "10.0.0.0/16" : var.extra_args.pod_cidr 9 | node_cidr_mask = try(var.extra_args.node_cidr_mask, null) == null ? "24" : var.extra_args.node_cidr_mask 10 | service_cidr = try(var.extra_args.service_cidr, null) == null ? "172.16.0.0/16" : var.extra_args.service_cidr 11 | } 12 | 13 | k8s-addresses-main = { 14 | base_kube_apiserver_lb_fqdn = format("%s.%s.%s", "api" , local.cluster_metadata.cluster_name, local.cluster_metadata.base_domain) 15 | base_kube_apiserver_lb_fqdn_local = format("%s.%s.%s", "api-int" , local.cluster_metadata.cluster_name, local.cluster_metadata.base_domain) 16 | } 17 | 18 | k8s-addresses = { 19 | local_api_address = format("%s.1", join(".", slice(split(".",local.k8s_network.service_cidr), 0, 3)) ) 20 | dns_address = format("%s.10", join(".", slice(split(".",local.k8s_network.service_cidr), 0, 3)) ) 21 | 22 | idp_provider_fqdn = format("auth.%s" , local.cluster_metadata.base_domain) 23 | base_cluster_fqdn = format("%s.%s" , local.cluster_metadata.cluster_name, local.cluster_metadata.base_domain) 24 | wildcard_base_cluster_fqdn = format("%s.%s.%s", "*" , local.cluster_metadata.cluster_name, local.cluster_metadata.base_domain) 25 | etcd_server_lb_fqdn = format("%s.%s.%s", "etcd" , local.cluster_metadata.cluster_name, local.cluster_metadata.base_domain) 26 | 27 | extra_cluster_name = substr(sha256(local.cluster_metadata.cluster_name), 0, 8) 28 | 29 | kube_apiserver_lb_fqdn_local = try(var.extra_args.kube_apiserver_lb_fqdn_local, null) == null ? local.k8s-addresses-main.base_kube_apiserver_lb_fqdn_local : var.extra_args.kube_apiserver_lb_fqdn_local 30 | kube_apiserver_lb_fqdn = try(var.extra_args.kube_apiserver_lb_fqdn, null) == null ? local.k8s-addresses-main.base_kube_apiserver_lb_fqdn : var.extra_args.kube_apiserver_lb_fqdn 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/modules.kubeconfig.tf: -------------------------------------------------------------------------------- 1 | module "kube-apiserver-admin-kubeconfig" { 2 | source = "../kubeconfig" 3 | component-name = "kube-apiserver" 4 | certificate-authority = local.kube-apiserver-admin-kubeconfig-certificate-authority 5 | client-certificate = local.kube-apiserver-admin-kubeconfig-client-certificate 6 | client-key = local.kube-apiserver-admin-kubeconfig-client-key 7 | kube-apiserver-port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port 8 | } 9 | 10 | module "kubelet-kubeconfig" { 11 | source = "../kubeconfig" 12 | component-name = "kubelet" 13 | certificate-authority = local.kubelet-kubeconfig-certificate-authority 14 | client-certificate = local.kubelet-kubeconfig-client-certificate 15 | client-key = local.kubelet-kubeconfig-client-key 16 | kube-apiserver-port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port 17 | } 18 | 19 | module "kubelet-bootstrap-kubeconfig" { 20 | source = "../kubeconfig" 21 | component-name = "kubelet" 22 | certificate-authority = local.kubelet-bootstrap-kubeconfig-certificate-authority 23 | client-certificate = local.kubelet-bootstrap-kubeconfig-client-certificate 24 | client-key = local.kubelet-bootstrap-kubeconfig-client-key 25 | kube-apiserver-port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port-lb 26 | kube-apiserver = var.k8s_global_vars.k8s-addresses.kube_apiserver_lb_fqdn 27 | 28 | } 29 | 30 | module "kube-scheduler-kubeconfig" { 31 | source = "../kubeconfig" 32 | component-name = "kube-scheduler" 33 | certificate-authority = local.kube-scheduler-kubeconfig-certificate-authority 34 | client-certificate = local.kube-scheduler-kubeconfig-client-certificate 35 | client-key = local.kube-scheduler-kubeconfig-client-key 36 | kube-apiserver-port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port 37 | } 38 | 39 | module "kube-controller-manager-kubeconfig" { 40 | source = "../kubeconfig" 41 | component-name = "kube-controller-manager" 42 | certificate-authority = local.kube-controller-manager-kubeconfig-certificate-authority 43 | client-certificate = local.kube-controller-manager-kubeconfig-client-certificate 44 | client-key = local.kube-controller-manager-kubeconfig-client-key 45 | kube-apiserver-port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port 46 | } -------------------------------------------------------------------------------- /modules/k8s-vault/README.md: -------------------------------------------------------------------------------- 1 | Module K8S-vault 2 | ================ 3 | 4 | #### This module is focused on creating a Vault environment for your future K8S cluster. 5 | 6 | 7 | Resources created: 8 | --------- 9 | | NAME | TYPE | DESCRIPTIONS | 10 | | ----------------------------------------------------------| -------- | ---------------------------------------------------------- | 11 | | **`vault_mount `** | resource | Created a PKI/KV2 store for future certificates/secrets | 12 | | **`vault_pki_secret_backend_root_cert `** | resource | Generates a new self-signed CA certificate for the PKI | 13 | | **`vault_pki_secret_backend_intermediate_cert_request `** | resource | Generates a new private key and a CSR for signing the PKI | 14 | | **`vault_pki_secret_backend_root_sign_intermediate `** | resource | Creates PKI certificate | 15 | | **`vault_pki_secret_backend_intermediate_set_signed `** | resource | Submits the CA certificate to the PKI | 16 | | **`vault_auth_backend `** | resource | Creates a base authorization path via approle | 17 | | **`vault_pki_secret_backend_role `** | resource | Creates a role on an PKI Secret Backend for Vault | 18 | 19 | Input variables: 20 | --------- 21 | | NAME | TYPE | DESCRIPTIONS | 22 | | ----------------------------------------------------------| -------- | ---------------------------------------------------------- | 23 | | **`k8s_global_vars `** | any | Module:k8s-config-vars: base vars | 24 | 25 | 26 | Example: 27 | --------- 28 | ```terraform 29 | module "k8s-global-vars" { 30 | source = "../k8s-config-vars" 31 | extra_args = var.global_vars 32 | } 33 | 34 | module "k8s-vault" { 35 | depends_on = [ 36 | module.k8s-global-vars 37 | ] 38 | source = "../k8s-vault" 39 | k8s_global_vars = module.k8s-global-vars 40 | } 41 | ``` 42 | 43 | Attention: 44 | ----------- 45 | 46 | #### The creation structure depends on the structure of the k8s_global_vars input variable defined through the k8s-global-vars module 47 | #### SEE SPECIFICATION -> [LINK](https://github.com/fraima/terraform-modules/blob/main/modules/k8s-config-vars/locals.certs.tf) -------------------------------------------------------------------------------- /modules/k8s-vault/vault.certificate.roles.tf: -------------------------------------------------------------------------------- 1 | resource "vault_pki_secret_backend_role" "kubernetes-role" { 2 | depends_on = [ 3 | vault_mount.intermediate 4 | ] 5 | 6 | for_each = var.k8s_global_vars.ssl_for_each_map.issuers_content_map_only 7 | 8 | backend = var.k8s_global_vars.ssl.intermediate[split(":","${each.key}")[0]].default.path 9 | name = split(":","${each.key}")[1] 10 | 11 | allow_any_name = each.value.allow_any_name 12 | allow_bare_domains = each.value.allow_bare_domains 13 | allow_glob_domains = each.value.allow_glob_domains 14 | allow_ip_sans = each.value.allow_ip_sans 15 | allow_localhost = each.value.allow_localhost 16 | allow_subdomains = each.value.allow_subdomains 17 | allowed_domains_template = each.value.allowed_domains_template 18 | allowed_domains = each.value.allowed_domains 19 | allowed_other_sans = each.value.allowed_other_sans 20 | allowed_serial_numbers = each.value.allowed_serial_numbers 21 | allowed_uri_sans = each.value.allowed_uri_sans 22 | basic_constraints_valid_for_non_ca = each.value.basic_constraints_valid_for_non_ca 23 | client_flag = each.value.client_flag 24 | server_flag = each.value.server_flag 25 | code_signing_flag = each.value.code_signing_flag 26 | email_protection_flag = each.value.email_protection_flag 27 | enforce_hostnames = each.value.enforce_hostnames 28 | generate_lease = each.value.generate_lease 29 | key_bits = each.value.key_bits 30 | key_type = each.value.key_type 31 | key_usage = each.value.key_usage 32 | ext_key_usage = each.value.ext_key_usage 33 | no_store = each.value.no_store 34 | require_cn = each.value.require_cn 35 | ttl = each.value.ttl 36 | use_csr_common_name = each.value.use_csr_common_name 37 | organization = each.value.organization 38 | country = each.value.country 39 | locality = each.value.locality 40 | ou = each.value.ou 41 | postal_code = each.value.postal_code 42 | province = each.value.province 43 | street_address = each.value.street_address 44 | } 45 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/locals.instances.tf: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | locals { 5 | # Формируется массивы для будущих for_each с маской "${intermediate_name}:${issuer_name}" 6 | issuers_content_only = flatten([ 7 | for intermediate_name in keys(local.ssl.intermediate) : [ 8 | for issuer_name,issuer in local.ssl.intermediate[intermediate_name].issuers : 9 | { 10 | "${intermediate_name}:${issuer_name}" = merge(local.ssl["global-args"]["issuer-args"], issuer["issuer-args"]) 11 | } 12 | 13 | ] 14 | ] 15 | ) 16 | root_ca_default_only = flatten([ 17 | for root_name, root_value in local.ssl.root_ca : { 18 | "${root_name}" = merge(local.ssl["global-args"]["root-ca-args"], try(root_value["default"], {})) 19 | } 20 | ]) 21 | 22 | intermediate_ca_default_only = flatten([ 23 | for intermediate_name, intermediate_value in local.ssl.intermediate : { 24 | "${intermediate_name}" = merge(local.ssl["global-args"]["intermediate-ca-args"], try(intermediate_value["default"], {})) 25 | } 26 | ]) 27 | 28 | secret_content_only = flatten([ 29 | for secret_name in keys(local.secrets) : 30 | {"${secret_name}" = {}} 31 | ] 32 | ) 33 | 34 | ssl_for_each_map = { 35 | 36 | secret_content_map_only = { for item in local.secret_content_only : 37 | keys(item)[0] => values(item)[0] 38 | } 39 | 40 | issuers_content_map_only = { for item in local.issuers_content_only : 41 | keys(item)[0] => values(item)[0] 42 | } 43 | 44 | root_ca_default_map_only = { for item in local.root_ca_default_only : 45 | keys(item)[0] => values(item)[0] 46 | } 47 | 48 | intermediate_ca_default_map_only = { for item in local.intermediate_ca_default_only : 49 | keys(item)[0] => values(item)[0] 50 | } 51 | 52 | } 53 | 54 | master_instance_list = flatten([ 55 | for master-index in range(var.extra_args.master_group.count): [ 56 | "${var.extra_args.master_group.name}-${sum([master-index, 1])}" 57 | ] 58 | ]) 59 | 60 | master_instance_list_map = { for item in local.master_instance_list : 61 | item => {} 62 | } 63 | 64 | master_instance_extra_list = flatten([ 65 | for master-index in range(var.extra_args.master_group.count): [ 66 | "${var.extra_args.master_group.name}-${local.k8s-addresses.extra_cluster_name}-${sum([master-index, 1])}" 67 | ] 68 | ]) 69 | 70 | master_instance_extra_list_map = { for item in local.master_instance_extra_list : 71 | item => {} 72 | } 73 | 74 | master_vars = { 75 | master_group = var.extra_args.master_group 76 | master_instance_list = local.master_instance_list 77 | master_instance_list_map = local.master_instance_list_map 78 | master_instance_extra_list = local.master_instance_extra_list 79 | master_instance_extra_list_map = local.master_instance_extra_list_map 80 | } 81 | } -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/locals.kubeconfig.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | kubernetes-ca-ssl = var.k8s_global_vars.ssl.intermediate["kubernetes-ca"] 4 | 5 | kube-apiserver-admin-kubeconfig-certificate-authority = "${local.kubernetes-ca-ssl.default.host_path}/kubernetes-ca.pem" 6 | kube-apiserver-admin-kubeconfig-client-certificate = "${local.kubernetes-ca-ssl.issuers["kubeadm-client"].certificates["kubeadm-client"].key-keeper-args.host_path}/kubeadm-client.pem" 7 | kube-apiserver-admin-kubeconfig-client-key = "${local.kubernetes-ca-ssl.issuers["kubeadm-client"].certificates["kubeadm-client"].key-keeper-args.host_path}/kubeadm-client-key.pem" 8 | 9 | kubelet-bootstrap-kubeconfig-certificate-authority = "${local.kubernetes-ca-ssl.default.host_path}/kubernetes-ca.pem" 10 | kubelet-bootstrap-kubeconfig-client-certificate = "${local.kubernetes-ca-ssl.issuers["bootstrappers-client"].certificates["bootstrappers-client"].key-keeper-args.host_path}/bootstrappers-client.pem" 11 | kubelet-bootstrap-kubeconfig-client-key = "${local.kubernetes-ca-ssl.issuers["bootstrappers-client"].certificates["bootstrappers-client"].key-keeper-args.host_path}/bootstrappers-client-key.pem" 12 | 13 | kubelet-kubeconfig-certificate-authority = "${local.kubernetes-ca-ssl.default.host_path}/kubernetes-ca.pem" 14 | kubelet-kubeconfig-client-certificate = "${local.kubernetes-ca-ssl.issuers["kubelet-client"].certificates["kubelet-client"].key-keeper-args.host_path}/kubelet-client.pem" 15 | kubelet-kubeconfig-client-key = "${local.kubernetes-ca-ssl.issuers["kubelet-client"].certificates["kubelet-client"].key-keeper-args.host_path}/kubelet-client-key.pem" 16 | 17 | kube-scheduler-kubeconfig-certificate-authority = "${local.kubernetes-ca-ssl.default.host_path}/kubernetes-ca.pem" 18 | kube-scheduler-kubeconfig-client-certificate = "${local.kubernetes-ca-ssl.issuers["kube-scheduler-client"].certificates["kube-scheduler-client"].key-keeper-args.host_path}/kube-scheduler-client.pem" 19 | kube-scheduler-kubeconfig-client-key = "${local.kubernetes-ca-ssl.issuers["kube-scheduler-client"].certificates["kube-scheduler-client"].key-keeper-args.host_path}/kube-scheduler-client-key.pem" 20 | 21 | kube-controller-manager-kubeconfig-certificate-authority = "${local.kubernetes-ca-ssl.default.host_path}/kubernetes-ca.pem" 22 | kube-controller-manager-kubeconfig-client-certificate = "${local.kubernetes-ca-ssl.issuers["kube-controller-manager-client"].certificates["kube-controller-manager-client"].key-keeper-args.host_path}/kube-controller-manager-client.pem" 23 | kube-controller-manager-kubeconfig-client-key = "${local.kubernetes-ca-ssl.issuers["kube-controller-manager-client"].certificates["kube-controller-manager-client"].key-keeper-args.host_path}/kube-controller-manager-client-key.pem" 24 | } 25 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/default/kubelet.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: "kubelet.config.k8s.io/v1beta1" 3 | kind: "KubeletConfiguration" 4 | registerNode: true 5 | cgroupDriver: "systemd" 6 | clusterDomain: "cluster.local" 7 | cpuManagerReconcilePeriod: "0s" 8 | fileCheckFrequency: "0s" 9 | healthzBindAddress: "127.0.0.1" 10 | httpCheckFrequency: "0s" 11 | imageMinimumGCAge: "0s" 12 | memorySwap: {} 13 | nodeStatusReportFrequency: "1s" 14 | nodeStatusUpdateFrequency: "1s" 15 | resolvConf: "/run/systemd/resolve/resolv.conf" 16 | runtimeRequestTimeout: "0s" 17 | shutdownGracePeriod: "15s" 18 | shutdownGracePeriodCriticalPods: "5s" 19 | streamingConnectionIdleTimeout: "0s" 20 | syncFrequency: "0s" 21 | volumeStatsAggPeriod: "0s" 22 | containerLogMaxSize: "50Mi" 23 | maxPods: 250 24 | kubeAPIQPS: 50 25 | kubeAPIBurst: 100 26 | podPidsLimit: 4096 27 | serializeImagePulls: false 28 | rotateCertificates: false 29 | serverTLSBootstrap: true 30 | tlsMinVersion: "VersionTLS12" 31 | evictionPressureTransitionPeriod: "5s" 32 | imageGCHighThresholdPercent: 55 33 | imageGCLowThresholdPercent: 50 34 | 35 | authentication: 36 | anonymous: 37 | enabled: false 38 | webhook: 39 | cacheTTL: "0s" 40 | enabled: true 41 | 42 | authorization: 43 | mode: "Webhook" 44 | webhook: 45 | cacheAuthorizedTTL: "0s" 46 | cacheUnauthorizedTTL: "0s" 47 | 48 | logging: 49 | flushFrequency: 0 50 | options: 51 | json: 52 | infoBufferSize: 0 53 | verbosity: 0 54 | 55 | systemReserved: 56 | ephemeral-storage: "1Gi" 57 | 58 | 59 | featureGates: 60 | RotateKubeletServerCertificate: true 61 | APIPriorityAndFairness: true 62 | DownwardAPIHugePages: true 63 | PodSecurity: true 64 | CSIMigrationAWS: false 65 | CSIMigrationAzureFile: false 66 | CSIMigrationGCE: false 67 | CSIMigrationvSphere: false 68 | 69 | 70 | tlsCipherSuites: 71 | - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" 72 | - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" 73 | - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" 74 | - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" 75 | - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" 76 | - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" 77 | 78 | allowedUnsafeSysctls: 79 | - "net.core.somaxconn" 80 | 81 | evictionSoft: 82 | "memory.available": "3Gi" 83 | "nodefs.available": "25%" 84 | "nodefs.inodesFree": "15%" 85 | "imagefs.available": "30%" 86 | "imagefs.inodesFree": "25%" 87 | 88 | evictionSoftGracePeriod: 89 | "memory.available": "2m30s" 90 | "nodefs.available": "2m30s" 91 | "nodefs.inodesFree": "2m30s" 92 | "imagefs.available": "2m30s" 93 | "imagefs.inodesFree": "2m30s" 94 | 95 | evictionHard: 96 | "memory.available": "2Gi" 97 | "nodefs.available": "20%" 98 | "nodefs.inodesFree": "10%" 99 | "imagefs.available": "25%" 100 | "imagefs.inodesFree": "15%" 101 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/yandex.cluster.instance.masters.tf: -------------------------------------------------------------------------------- 1 | #### MASTERS ###### 2 | ##--> 3 | 4 | resource "yandex_compute_instance" "master" { 5 | depends_on = [ 6 | yandex_lockbox_secret_version.master_key_keeper_approles_secret_id_all, 7 | yandex_lockbox_secret_version.master_key_keeper_approles_role_id_all, 8 | ] 9 | for_each = var.k8s_global_vars.master_vars.master_instance_list_map 10 | 11 | name = "${replace(each.key, "-", "-${local.extra_cluster_name}-")}" 12 | hostname = "${replace(each.key, "-", "-${local.extra_cluster_name}-")}.${local.base_cluster_fqdn}" 13 | 14 | description = local.cluster_name 15 | 16 | platform_id = "standard-v1" 17 | 18 | zone = try( 19 | var.k8s_global_vars.master_vars.master_group.resources_override[each.key].network_interface.zone, 20 | var.k8s_global_vars.master_vars.master_group.default_zone 21 | ) 22 | 23 | service_account_id = data.yandex_iam_service_account.yandex-k8s-controllers.id 24 | 25 | resources { 26 | cores = var.k8s_global_vars.master_vars.master_group.resources.core 27 | memory = var.k8s_global_vars.master_vars.master_group.resources.memory 28 | core_fraction = var.k8s_global_vars.master_vars.master_group.resources.core_fraction 29 | } 30 | 31 | boot_disk { 32 | initialize_params { 33 | image_id = try( 34 | var.k8s_global_vars.master_vars.master_group.resources_override[each.key].disk.boot.image_id, 35 | var.k8s_global_vars.master_vars.master_group.resources.disk.boot.image_id 36 | ) 37 | size = var.k8s_global_vars.master_vars.master_group.resources.disk.boot.size 38 | type = var.k8s_global_vars.master_vars.master_group.resources.disk.boot.type 39 | } 40 | } 41 | 42 | dynamic "secondary_disk" { 43 | for_each = { 44 | for k, v in local.instances_disk_map: 45 | k => v 46 | if split("_", k )[1] == each.key 47 | } 48 | content { 49 | disk_id = yandex_compute_disk.etcd[secondary_disk.key].id 50 | auto_delete = local.master_secondary_disk[split("_", secondary_disk.key)[0]].auto_delete 51 | mode = local.master_secondary_disk[split("_", secondary_disk.key)[0]].mode 52 | device_name = join("-", [split("_", secondary_disk.key)[0], "data"]) 53 | } 54 | } 55 | 56 | network_interface { 57 | subnet_id = yandex_vpc_subnet.master-subnets[ 58 | "${try( 59 | var.k8s_global_vars.master_vars.master_group.resources_override[each.key].network_interface.subnet, 60 | var.k8s_global_vars.master_vars.master_group.default_subnet 61 | )}:${try( 62 | var.k8s_global_vars.master_vars.master_group.resources_override[each.key].network_interface.zone, 63 | var.k8s_global_vars.master_vars.master_group.default_zone 64 | )}" 65 | ].id 66 | nat = var.k8s_global_vars.master_vars.master_group.resources.network_interface.nat 67 | } 68 | 69 | labels = {} 70 | 71 | lifecycle { 72 | ignore_changes = [ 73 | metadata 74 | ] 75 | } 76 | 77 | metadata = { 78 | user-data = local.user_data[replace(each.key, "-", "-${local.extra_cluster_name}-")] 79 | } 80 | } -------------------------------------------------------------------------------- /modules/helm-victoria-metrics-stack-operator/helm/vm-operator.yaml: -------------------------------------------------------------------------------- 1 | operator: 2 | createCRD: true 3 | cleanupCRD: false 4 | cleanupSA: 5 | create: false 6 | 7 | serviceAccount: 8 | create: false 9 | 10 | victoria-metrics-operator: 11 | replicaCount: 2 12 | 13 | tolerations: 14 | - key: node-role.kubernetes.io/master 15 | effect: NoSchedule 16 | - key: "CriticalAddonsOnly" 17 | operator: "Exists" 18 | - key: node.cloudprovider.kubernetes.io/uninitialized 19 | value: "true" 20 | effect: NoSchedule 21 | 22 | rbac: 23 | create: true 24 | pspEnabled: true 25 | 26 | operator: 27 | # -- By default, operator converts prometheus-operator objects. 28 | disable_prometheus_converter: true 29 | # -- By default, operator creates psp for its objects. 30 | psp_auto_creation_enabled: false 31 | # -- Enables ownership reference for converted prometheus-operator objects, 32 | # it will remove corresponding victoria-metrics objects in case of deletion prometheus one. 33 | enable_converter_ownership: false 34 | # -- Enables custom config-reloader, bundled with operator. 35 | # It should reduce vmagent and vmauth config sync-time and make it predictable. 36 | useCustomConfigReloader: true 37 | 38 | env: 39 | - name: VM_VMAGENTDEFAULT_CONFIGRELOADERMEMORY 40 | value: '100Mi' 41 | - name: VM_VMALERTMANAGER_CONFIGRELOADERMEMORY 42 | value: '100Mi' 43 | - name: VM_VMALERTDEFAULT_CONFIGRELOADERMEMORY 44 | value: '100Mi' 45 | 46 | resources: 47 | requests: 48 | # C-0050 - Resources CPU limit and request 49 | cpu: 2000m 50 | # C-0004 - Resources memory limit and request 51 | memory: 2Gi 52 | limits: 53 | # C-0050 - Resources CPU limit and request 54 | cpu: 2000m 55 | # C-0004 - Resources memory limit and request 56 | memory: 2Gi 57 | 58 | # securityContext: 59 | # C-0013 - Non-root containers 60 | # Error: container has runAsNonRoot and image has non-numeric user (vm-operator), cannot verify user is non-root 61 | # runAsNonRoot: true 62 | # C-0017 - Immutable container filesystem 63 | # unknown field "readOnlyRootFilesystem" in io.k8s.api.core.v1.PodSecurityContext 64 | # readOnlyRootFilesystem: true 65 | 66 | affinity: 67 | podAntiAffinity: 68 | preferredDuringSchedulingIgnoredDuringExecution: 69 | - podAffinityTerm: 70 | labelSelector: 71 | matchExpressions: 72 | - key: app.kubernetes.io/name 73 | operator: In 74 | values: 75 | - victoria-metrics-operator 76 | - key: app.kubernetes.io/instance 77 | operator: In 78 | values: 79 | - ${release_name} 80 | topologyKey: failure-domain.beta.kubernetes.io/region 81 | weight: 100 82 | 83 | admissionWebhooks: 84 | enabled: false 85 | policy: Fail 86 | # -- Enables custom ca bundle, if you are not using cert-manager. 87 | # -- in case of custom ca, you have to create secret - \{\{chart-name\}\}-validation 88 | # -- with keys: tls.key, tls.crt, ca.crt 89 | caBundle: "" 90 | certManager: 91 | # -- Enables cert creation and injection by cert-manager. 92 | enabled: false 93 | # --If needed, provide own issuer. Operator will create self-signed if empty. 94 | issuer: {} 95 | -------------------------------------------------------------------------------- /modules/k8s-vault/vault.core.create.ca.tf: -------------------------------------------------------------------------------- 1 | 2 | ##### K8S INTERMEDIATE ROOT ##################### 3 | ### -> 4 | resource "vault_pki_secret_backend_root_cert" "root" { 5 | 6 | for_each = var.k8s_global_vars.ssl_for_each_map.root_ca_default_map_only 7 | 8 | backend = vault_mount.root_ca[each.key].path 9 | 10 | common_name = each.value.common_name 11 | type = each.value.type 12 | format = each.value.format 13 | private_key_format = each.value.private_key_format 14 | key_type = each.value.key_type 15 | key_bits = each.value.key_bits 16 | exclude_cn_from_sans = each.value.exclude_cn_from_sans 17 | ou = each.value.ou 18 | organization = each.value.organization 19 | country = each.value.country 20 | locality = each.value.locality 21 | province = each.value.province 22 | street_address = each.value.street_address 23 | postal_code = each.value.postal_code 24 | alt_names = each.value.alt_names 25 | ip_sans = each.value.ip_sans 26 | uri_sans = each.value.uri_sans 27 | other_sans = each.value.other_sans 28 | ttl = each.value.ttl 29 | max_path_length = each.value.max_path_length 30 | permitted_dns_domains = each.value.permitted_dns_domains 31 | 32 | } 33 | 34 | ##### K8S INTERMEDIATE ##################### 35 | ### -> 36 | resource "vault_pki_secret_backend_intermediate_cert_request" "intermediate" { 37 | depends_on = [ 38 | vault_pki_secret_backend_root_cert.root 39 | ] 40 | 41 | for_each = var.k8s_global_vars.ssl_for_each_map.intermediate_ca_default_map_only 42 | 43 | backend = vault_mount.intermediate[each.key].path 44 | 45 | common_name = each.value.common_name 46 | type = each.value.type 47 | format = each.value.format 48 | private_key_format = each.value.private_key_format 49 | key_type = each.value.key_type 50 | key_bits = each.value.key_bits 51 | exclude_cn_from_sans = each.value.exclude_cn_from_sans 52 | ou = each.value.ou 53 | organization = each.value.organization 54 | country = each.value.country 55 | locality = each.value.locality 56 | province = each.value.province 57 | street_address = each.value.street_address 58 | postal_code = each.value.postal_code 59 | alt_names = each.value.alt_names 60 | ip_sans = each.value.ip_sans 61 | uri_sans = each.value.uri_sans 62 | other_sans = each.value.other_sans 63 | add_basic_constraints = each.value.add_basic_constraints 64 | 65 | } 66 | 67 | resource "vault_pki_secret_backend_root_sign_intermediate" "intermediate" { 68 | for_each = var.k8s_global_vars.ssl_for_each_map.intermediate_ca_default_map_only 69 | backend = each.value.root_path 70 | csr = vault_pki_secret_backend_intermediate_cert_request.intermediate[each.key].csr 71 | common_name = vault_pki_secret_backend_intermediate_cert_request.intermediate[each.key].common_name 72 | revoke = each.value.sign.revoke 73 | } 74 | 75 | resource "vault_pki_secret_backend_intermediate_set_signed" "intermediate" { 76 | for_each = var.k8s_global_vars.ssl.intermediate 77 | backend = vault_mount.intermediate[each.key].path 78 | certificate = vault_pki_secret_backend_root_sign_intermediate.intermediate[each.key].certificate 79 | } 80 | -------------------------------------------------------------------------------- /modules/k8s-templates/cloud-init-master/locals.template.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | cloud-init-master = flatten([ 4 | for master_name, master_content in var.k8s_global_vars.master_vars.master_instance_extra_list_map: 5 | {"${master_name}" = templatefile("${path.module}/templates/cloud-init-kubeadm-master-${try(var.k8s_global_vars.master_vars.master_group.metadata.user_data_template, "all")}.tftpl", { 6 | ssh_username = var.k8s_global_vars.base.ssh_username 7 | ssh_key = file(var.k8s_global_vars.base.ssh_rsa_path) 8 | base_local_path_certs = var.k8s_global_vars.global_path.base_local_path_certs 9 | ssl = var.k8s_global_vars.ssl 10 | base_path = var.k8s_global_vars.global_path 11 | main_path = var.k8s_global_vars.main_path 12 | # hostname = "${master_name}-${var.k8s_global_vars.cluster_metadata.cluster_name}" 13 | hostname = "${master_name}" 14 | node_name = "${master_name}" 15 | actual_release = var.actual-release 16 | 17 | # kube_apiserver_lb_fqdn = var.kube-apiserver-lb-fqdn 18 | # kube_apiserver_port_lb = var.kube-apiserver-port-lb 19 | # bootstrap_token_all = var.bootstrap_token_all[master_name].client_token 20 | 21 | # DEDICATED VAULT BOOTSTRAP TOKENS 22 | # vault-bootstrap-issuer-master-token = module.k8s-vault-master.bootstrap-issuer-master-token 23 | # vault-bootstrap-ca-master-token = module.k8s-vault-master.bootstrap-ca-master-token 24 | # vault-bootstrap-external-ca-master-token = module.k8s-vault-master.bootstrap-external-ca-master-token 25 | # vault-bootstrap-secret-master-token = module.k8s-vault-master.bootstrap-secret-master-token 26 | 27 | kube-apiserver-admin-kubeconfig = module.kube-apiserver-admin-kubeconfig.kubeconfig 28 | kubelet-kubeconfig = module.kubelet-kubeconfig.kubeconfig 29 | kube-scheduler-kubeconfig = module.kube-scheduler-kubeconfig.kubeconfig 30 | kube-controller-manager-kubeconfig = module.kube-controller-manager-kubeconfig.kubeconfig 31 | 32 | # kubelet-service = module.kubelet-service-master.kubelet-service 33 | # kubelet-service-d-fraima = module.kubelet-service-master.kubelet-service-d-fraima 34 | # kubelet-service-args = module.kubelet-service-master.kubelet-service-args[master_name] 35 | kubelet-config = module.kubelet-service-master.kubelet-config 36 | 37 | key-keeper-config = module.key-keeper-service-master.key-keeper-config[master_name] 38 | key-keeper-service = module.key-keeper-service-master.key-keeper-service 39 | 40 | static-pod-etcd = module.static-pod-etcd.manifest[master_name] 41 | static-pod-kubeadm-config = module.static-pod-kubeadm-config.manifest[master_name] 42 | # static-pod-kube-apiserver = module.static-pod-kube-apiserver.manifest[master_name] 43 | # static-pod-kube-controller-manager = module.static-pod-kube-controller-manager.manifest[master_name] 44 | # static-pod-kube-scheduler = module.static-pod-kube-scheduler.manifest[master_name] 45 | # kube-apiserver-audit = module.static-pod-kube-apiserver.kube-apiserver-audit 46 | 47 | # containerd-service = module.containerd-service.service 48 | # containerd-service-config = module.containerd-service.service-config 49 | # sysctl-network = module.sysctl.network 50 | # modprobe-k8s = module.modprobe.k8s 51 | # cni-base = module.cni.base 52 | bashrc-k8s = module.bashrc.k8s 53 | 54 | component_versions = var.k8s_global_vars.component_versions 55 | })} 56 | ]) 57 | cloud-init-master-map = { for item in local.cloud-init-master : 58 | keys(item)[0] => values(item)[0]} 59 | 60 | } 61 | 62 | -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/etcd/templates/etcd.yaml.tftpl: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Pod 4 | metadata: 5 | creationTimestamp: null 6 | labels: 7 | component: etcd 8 | tier: control-plane 9 | name: etcd 10 | namespace: kube-system 11 | spec: 12 | containers: 13 | - name: etcd 14 | command: 15 | - etcd 16 | args: 17 | #- --discovery-srv=${discovery_srv} 18 | - --name=${full_instance_name} 19 | - --initial-cluster=${etcd_initial_cluster} 20 | - --initial-cluster-token=etcd 21 | - --initial-cluster-state=new 22 | - --initial-advertise-peer-urls=https://${full_instance_name}:${etcd_peer_port} 23 | - --advertise-client-urls=https://${full_instance_name}:${etcd_server_port} 24 | - --listen-client-urls=https://0.0.0.0:${etcd_server_port} 25 | - --listen-peer-urls=https://0.0.0.0:${etcd_peer_port} 26 | - --listen-metrics-urls=http://0.0.0.0:${etcd_metrics_port} 27 | 28 | %{~ for intermediate_name, intermediate_content in ssl.intermediate ~} 29 | %{~ if try(intermediate_content.labels.static-pod-etcd-args, {}) != {} ~} 30 | %{~ for key, value in intermediate_content.labels.static-pod-etcd-args ~} 31 | %{~ if value == "cert-public-arg" ~} 32 | - --${key}=${intermediate_content.default.host_path}/${intermediate_name}.pem 33 | %{~ endif ~} 34 | %{~ if value == "cert-private-arg" ~} 35 | - --${key}=${intermediate_content.default.host_path}/${intermediate_name}-key.pem 36 | %{~ endif ~} 37 | %{~ endfor ~} 38 | %{~ endif ~} 39 | %{~ for issuer_name, issuer_content in intermediate_content.issuers ~} 40 | %{~ for certificate_name, certificate_content in issuer_content.certificates ~} 41 | %{~ if try(certificate_content.labels.static-pod-etcd-args, {}) != {} ~} 42 | %{~ for key, value in certificate_content.labels.static-pod-etcd-args ~} 43 | %{~ if value == "cert-public-arg" ~} 44 | - --${key}=${certificate_content.key-keeper-args.host_path}/${certificate_name}.pem 45 | %{~ endif ~} 46 | %{~ if value == "cert-private-arg" ~} 47 | - --${key}=${certificate_content.key-keeper-args.host_path}/${certificate_name}-key.pem 48 | %{~ endif ~} 49 | %{~ endfor ~} 50 | %{~ endif ~} 51 | %{~ endfor ~} 52 | %{~ endfor ~} 53 | %{~ endfor ~} 54 | 55 | - --data-dir=${data_dir} 56 | - --strict-reconfig-check=true 57 | - --peer-client-cert-auth=true 58 | - --peer-auto-tls=true 59 | - --client-cert-auth=true 60 | - --snapshot-count=10000 61 | - --heartbeat-interval=250 62 | - --election-timeout=1500 63 | - --quota-backend-bytes=0 64 | - --max-snapshots=10 65 | - --max-wals=10 66 | - --discovery-fallback=proxy 67 | - --auto-compaction-retention=8 68 | - --force-new-cluster=false 69 | - --enable-v2=false 70 | - --proxy=off 71 | - --proxy-failure-wait=5000 72 | - --proxy-refresh-interval=30000 73 | - --proxy-dial-timeout=1000 74 | - --proxy-write-timeout=5000 75 | - --proxy-read-timeout=0 76 | - --metrics=extensive 77 | - --logger=zap 78 | image: ${component_versions.static_pod.etcd.registry}:${component_versions.static_pod.etcd.version} 79 | imagePullPolicy: IfNotPresent 80 | livenessProbe: 81 | failureThreshold: 8 82 | httpGet: 83 | host: 127.0.0.1 84 | path: /health 85 | port: ${etcd_metrics_port} 86 | scheme: HTTP 87 | initialDelaySeconds: 10 88 | periodSeconds: 10 89 | timeoutSeconds: 15 90 | resources: 91 | requests: 92 | cpu: 100m 93 | memory: 100Mi 94 | startupProbe: 95 | failureThreshold: 24 96 | httpGet: 97 | host: 127.0.0.1 98 | path: /health 99 | port: ${etcd_metrics_port} 100 | scheme: HTTP 101 | volumeMounts: 102 | - mountPath: /var/lib/etcd 103 | name: etcd-data 104 | - mountPath: ${base_local_path_certs}/certs/etcd 105 | name: etcd-certs 106 | - mountPath: ${base_local_path_certs}/ca 107 | name: ca 108 | hostNetwork: true 109 | priorityClassName: system-node-critical 110 | securityContext: 111 | null 112 | volumes: 113 | - hostPath: 114 | path: ${base_local_path_certs}/certs/etcd 115 | type: DirectoryOrCreate 116 | name: etcd-certs 117 | - hostPath: 118 | path: ${base_local_path_certs}/ca 119 | type: DirectoryOrCreate 120 | name: ca 121 | - hostPath: 122 | path: /var/lib/etcd 123 | type: DirectoryOrCreate 124 | name: etcd-data 125 | status: {} 126 | -------------------------------------------------------------------------------- /modules/k8s-addons/main.tf: -------------------------------------------------------------------------------- 1 | module "base-roles" { 2 | count = try(var.extra_values.addons.base.enabled, false) == true ? 1 : 0 3 | source = "../helm-base-roles" 4 | 5 | global_vars = var.k8s_global_vars 6 | extra_values = try(var.extra_values.addons.cilium, {}) 7 | } 8 | 9 | module "cilium" { 10 | count = try(var.extra_values.addons.cilium.enabled, false) == true ? 1 : 0 11 | 12 | source = "../helm-yandex-cilium" 13 | depends_on = [ 14 | module.yandex-cloud-controller, 15 | ] 16 | chart_version = "1.12.6" 17 | 18 | global_vars = var.k8s_global_vars 19 | extra_values = try(var.extra_values.addons.cilium, {}) 20 | } 21 | 22 | module "yandex-cloud-controller" { 23 | count = try(var.extra_values.addons.yandex-cloud-controller.enabled, false) == true ? 1 : 0 24 | 25 | source = "../helm-yandex-cloud-controller" 26 | 27 | chart_version = "0.0.7" 28 | 29 | yandex_default_vpc_name = var.k8s_global_vars.master_vars.master_group.vpc_name 30 | yandex_default_route_table_name = var.k8s_global_vars.master_vars.master_group.route_table_name 31 | namespace = "kube-fraima-yandex-cloud-controller" 32 | 33 | global_vars = var.k8s_global_vars 34 | 35 | extra_values = try(var.extra_values.addons.yandex-cloud-controller, {}) 36 | } 37 | 38 | module "yandex-csi-controller" { 39 | count = try(var.extra_values.addons.yandex-csi-controller.enabled, false) == true ? 1 : 0 40 | 41 | source = "../helm-yandex-csi-controller" 42 | 43 | depends_on = [ 44 | module.cilium 45 | ] 46 | 47 | namespace = "kube-fraima-yandex-csi-controller" 48 | 49 | chart_version = "0.0.8" 50 | global_vars = var.k8s_global_vars 51 | extra_values = try(var.extra_values.addons.yandex-csi-controller, {}) 52 | } 53 | 54 | module "coredns" { 55 | count = try(var.extra_values.addons.coredns.enabled, false) == true ? 1 : 0 56 | 57 | source = "../helm-coredns" 58 | depends_on = [ 59 | module.cilium, 60 | ] 61 | chart_version = "1.19.4" 62 | 63 | global_vars = var.k8s_global_vars 64 | extra_values = try(var.extra_values.addons.coredns, {}) 65 | } 66 | 67 | module "gatekeeper" { 68 | count = try(var.extra_values.addons.gatekeeper.enabled, false) == true ? 1 : 0 69 | 70 | source = "../helm-gatekeeper" 71 | depends_on = [ 72 | module.coredns, 73 | ] 74 | 75 | global_vars = var.k8s_global_vars 76 | extra_values = try(var.extra_values.addons.gatekeeper, {}) 77 | } 78 | 79 | module "certmanager" { 80 | count = try(var.extra_values.addons.certmanager.enabled, false) == true ? 1 : 0 81 | 82 | source = "../helm-certmanager" 83 | depends_on = [ 84 | module.gatekeeper, 85 | ] 86 | 87 | global_vars = var.k8s_global_vars 88 | extra_values = try(var.extra_values.addons.certmanager, {}) 89 | } 90 | 91 | module "vault-issuer" { 92 | count = try(var.extra_values.addons.vault-issuer.enabled, false) == true ? 1 : 0 93 | 94 | source = "../helm-vault-issuer" 95 | depends_on = [ 96 | module.certmanager, 97 | module.gatekeeper, 98 | ] 99 | 100 | global_vars = var.k8s_global_vars 101 | 102 | } 103 | 104 | module "machine-controller-manager" { 105 | count = try(var.extra_values.addons.machine-controller-manager.enabled, false) == true ? 1 : 0 106 | 107 | source = "../helm-machine-controller-manager" 108 | depends_on = [ 109 | module.gatekeeper, 110 | ] 111 | 112 | global_vars = var.k8s_global_vars 113 | extra_values = try(var.extra_values.addons.machine-controller-manager, {}) 114 | } 115 | 116 | module "compute-instance" { 117 | source = "../helm-yandex-machine-instance" 118 | 119 | count = try(var.extra_values.addons.compute-instance.enabled, false) == true ? 1 : 0 120 | 121 | depends_on = [ 122 | module.certmanager, 123 | module.machine-controller-manager 124 | ] 125 | 126 | global_vars = var.k8s_global_vars 127 | custom_values = try(var.extra_values.addons.compute-instance.custom_values, {}) 128 | extra_values = try(var.extra_values.addons.compute-instance, {}) 129 | } 130 | 131 | module "victoria-metrics-stack-operator" { 132 | source = "../helm-victoria-metrics-stack-operator" 133 | 134 | count = try(var.extra_values.addons.victoria-metrics-stack-operator.enabled, false) == true ? 1 : 0 135 | 136 | depends_on = [ 137 | module.coredns, 138 | ] 139 | 140 | global_vars = var.k8s_global_vars 141 | extra_values = try(var.extra_values.addons.victoria-metrics-stack-operator, {}) 142 | } 143 | -------------------------------------------------------------------------------- /modules/k8s-yandex-master-infra/locals.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | base_cluster_fqdn = var.k8s_global_vars.k8s-addresses.base_cluster_fqdn 4 | base_cluster_dns_zone = "${local.base_cluster_fqdn}." 5 | 6 | extra_cluster_name = var.k8s_global_vars.k8s-addresses.extra_cluster_name 7 | cluster_name = var.k8s_global_vars.cluster_metadata.cluster_name 8 | 9 | etcd_peer_port = var.k8s_global_vars.kubernetes-ports.etcd-peer-port 10 | etcd_server_port = var.k8s_global_vars.kubernetes-ports.etcd-server-port 11 | etcd_srv_server_record = "_etcd-server-ssl._tcp.${local.base_cluster_fqdn}." 12 | etcd_srv_client_record = "_etcd-client-ssl._tcp.${local.base_cluster_fqdn}." 13 | 14 | master_secondary_disk = var.k8s_global_vars.master_vars.master_group.resources.disk.secondary_disk 15 | master_subnet_prefix_name = "${local.cluster_name}-${var.k8s_global_vars.master_vars.master_group.name}" 16 | 17 | kube_apiserver_port_lb = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port-lb 18 | kube_apiserver_port = var.k8s_global_vars.kubernetes-ports.kube-apiserver-port 19 | kube_apiserver_listener_name = "api-listener-${local.cluster_name}" 20 | kube_apiserver_lb_name = "api-loadbalancer${local.cluster_name}" 21 | kube_apiserver_lb_ip = (tolist(yandex_lb_network_load_balancer.api-external.listener)[0].external_address_spec)[*].address 22 | kube_apiserver_lb_fqdn = "${var.k8s_global_vars.k8s-addresses.kube_apiserver_lb_fqdn}." 23 | 24 | yandex_lb_target_group_master = "${local.cluster_name}${data.yandex_vpc_network.cluster-vpc.id}" 25 | 26 | secret_id_all = var.k8s_vault_master.secret_id_all 27 | role_id_all = var.k8s_vault_master.role_id_all 28 | 29 | user_data = module.k8s-cloud-init-master.cloud-init-render 30 | } 31 | 32 | 33 | locals { 34 | master_vars = var.k8s_global_vars.master_vars 35 | master_group = local.master_vars.master_group 36 | 37 | master_regexp = "${var.k8s_global_vars.master_vars.master_group.name}-\\d*" 38 | 39 | instances_disk = flatten([ 40 | for disk_index, disk_name in keys(var.k8s_global_vars.master_vars.master_group.resources.disk.secondary_disk) : [ 41 | for instance_name in var.k8s_global_vars.master_vars.master_instance_list: 42 | {"${disk_name}_${instance_name}" = {}} 43 | ] 44 | ] 45 | ) 46 | instances_disk_map = { for item in local.instances_disk : 47 | keys(item)[0] => values(item)[0] 48 | } 49 | 50 | etcd_member_servers_srv = flatten([ 51 | for master_index, master_value in var.k8s_global_vars.master_vars.master_instance_extra_list: [ 52 | "0 0 ${local.etcd_peer_port} ${master_value}.${local.base_cluster_fqdn}." 53 | ] 54 | ]) 55 | etcd_member_clients_srv = flatten([ 56 | for master_index, master_value in var.k8s_global_vars.master_vars.master_instance_extra_list: [ 57 | "0 0 ${local.etcd_peer_port} ${master_value}.${local.base_cluster_fqdn}." 58 | ] 59 | ]) 60 | 61 | 62 | flatten_role_id_all = flatten([ 63 | for index, value in local.role_id_all: [ 64 | index 65 | ] 66 | ]) 67 | 68 | set_role_id_all = toset(local.flatten_role_id_all) 69 | map_role_id_all = { for item in local.set_role_id_all : 70 | item => {} 71 | } 72 | 73 | flatten_secret_id_alls = flatten([ 74 | for index, value in local.secret_id_all: [ 75 | "${index}" 76 | ] 77 | ]) 78 | 79 | set_secret_id_alls = toset(local.flatten_secret_id_alls) 80 | map_secret_id_alls = { for item in local.set_secret_id_alls : 81 | item => {} 82 | } 83 | 84 | subnets = flatten([ 85 | for instance_name in keys(local.master_vars.master_instance_list_map) : 86 | "${try( 87 | var.k8s_global_vars.master_vars.master_group.resources_override[instance_name].network_interface.subnet, 88 | var.k8s_global_vars.master_vars.master_group.default_subnet 89 | )}:${try( 90 | var.k8s_global_vars.master_vars.master_group.resources_override[instance_name].network_interface.zone, 91 | var.k8s_global_vars.master_vars.master_group.default_zone 92 | )}" 93 | ] 94 | ) 95 | subnets_set = toset(local.subnets) 96 | 97 | subnets_set_map = { for item in local.subnets_set : 98 | item => {} 99 | } 100 | 101 | cluster_external_instances_map = { for key, value in yandex_compute_instance.master : 102 | key => value.network_interface.0.nat_ip_address 103 | } 104 | cluster_internal_instances_map = { for key, value in yandex_compute_instance.master : 105 | key => value.network_interface[0].ip_address 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /modules/k8s-config-vars/local.component-versions.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | base_versions = { 3 | kubernetes_version = "v1.23.12" 4 | # TODO 1.27.* запланировать переезд на новый регистри 5 | image_repository = "k8s.gcr.io" 6 | } 7 | component_versions = { 8 | kubernetes_version = yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version 9 | image_repository = yamldecode(data.utils_deep_merge_yaml.base_versions.output).image_repository 10 | bin = { 11 | swarm = { 12 | bin_url = "https://github.com/fraima/swarm/releases/download/1.1.3/to-nft-1.1.3-linux-amd64.tar.gz" 13 | sha256_url = "https://github.com/fraima/swarm/releases/download/1.1.3/to-nft-1.1.3-linux-amd64.tar.gz.sha256" 14 | } 15 | fraimactl = { 16 | bin_url = "https://github.com/fraima/fraima/releases/download/v0.0.1/fraimactl-v0.0.1-linux-amd64.tar.gz" 17 | sha256_url = "https://github.com/fraima/fraima/releases/download/v0.0.1/fraimactl-v0.0.1-linux-amd64.tar.gz.sha256" 18 | } 19 | containerd = { 20 | bin_url = "https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz" 21 | sha256_url = "https://github.com/containerd/containerd/releases/download/v1.6.6/containerd-1.6.6-linux-amd64.tar.gz.sha256sum" 22 | } 23 | runc = { 24 | bin_url = "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64" 25 | sha256_url = "" 26 | } 27 | kubelet = { 28 | bin_url = "https://dl.k8s.io/release/${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}/bin/linux/amd64/kubelet" 29 | sha256_url = "https://dl.k8s.io/${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}/bin/linux/amd64/kubelet.sha256" 30 | } 31 | kubeadm = { 32 | bin_url = "https://dl.k8s.io/release/${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}/bin/linux/amd64/kubeadm" 33 | sha256_url = "https://dl.k8s.io/${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}/bin/linux/amd64/kubeadm.sha256" 34 | } 35 | kubectl = { 36 | bin_url = "https://dl.k8s.io/release/${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}/bin/linux/amd64/kubectl" 37 | sha256_url = "https://dl.k8s.io/${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}/bin/linux/amd64/kubectl.sha256" 38 | } 39 | crictl = { 40 | bin_url = "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz" 41 | sha256_url = "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.26.0/crictl-v1.26.0-linux-amd64.tar.gz.sha256" 42 | } 43 | etcdctl = { 44 | bin_url = "https://github.com/etcd-io/etcd/releases/download/v3.5.5/etcd-v3.5.5-linux-amd64.tar.gz" 45 | sha256_url = "" 46 | } 47 | key_keeper = { 48 | bin_url = "https://github.com/fraima/key-keeper/releases/download/v0.0.1/key-keeper-v0.0.1-linux-amd64.tar.gz" 49 | sha256_url = "https://github.com/fraima/key-keeper/releases/download/v0.0.1/key-keeper-v0.0.1-linux-amd64.tar.gz.sha256" 50 | } 51 | yc = { 52 | bin_url = "https://storage.yandexcloud.net/yandexcloud-yc/release/0.102.0/linux/amd64/yc" 53 | sha256_url = "" 54 | } 55 | } 56 | static_pod = { 57 | etcd = { 58 | registry = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).image_repository}/etcd" 59 | version = "3.5.3-0" 60 | } 61 | kube-apiserver = { 62 | registry = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).image_repository}/kube-apiserver" 63 | version = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}" 64 | } 65 | kube-controller-manager = { 66 | registry = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).image_repository}/kube-controller-manager" 67 | version = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}" 68 | } 69 | kube-scheduler = { 70 | registry = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).image_repository}/kube-scheduler" 71 | version = "${yamldecode(data.utils_deep_merge_yaml.base_versions.output).kubernetes_version}" 72 | } 73 | } 74 | } 75 | } 76 | 77 | data "utils_deep_merge_yaml" "component_versions" { 78 | input = [ 79 | yamlencode(local.component_versions), 80 | yamlencode(try(var.extra_args.component_versions, {})) 81 | ] 82 | } 83 | 84 | data "utils_deep_merge_yaml" "base_versions" { 85 | input = [ 86 | yamlencode(local.base_versions), 87 | yamlencode(try(var.extra_args.base_versions, {})) 88 | ] 89 | } 90 | -------------------------------------------------------------------------------- /modules/charlotte/locals.tf: -------------------------------------------------------------------------------- 1 | 2 | locals { 3 | 4 | #### Формируем список данных, где будет фигурировать: 5 | #### Имена SG, новые имена Networks и CIDR от Networks 6 | ##-> 7 | # [ 8 | # { 9 | # "teamA_backend" = [ 10 | # "27ccd286ef:10.143.0.3/32", 11 | # ] 12 | # }, 13 | # { 14 | # "teamA_frontend" = [ 15 | # "4894792f26:10.143.0.16/32", 16 | # ] 17 | # }, 18 | # { 19 | # "hbf-server" = [ 20 | # "53503b3b29:193.32.219.99/32", 21 | # ] 22 | # }, 23 | # { 24 | # "world" = [ 25 | # "06ee3732a5:176.0.0.0/8", 26 | # ] 27 | # }, 28 | # ] 29 | security_groups_network__name_cidr__flatten = flatten([ 30 | for security_group in var.security_groups : { 31 | "${security_group.name}": flatten([ 32 | for cidr in try(security_group.cidrs, []): 33 | "${cidr}:${cidr}" 34 | ]) 35 | } 36 | ]) 37 | 38 | security_group_map = { for item in local.security_groups_network__name_cidr__flatten : 39 | keys(item)[0] => values(item)[0] 40 | if item != {} 41 | } 42 | 43 | #### Формируем массив в котором подсети получают уникальные имена и находятся в одномерном массиве 44 | ##-> 45 | # [ 46 | # "27ccd286ef: 10.143.0.3/32", 47 | # "4894792f26: 10.143.0.16/32", 48 | # "53503b3b29: 193.32.219.99/32", 49 | # "06ee3732a5: 176.0.0.0/8", 50 | # ] 51 | networks_flatten = flatten([ 52 | for security_group in local.security_groups_network__name_cidr__flatten: [ 53 | for key, value in security_group: [ 54 | value 55 | ] 56 | ] 57 | ]) 58 | 59 | # Конвертация flatten в map 60 | networks_map = { for network in local.networks_flatten : 61 | keys ({split(":",network)[0]: split(":",network)[1]})[0] => 62 | values ({split(":",network)[0]: split(":",network)[1]})[0] 63 | } 64 | 65 | # Формируем массив данных, где будет фигурировать sgName, networkName(list) в виде строки 66 | # [ 67 | # { 68 | # "teamA_backend" = "27ccd286ef" 69 | # }, 70 | # { 71 | # "teamA_frontend" = "4894792f26" 72 | # }, 73 | # { 74 | # "hbf-server" = "53503b3b29" 75 | # }, 76 | # { 77 | # "world" = "06ee3732a5" 78 | # }, 79 | # ] 80 | security_groups_network__name__flatten = flatten([ 81 | for security_group in var.security_groups : { 82 | "${security_group.name}": join(",",flatten([ 83 | for cidr in try(security_group.cidrs, []): 84 | "${cidr}" 85 | ])) 86 | } 87 | ]) 88 | 89 | # Конвертация flatten в map 90 | security_groups_network__name__map = { for item in local.security_groups_network__name__flatten : 91 | keys(item)[0] => values(item)[0] 92 | # Удаляет SG если в ней нету Networks 93 | # Нужна, что бы можно было сначала создать SG и Networks потом добавить правила иначе будет перезапись в 0 94 | if values(item)[0] != "" 95 | } 96 | 97 | #### Формирует список массивов в котором, указана исходная SG и набор правил, 98 | #### которые открываются по прринципу ОТ -> ДО *sg_from подставляется автоматически 99 | ##-> 100 | # [ 101 | # { 102 | # "teamA_backend" = [ 103 | # { 104 | # "access" = { 105 | # tcp = [ 106 | # { 107 | # "description" = "access from teamA_backend to teamA_frontend" 108 | # "ports_to" = tolist([ 109 | # "80", 110 | # "443", 111 | # ]) 112 | # }, 113 | # ] 114 | # "sg_from" = "teamA_backend" 115 | # "sg_to" = "teamA_frontend" 116 | # } 117 | # ] 118 | # }, 119 | # ] 120 | security_group_rules_flatten = flatten([ 121 | for security_group in var.security_groups : { 122 | "${security_group.name}": flatten([ 123 | for rule in try(security_group.rules, []): 124 | merge(rule, {"sg_from": security_group.name}) 125 | ]) 126 | } 127 | ]) 128 | 129 | #### Формируется список массивов с полным набором оперируемых правил 130 | ##-> 131 | # [ 132 | # { 133 | # "access" = { 134 | # tcp = [ 135 | # { 136 | # "description" = "access from teamA_backend to teamA_frontend" 137 | # "ports_to" = tolist([ 138 | # "80", 139 | # "443", 140 | # ]) 141 | # }, 142 | # ] 143 | # } 144 | 145 | # "sg_from" = "teamA_backend" 146 | # "sg_to" = "teamA_frontend" 147 | # }, 148 | # ] 149 | rules_flatten = flatten([ 150 | for security_group in local.security_group_rules_flatten: [ 151 | for key, value in security_group: [ 152 | value 153 | ] 154 | ] 155 | ]) 156 | 157 | # Конвертация flatten в map с уникальным именем по входной паре FROM_SG:TO_SG 158 | rules_map = { for item in local.rules_flatten : 159 | "${item.sg_from}:${item.sg_to}" => item 160 | } 161 | 162 | } 163 | -------------------------------------------------------------------------------- /modules/keycloak-vault/vault.keycloak.role.tf: -------------------------------------------------------------------------------- 1 | resource "vault_pki_secret_backend_role" "keycloak-server" { 2 | for_each = local.certificates 3 | 4 | backend = var.root_ca_path 5 | name = each.key 6 | 7 | allow_any_name = try(local.certificates[each.key].allow_any_name, null) == null ? false : local.certificates[each.key].allow_any_name 8 | allow_bare_domains = try(local.certificates[each.key].allow_bare_domains, null) == null ? false : local.certificates[each.key].allow_bare_domains 9 | allow_glob_domains = try(local.certificates[each.key].allow_glob_domains, null) == null ? false : local.certificates[each.key].allow_glob_domains 10 | allow_subdomains = try(local.certificates[each.key].allow_subdomains, null) == null ? false : local.certificates[each.key].allow_subdomains 11 | allowed_domains_template = try(local.certificates[each.key].allowed_domains_template, null) == null ? false : local.certificates[each.key].allowed_domains_template 12 | basic_constraints_valid_for_non_ca = try(local.certificates[each.key].basic_constraints_valid_for_non_ca, null) == null ? false : local.certificates[each.key].basic_constraints_valid_for_non_ca 13 | code_signing_flag = try(local.certificates[each.key].code_signing_flag, null) == null ? false : local.certificates[each.key].code_signing_flag 14 | email_protection_flag = try(local.certificates[each.key].email_protection_flag, null) == null ? false : local.certificates[each.key].email_protection_flag 15 | enforce_hostnames = try(local.certificates[each.key].enforce_hostnames, null) == null ? false : local.certificates[each.key].enforce_hostnames 16 | generate_lease = try(local.certificates[each.key].generate_lease, null) == null ? false : local.certificates[each.key].generate_lease 17 | allow_ip_sans = try(local.certificates[each.key].allow_ip_sans, null) == null ? false : local.certificates[each.key].allow_ip_sans 18 | allow_localhost = try(local.certificates[each.key].allow_localhost, null) == null ? false : local.certificates[each.key].allow_localhost 19 | client_flag = try(local.certificates[each.key].client_flag, null) == null ? false : local.certificates[each.key].client_flag 20 | server_flag = try(local.certificates[each.key].server_flag, null) == null ? false : local.certificates[each.key].server_flag 21 | key_bits = try(local.certificates[each.key].key_bits, null) == null ? 4096 : local.certificates[each.key].key_bits 22 | key_type = try(local.certificates[each.key].key_type, null) == null ? "rsa" : local.certificates[each.key].key_type 23 | key_usage = try(local.certificates[each.key].key_usage, null) == null ? [] : local.certificates[each.key].key_usage 24 | organization = try(local.certificates[each.key].organization, null) == null ? [] : local.certificates[each.key].organization 25 | country = try(local.certificates[each.key].country, null) == null ? [] : local.certificates[each.key].country 26 | locality = try(local.certificates[each.key].locality, null) == null ? [] : local.certificates[each.key].locality 27 | ou = try(local.certificates[each.key].ou, null) == null ? [] : local.certificates[each.key].ou 28 | postal_code = try(local.certificates[each.key].postal_code, null) == null ? [] : local.certificates[each.key].postal_code 29 | province = try(local.certificates[each.key].province, null) == null ? [] : local.certificates[each.key].province 30 | street_address = try(local.certificates[each.key].street_address, null) == null ? [] : local.certificates[each.key].street_address 31 | allowed_domains = try(local.certificates[each.key].allowed_domains, null) == null ? [] : local.certificates[each.key].allowed_domains 32 | allowed_other_sans = try(local.certificates[each.key].allowed_other_sans, null) == null ? [] : local.certificates[each.key].allowed_other_sans 33 | allowed_serial_numbers = try(local.certificates[each.key].allowed_serial_numbers, null) == null ? [] : local.certificates[each.key].allowed_serial_numbers 34 | allowed_uri_sans = try(local.certificates[each.key].allowed_uri_sans, null) == null ? [] : local.certificates[each.key].allowed_uri_sans 35 | ext_key_usage = try(local.certificates[each.key].ext_key_usage, null) == null ? [] : local.certificates[each.key].ext_key_usage 36 | no_store = try(local.certificates[each.key].no_store, null) == null ? false : local.certificates[each.key].no_store 37 | require_cn = try(local.certificates[each.key].require_cn, null) == null ? false : local.certificates[each.key].require_cn 38 | ttl = try(local.certificates[each.key].ttl, null) == null ? 31540000 : local.certificates[each.key].ttl 39 | use_csr_common_name = try(local.certificates[each.key].use_csr_common_name, null) == null ? false : local.certificates[each.key].use_csr_common_name 40 | 41 | } 42 | -------------------------------------------------------------------------------- /modules/helm-victoria-metrics-stack-operator/helm/common.yaml: -------------------------------------------------------------------------------- 1 | victoria-metrics-operator: 2 | createCRD: false 3 | useLegacyCRD: false 4 | rbac: 5 | create: false 6 | pspEnabled: false 7 | 8 | operator: 9 | enabled: false 10 | createCRD: false 11 | cleanupCRD: false 12 | cleanupSA: 13 | create: false 14 | 15 | serviceAccount: 16 | create: false 17 | 18 | vmsingle: 19 | enabled: false 20 | 21 | vmcluster: 22 | enabled: false 23 | 24 | grafana: 25 | enabled: true 26 | defaultDashboardsEnabled: true 27 | additionalDataSources: 28 | - name: prometheus-sample 29 | access: proxy 30 | editable: false 31 | jsonData: 32 | tlsSkipVerify: true 33 | orgId: 1 34 | type: prometheus 35 | url: http://vmselect-vm-vmcluster-persistent:8481/select/0/prometheus/ 36 | version: 1 37 | 38 | tolerations: 39 | - key: node-role.kubernetes.io/master 40 | effect: NoSchedule 41 | - key: "CriticalAddonsOnly" 42 | operator: "Exists" 43 | - key: node.cloudprovider.kubernetes.io/uninitialized 44 | value: "true" 45 | effect: NoSchedule 46 | 47 | prometheus-node-exporter: 48 | enabled: true 49 | 50 | kube-state-metrics: 51 | enabled: true 52 | 53 | tolerations: 54 | - key: node-role.kubernetes.io/master 55 | effect: NoSchedule 56 | - key: "CriticalAddonsOnly" 57 | operator: "Exists" 58 | - key: node.cloudprovider.kubernetes.io/uninitialized 59 | value: "true" 60 | effect: NoSchedule 61 | 62 | resources: 63 | requests: 64 | # C-0050 - Resources CPU limit and request 65 | cpu: 100m 66 | # C-0004 - Resources memory limit and request 67 | memory: 256Mi 68 | limits: 69 | # C-0050 - Resources CPU limit and request 70 | cpu: 1000m 71 | # C-0004 - Resources memory limit and request 72 | memory: 1Gi 73 | 74 | containerSecurityContext: 75 | # C-0013 - Non-root containers 76 | runAsNonRoot: true 77 | # C-0017 - Immutable container filesystem 78 | readOnlyRootFilesystem: true 79 | # C-0197 - Minimize the admission of containers with allowPrivilegeEscalation 80 | allowPrivilegeEscalation: false 81 | 82 | kubelet: 83 | enabled: true 84 | 85 | kubeApiServer: 86 | enabled: true 87 | 88 | kubeControllerManager: 89 | enabled: true 90 | 91 | kubeDns: 92 | enabled: false 93 | 94 | coreDns: 95 | enabled: false 96 | 97 | kubeEtcd: 98 | enabled: true 99 | 100 | kubeScheduler: 101 | enabled: true 102 | 103 | kubeProxy: 104 | enabled: false 105 | 106 | defaultRules: 107 | create: true 108 | 109 | # labels: 110 | # pfm-admins: 'true' 111 | 112 | rules: 113 | etcd: true 114 | general: true 115 | k8s: true 116 | kubeApiserver: true 117 | kubeApiserverAvailability: true 118 | kubeApiserverSlos: true 119 | kubelet: true 120 | kubePrometheusGeneral: true 121 | kubePrometheusNodeRecording: true 122 | kubernetesApps: true 123 | kubernetesResources: true 124 | kubernetesStorage: true 125 | kubernetesSystem: true 126 | kubeScheduler: true 127 | kubeStateMetrics: true 128 | network: true 129 | node: true 130 | vmagent: true 131 | vmsingle: true 132 | vmhealth: true 133 | alertmanager: true 134 | 135 | # Custom conditions 136 | kubeControllerManager: true 137 | kubeApiserverBurnrate: true 138 | kubeApiserverHistogram: true 139 | 140 | 141 | alertmanager: 142 | enabled: false 143 | ingress: 144 | enabled: false 145 | 146 | annotations: {} 147 | 148 | monzoTemplate: 149 | enabled: false 150 | 151 | spec: 152 | selectAllByDefault: false 153 | replicaCount: 1 154 | 155 | priorityClassName: system-cluster-critical 156 | 157 | configSelector: 158 | matchLabels: 159 | pfm-admins: 'true' 160 | 161 | configNamespaceSelector: {} 162 | 163 | resources: 164 | requests: 165 | cpu: '4' 166 | memory: 2Gi 167 | limits: 168 | cpu: '4' 169 | memory: 2Gi 170 | 171 | vmalert: 172 | enabled: false 173 | ingress: 174 | enabled: false 175 | 176 | spec: 177 | 178 | imagePullSecrets: [] 179 | 180 | resources: 181 | requests: 182 | cpu: '2' 183 | memory: 2Gi 184 | limits: 185 | cpu: '2' 186 | memory: 2Gi 187 | 188 | replicaCount: 3 189 | updateStrategy: RollingUpdate 190 | rollingUpdate: 191 | maxSurge: 10% 192 | maxUnavailable: 0 193 | 194 | remoteWrite: 195 | concurrency: 4 196 | 197 | 198 | evaluationInterval: 15s 199 | 200 | selectAllByDefault: false 201 | ruleNamespaceSelector: {} 202 | ruleSelector: 203 | matchLabels: 204 | pfm-admins: 'true' 205 | 206 | 207 | vmagent: 208 | enabled: false 209 | ingress: 210 | enabled: false 211 | 212 | spec: 213 | 214 | imagePullSecrets: [] 215 | 216 | resources: 217 | requests: 218 | cpu: '4' 219 | memory: 2Gi 220 | limits: 221 | cpu: '4' 222 | memory: 2Gi 223 | 224 | extraArgs: 225 | promscrape.streamParse: 'false' 226 | promscrape.suppressDuplicateScrapeTargetErrors: 'true' 227 | 228 | replicaCount: 1 # we don't support deduplication for multiple vmagent replicas 229 | shardCount: 4 230 | 231 | updateStrategy: RollingUpdate 232 | rollingUpdate: 233 | maxSurge: 10% 234 | maxUnavailable: 0 235 | 236 | # because multiple deployments are used, one deployment per shard 237 | topologySpreadConstraints: 238 | - maxSkew: 1 239 | minDomains: 1 240 | topologyKey: region 241 | whenUnsatisfiable: ScheduleAnyway 242 | labelSelector: 243 | matchLabels: 244 | app.kubernetes.io/name: vmagent 245 | app.kubernetes.io/instance: vmagent 246 | pfm-admins: 'true' 247 | 248 | # ALERT: This usually means that `-remoteWrite.queues` 249 | # https://docs.victoriametrics.com/operator/api.html#vmagentremotewritesettings 250 | remoteWriteSettings: 251 | queues: 16 252 | 253 | statefulMode: false 254 | 255 | selectAllByDefault: false 256 | serviceScrapeSelector: 257 | matchLabels: 258 | pfm-admins: 'true' 259 | serviceScrapeNamespaceSelector: {} 260 | 261 | podScrapeSelector: 262 | matchLabels: 263 | pfm-admins: 'true' 264 | podScrapeNamespaceSelector: {} 265 | 266 | nodeScrapeSelector: 267 | matchLabels: 268 | pfm-admins: 'true' 269 | nodeScrapeNamespaceSelector: {} 270 | 271 | staticScrapeSelector: 272 | matchLabels: 273 | pfm-admins: 'true' 274 | staticScrapeNamespaceSelector: {} 275 | 276 | scrapeInterval: 15s 277 | minScrapeInterval: 15s 278 | maxScrapeInterval: 15s 279 | -------------------------------------------------------------------------------- /modules/helm-yandex-cilium/helm/values.yaml.tftpl: -------------------------------------------------------------------------------- 1 | # -- Configure the kube-proxy replacement in Cilium BPF datapath 2 | # Valid options are "disabled", "probe", "partial", "strict". 3 | # ref: https://docs.cilium.io/en/stable/gettingstarted/kubeproxy-free/ 4 | kubeProxyReplacement: strict 5 | 6 | k8sServiceHost: ${k8s_api_server_fqdn} 7 | k8sServicePort: ${k8s_api_server_port} 8 | 9 | cluster: 10 | # -- Name of the cluster. Only required for Cluster Mesh. 11 | name: example 12 | # -- (int) Unique ID of the cluster. Must be unique across all connected 13 | # clusters and in the range of 1 to 255. Only required for Cluster Mesh. 14 | id: 11 15 | 16 | ipam: 17 | # -- Configure IP Address Management mode. 18 | # ref: https://docs.cilium.io/en/stable/concepts/networking/ipam/ 19 | mode: kubernetes 20 | 21 | # Не используется, если установлен mode: kubernetes 22 | # operator: 23 | # # -- IPv4 CIDR mask size to delegate to individual nodes for IPAM. 24 | # clusterPoolIPv4MaskSize: {node_cidr_mask} 25 | # 26 | # # -- IPv4 CIDR list range to delegate to individual nodes for IPAM. 27 | # clusterPoolIPv4PodCIDRList: 28 | # - {pod_cidr} 29 | 30 | operator: 31 | # -- cilium-operator resource limits & requests 32 | # ref: https://kubernetes.io/docs/user-guide/compute-resources/ 33 | resources: 34 | limits: 35 | cpu: 1000m 36 | memory: 1Gi 37 | requests: 38 | cpu: 100m 39 | memory: 128Mi 40 | 41 | resources: 42 | limits: 43 | cpu: 4000m 44 | memory: 4Gi 45 | requests: 46 | cpu: 100m 47 | memory: 512Mi 48 | 49 | # -- Configure service load balancing 50 | loadBalancer: 51 | # -- algorithm is the name of the load balancing algorithm for backend 52 | # selection e.g. random or maglev 53 | algorithm: random 54 | 55 | # -- mode is the operation mode of load balancing for remote backends 56 | # e.g. snat, dsr, hybrid 57 | mode: snat 58 | # dsrDispatch: opt 59 | 60 | wellKnownIdentities: 61 | # -- Enable the use of well-known identities. 62 | enabled: true 63 | 64 | l2NeighDiscovery: 65 | # -- Enable L2 neighbor discovery in the agent 66 | enabled: true 67 | 68 | # -- Allows to explicitly specify the IPv4 CIDR for native routing. 69 | # When specified, Cilium assumes networking for this CIDR is preconfigured and 70 | # hands traffic destined for that range to the Linux network stack without 71 | # applying any SNAT. 72 | # Generally speaking, specifying a native routing CIDR implies that Cilium can 73 | # depend on the underlying networking stack to route packets to their 74 | # destination. To offer a concrete example, if Cilium is configured to use 75 | # direct routing and the Kubernetes CIDR is included in the native routing CIDR, 76 | # the user must configure the routes to reach pods, either manually or by 77 | # setting the auto-direct-node-routes flag. 78 | ipv4NativeRoutingCIDR: "10.0.0.0/8" 79 | 80 | # -- Configures the use of the KVStore to optimize Kubernetes event handling by 81 | # mirroring it into the KVstore for reduced overhead in large clusters. 82 | enableK8sEventHandover: true 83 | 84 | # -- Configure the encapsulation configuration for communication between nodes. 85 | # Possible values: 86 | # - disabled 87 | # - vxlan (default) 88 | # - geneve 89 | tunnel: "disabled" 90 | 91 | # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. 92 | enableIPv4Masquerade: false 93 | 94 | # -- Enables masquerading of IPv6 traffic leaving the node from endpoints. 95 | enableIPv6Masquerade: false 96 | 97 | # -- Enable hostPort service support. 98 | hostPort: 99 | enabled: false 100 | 101 | # -- Enable Layer 7 network policy. 102 | l7Proxy: false 103 | 104 | # -- Enable setting identity mark for local traffic. 105 | enableIdentityMark: true 106 | 107 | # -- Configure whether to install iptables rules to allow for TPROXY 108 | # (L7 proxy injection), iptables-based masquerading and compatibility 109 | # with kube-proxy. 110 | installIptablesRules: true 111 | 112 | bpf: 113 | # -- Configure the eBPF-based TPROXY to reduce reliance on iptables rules 114 | # for implementing Layer 7 policy. 115 | tproxy: true 116 | hostLegacyRouting: true 117 | 118 | # Требует включенное состояние при развертывании в облаках и интегрируя с cloud-controllers 119 | # -- Configure N-S k8s service loadbalancing 120 | nodePort: 121 | # -- Enable the Cilium NodePort service implementation. 122 | enabled: true 123 | 124 | # -- Set to true to prevent applications binding to service ports. 125 | bindProtection: true 126 | 127 | # -- Append NodePort range to ip_local_reserved_ports if clash with ephemeral 128 | # ports is detected. 129 | autoProtectPortRange: true 130 | 131 | # -- Enable healthcheck nodePort server for NodePort services 132 | enableHealthCheck: true 133 | 134 | clustermesh: 135 | # -- Deploy clustermesh-apiserver for clustermesh 136 | useAPIServer: false 137 | 138 | # -- cilium-monitor sidecar. 139 | monitor: 140 | # -- Enable the cilium-monitor sidecar. 141 | enabled: true 142 | 143 | hubble: 144 | # -- Enable Hubble (true by default). 145 | enabled: true 146 | tls: 147 | enabled: true 148 | auto: 149 | enabled: true 150 | method: "helm" 151 | 152 | # -- Hubble metrics configuration. 153 | # See https://docs.cilium.io/en/stable/operations/metrics/#hubble-metrics 154 | # for more comprehensive documentation about Hubble metrics. 155 | metrics: 156 | enabled: 157 | - dns 158 | - drop 159 | - tcp 160 | - flow 161 | - icmp 162 | - http 163 | 164 | relay: 165 | # -- Enable Hubble Relay (requires hubble.enabled=true) 166 | enabled: true 167 | tolerations: 168 | - key: node-role.kubernetes.io/master 169 | effect: NoSchedule 170 | - key: node-role.kubernetes.io/control-plane 171 | effect: NoSchedule 172 | ui: 173 | # -- Whether to enable the Hubble UI. 174 | enabled: true 175 | tolerations: 176 | - key: node-role.kubernetes.io/master 177 | effect: NoSchedule 178 | - key: node-role.kubernetes.io/control-plane 179 | effect: NoSchedule 180 | 181 | # -- Enable Local Redirect Policy. 182 | localRedirectPolicy: true 183 | 184 | # -- Configure external workloads support 185 | externalWorkloads: 186 | # -- Enable support for external workloads, such as VMs (false by default). 187 | enabled: false 188 | 189 | endpointRoutes: 190 | # -- Enable use of per endpoint routes instead of routing via 191 | # the cilium_host interface. (required "false" if bpf.hostLegacyRouting=false) 192 | enabled: true 193 | 194 | # -- Install Iptables rules to skip netfilter connection tracking on all pod 195 | # traffic. This option is only effective when Cilium is running in direct 196 | # routing and full KPR mode. Moreover, this option cannot be enabled when Cilium 197 | # is running in a managed Kubernetes environment or in a chained CNI setup. 198 | installNoConntrackIptablesRules: false 199 | -------------------------------------------------------------------------------- /modules/k8s-templates/static-pods/kubeadm-config/templates/kubeadm-config.yaml.tftpl: -------------------------------------------------------------------------------- 1 | apiVersion: kubeadm.k8s.io/v1beta3 2 | kind: InitConfiguration 3 | skipPhases: 4 | #- preflight 5 | - certs 6 | - kubeconfig 7 | - kubelet-start 8 | - etcd 9 | - upload-certs 10 | - kubelet-finalize 11 | - addon 12 | - bootstrap-token 13 | # TODO добавить условие для версии 14 | # 1.27.1 15 | #- show-join-command 16 | 17 | nodeRegistration: 18 | ignorePreflightErrors: 19 | - ExternalEtcdVersion 20 | - Port-10250 21 | - FileAvailable--etc-kubernetes-manifests-etcd.yaml 22 | - FileExisting-iptables 23 | # TODO добавить условие для версии 24 | # 1.27.1 25 | - ImagePull 26 | --- 27 | apiVersion: kubeadm.k8s.io/v1beta3 28 | kind: ClusterConfiguration 29 | networking: 30 | serviceSubnet: "${service_cidr}" 31 | dnsDomain: "cluster.local" 32 | kubernetesVersion: "${component_versions.kubernetes_version}" 33 | imageRepository: "${component_versions.image_repository}" 34 | controlPlaneEndpoint: "${kube_api_fqdn}:${kube_apiserver_port}" 35 | etcd: 36 | external: 37 | endpoints: 38 | %{~ for etcd in etcd_list_servers ~} 39 | - "${etcd}" 40 | %{~ endfor ~} 41 | "caFile": "/etc/kubernetes/pki/ca/etcd-ca.pem" 42 | "certFile": "/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-etcd-client.pem" 43 | "keyFile": "/etc/kubernetes/pki/certs/kube-apiserver/kube-apiserver-etcd-client-key.pem" 44 | apiServer: 45 | extraArgs: 46 | %{~ for intermediate_name, intermediate_content in ssl.intermediate ~} 47 | %{~ if try(intermediate_content.labels.static-pod-kube-apiserver-args, {}) != {} ~} 48 | %{~ for key, value in intermediate_content.labels.static-pod-kube-apiserver-args ~} 49 | %{~ if value == "cert-public-arg" ~} 50 | "${key}": "${intermediate_content.default.host_path}/${intermediate_name}.pem" 51 | %{~ endif ~} 52 | %{~ if value == "cert-private-arg" ~} 53 | "${key}": "${intermediate_content.default.host_path}/${intermediate_name}-key.pem" 54 | %{~ endif ~} 55 | %{~ endfor ~} 56 | %{~ endif ~} 57 | %{~ for issuer_name, issuer_content in intermediate_content.issuers ~} 58 | %{~ for certificate_name, certificate_content in issuer_content.certificates ~} 59 | %{~ if try(certificate_content.labels.static-pod-kube-apiserver-args, {}) != {} ~} 60 | %{~ for key, value in certificate_content.labels.static-pod-kube-apiserver-args ~} 61 | %{~ if value == "cert-public-arg" ~} 62 | "${key}": "${certificate_content.key-keeper-args.host_path}/${certificate_name}.pem" 63 | %{~ endif ~} 64 | %{~ if value == "cert-private-arg" ~} 65 | "${key}": "${certificate_content.key-keeper-args.host_path}/${certificate_name}-key.pem" 66 | %{~ endif ~} 67 | %{~ endfor ~} 68 | %{~ endif ~} 69 | %{~ endfor ~} 70 | %{~ endfor ~} 71 | %{~ endfor ~} 72 | "service-account-key-file": "${secrets["kube-apiserver-sa"].keys["public"].host_path}" 73 | "service-account-signing-key-file": "${secrets["kube-apiserver-sa"].keys["private"].host_path}" 74 | "audit-log-path": "/var/log/kubernetes/audit/audit.log" 75 | "audit-policy-file": "${main_path.base_kubernetes_path}/kube-apiserver/audit-policy.yaml" 76 | "oidc-ca-file": "${base_path.base_local_path_certs}/ca/oidc-ca.pem" 77 | 78 | ${indent(4,kube_flags.kube_apiserver_flags)} 79 | 80 | extraVolumes: 81 | 82 | - name: "k8s-ca" 83 | hostPath: "${base_path.base_local_path_certs}/ca" 84 | mountPath: "${base_path.base_local_path_certs}/ca" 85 | readOnly: true 86 | pathType: DirectoryOrCreate 87 | 88 | - name: "k8s-certs" 89 | hostPath: "${base_path.base_local_path_certs}/certs" 90 | mountPath: "${base_path.base_local_path_certs}/certs" 91 | readOnly: true 92 | pathType: DirectoryOrCreate 93 | 94 | - name: "k8s-kube-apiserver-configs" 95 | hostPath: "${main_path.base_kubernetes_path}/kube-apiserver" 96 | mountPath: "${main_path.base_kubernetes_path}/kube-apiserver" 97 | readOnly: true 98 | pathType: DirectoryOrCreate 99 | 100 | - name: "k8s-audit" 101 | hostPath: "/var/log/kubernetes/audit/" 102 | mountPath: "/var/log/kubernetes/audit/" 103 | readOnly: false 104 | pathType: DirectoryOrCreate 105 | 106 | controllerManager: 107 | extraArgs: 108 | %{~ for intermediate_name, intermediate_content in ssl.intermediate ~} 109 | %{~ if try(intermediate_content.labels.static-pod-kube-controller-manager-args, {}) != {} ~} 110 | %{~ for key, value in intermediate_content.labels.static-pod-kube-controller-manager-args ~} 111 | %{~ if value == "cert-public-arg" ~} 112 | "${key}": "${intermediate_content.default.host_path}/${intermediate_name}.pem" 113 | %{~ endif ~} 114 | %{~ if value == "cert-private-arg" ~} 115 | "${key}": "${intermediate_content.default.host_path}/${intermediate_name}-key.pem" 116 | %{~ endif ~} 117 | %{~ endfor ~} 118 | %{~ endif ~} 119 | %{~ for issuer_name, issuer_content in intermediate_content.issuers ~} 120 | %{~ for certificate_name, certificate_content in issuer_content.certificates ~} 121 | %{~ if try(certificate_content.labels.static-pod-kube-controller-manager-args, {}) != {} ~} 122 | %{~ for key, value in certificate_content.labels.static-pod-kube-controller-manager-args ~} 123 | %{~ if value == "cert-public-arg" ~} 124 | "${key}": "${certificate_content.key-keeper-args.host_path}/${certificate_name}.pem" 125 | %{~ endif ~} 126 | %{~ if value == "cert-private-arg" ~} 127 | "${key}": "${certificate_content.key-keeper-args.host_path}/${certificate_name}-key.pem" 128 | %{~ endif ~} 129 | %{~ endfor ~} 130 | %{~ endif ~} 131 | %{~ endfor ~} 132 | %{~ endfor ~} 133 | %{~ endfor ~} 134 | "service-account-private-key-file": "${secrets["kube-apiserver-sa"].keys["private"].host_path}" 135 | "cluster-signing-key-file": "" 136 | "kubeconfig": "${main_path.base_kubernetes_path}/kube-controller-manager/kubeconfig" 137 | "authentication-kubeconfig": "${main_path.base_kubernetes_path}/kube-controller-manager/kubeconfig" 138 | "authorization-kubeconfig": "${main_path.base_kubernetes_path}/kube-controller-manager/kubeconfig" 139 | 140 | ${indent(4,kube_flags.kube_controller_manager_flags)} 141 | 142 | extraVolumes: 143 | # TODO донт ворк 144 | #- name: kubeconfig 145 | 146 | - name: "k8s-ca" 147 | hostPath: "${base_path.base_local_path_certs}/ca" 148 | mountPath: "${base_path.base_local_path_certs}/ca" 149 | readOnly: true 150 | pathType: DirectoryOrCreate 151 | 152 | - name: "k8s-certs" 153 | hostPath: "${base_path.base_local_path_certs}/certs" 154 | mountPath: "${base_path.base_local_path_certs}/certs" 155 | readOnly: true 156 | pathType: DirectoryOrCreate 157 | 158 | - name: "k8s-kube-controller-manager-configs" 159 | hostPath: "${main_path.base_kubernetes_path}/kube-controller-manager" 160 | mountPath: "${main_path.base_kubernetes_path}/kube-controller-manager" 161 | readOnly: true 162 | pathType: DirectoryOrCreate 163 | 164 | scheduler: 165 | extraArgs: 166 | %{~ for intermediate_name, intermediate_content in ssl.intermediate ~} 167 | %{~ if try(intermediate_content.labels.static-pod-kube-scheduler-args, {}) != {} ~} 168 | %{~ for key, value in intermediate_content.labels.static-pod-kube-scheduler-args ~} 169 | %{~ if value == "cert-public-arg" ~} 170 | "${key}": "${intermediate_content.default.host_path}/${intermediate_name}.pem" 171 | %{~ endif ~} 172 | %{~ if value == "cert-private-arg" ~} 173 | "${key}": "${intermediate_content.default.host_path}/${intermediate_name}-key.pem" 174 | %{~ endif ~} 175 | %{~ endfor ~} 176 | %{~ endif ~} 177 | %{~ for issuer_name, issuer_content in intermediate_content.issuers ~} 178 | %{~ for certificate_name, certificate_content in issuer_content.certificates ~} 179 | %{~ if try(certificate_content.labels.static-pod-kube-scheduler-args, {}) != {} ~} 180 | %{~ for key, value in certificate_content.labels.static-pod-kube-scheduler-args ~} 181 | %{~ if value == "cert-public-arg" ~} 182 | "${key}": "${certificate_content.key-keeper-args.host_path}/${certificate_name}.pem" 183 | %{~ endif ~} 184 | %{~ if value == "cert-private-arg" ~} 185 | "${key}": "${certificate_content.key-keeper-args.host_path}/${certificate_name}-key.pem" 186 | %{~ endif ~} 187 | %{~ endfor ~} 188 | %{~ endif ~} 189 | %{~ endfor ~} 190 | %{~ endfor ~} 191 | %{~ endfor ~} 192 | "authentication-kubeconfig": "${main_path.base_kubernetes_path}/kube-scheduler/kubeconfig" 193 | "authorization-kubeconfig": "${main_path.base_kubernetes_path}/kube-scheduler/kubeconfig" 194 | "kubeconfig": "${main_path.base_kubernetes_path}/kube-scheduler/kubeconfig" 195 | 196 | ${indent(4,kube_flags.kube_scheduler_flags)} 197 | 198 | extraVolumes: 199 | # TODO донт ворк 200 | #- name: kubeconfig 201 | 202 | - name: "k8s-ca" 203 | hostPath: "${base_path.base_local_path_certs}/ca" 204 | mountPath: "${base_path.base_local_path_certs}/ca" 205 | readOnly: true 206 | pathType: DirectoryOrCreate 207 | 208 | - name: "k8s-certs" 209 | hostPath: "${base_path.base_local_path_certs}/certs" 210 | mountPath: "${base_path.base_local_path_certs}/certs" 211 | readOnly: true 212 | pathType: DirectoryOrCreate 213 | 214 | - name: "k8s-kube-scheduler-configs" 215 | hostPath: "${main_path.base_kubernetes_path}/kube-scheduler" 216 | mountPath: "${main_path.base_kubernetes_path}/kube-scheduler" 217 | readOnly: true 218 | pathType: DirectoryOrCreate 219 | --------------------------------------------------------------------------------