├── .gitignore ├── LICENSE ├── README.md ├── provision.md └── src ├── config-connector-resource ├── Chart.yaml ├── templates │ └── configconnector.yaml └── values.yaml ├── gatekeeper ├── README.md ├── constraints │ └── only-allowed-gcp-types.yaml ├── resources │ ├── bad-service-account.yaml │ ├── good-compute-network.yaml │ ├── good-pubsubtopic.yaml │ └── not-checked-pod.yaml └── templates │ └── kccallowedresourcetypes_template.yaml ├── gcr-image ├── README.md └── resources │ ├── gcp-gcr-bucket-policy.yaml │ ├── gcp-gcr-service-account-key.yaml │ ├── gcp-gcr-service-account.yaml │ ├── gcr-bucket.yaml │ └── k8s-pod.yaml ├── helm1 ├── README.md └── mychart │ ├── .helmignore │ ├── Chart.yaml │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── configmap.yaml │ ├── deployment.yaml │ ├── ingress.yaml │ ├── service.yaml │ ├── serviceaccount.yaml │ └── tests │ │ └── test-connection.yaml │ └── values.yaml ├── mci ├── README.md └── resources │ ├── autoneg │ └── autoneg.yaml │ ├── clusters │ └── gcp-clusters.yaml │ ├── eu │ ├── deployment.yaml │ └── service.yaml │ ├── lb │ ├── gcp-backend-service.yaml │ ├── gcp-compute-firewall.yaml │ ├── gcp-compute-forwarding-rule.yaml │ ├── gcp-compute-health-check.yaml │ ├── gcp-default-network.yaml │ ├── gcp-target-http-proxy.yaml │ └── gcp-url-map.yaml │ └── na │ ├── deployment.yaml │ └── service.yaml ├── multiteam ├── README.md ├── provision-projects.sh ├── team-a │ ├── resources-admin │ │ ├── k8s-namespace-permissions.yaml │ │ └── k8s-namespace.yaml │ └── resources-team │ │ ├── gcp-bucket-policy.yaml │ │ ├── gcp-bucket.yaml │ │ ├── gcp-service-account.yaml │ │ ├── gcp-wi-policy.yaml │ │ ├── k8s-pod.yaml │ │ └── k8s-service-account.yaml └── team-b │ ├── resources-admin │ ├── k8s-namespace-permissions.yaml │ └── k8s-namespace.yaml │ └── resources-team │ ├── gcp-bucket-policy.yaml │ ├── gcp-bucket.yaml │ ├── gcp-service-account.yaml │ ├── gcp-wi-policy.yaml │ ├── k8s-pod.yaml │ └── k8s-service-account.yaml ├── provision.sh ├── pubsub ├── keyless-iam.yaml └── wi-iam.yaml ├── tf-provision ├── .terraform.lock.hcl └── provision.tf ├── wp-acm ├── README.md ├── acm-root │ ├── cluster │ │ └── ns-must-have-cost-center.yaml │ ├── namespaces │ │ └── online │ │ │ └── wp │ │ │ ├── gcp-sql-db.yaml │ │ │ ├── gcp-sql-service-account.yaml │ │ │ ├── k8s-external-load-balancer.yaml │ │ │ ├── k8s-service.yaml │ │ │ ├── k8s-stateful-set.yaml │ │ │ ├── wp-dev │ │ │ ├── gcp-sql-instance.yaml │ │ │ ├── gcp-sql-policy.yaml │ │ │ ├── gcp-sql-user.yaml │ │ │ ├── gcp-wi-policy.yaml │ │ │ ├── k8s-service-account.yaml │ │ │ ├── k8s-sql-db-credentials.yaml │ │ │ └── namespace.yaml │ │ │ └── wp-prod │ │ │ ├── gcp-sql-instance.yaml │ │ │ ├── gcp-sql-policy.yaml │ │ │ ├── gcp-sql-user.yaml │ │ │ ├── gcp-wi-policy.yaml │ │ │ ├── k8s-service-account.yaml │ │ │ ├── k8s-sql-db-credentials.yaml │ │ │ └── namespace.yaml │ └── system │ │ └── repo.yaml └── config-management.yaml ├── wp-simple ├── README.md └── resources │ ├── external-load-balancer.yaml │ ├── service.yaml │ ├── sql-db-credentials.yaml │ ├── sql-db.yaml │ ├── sql-instance.yaml │ ├── sql-user.yaml │ └── stateful-set.yaml └── wp-wi ├── README.md ├── constraints └── containers_must_be_limited.yaml └── resources ├── gcp-sql-db.yaml ├── gcp-sql-instance.yaml ├── gcp-sql-policy-member.yaml ├── gcp-sql-service-account.yaml ├── gcp-sql-user.yaml ├── gcp-wi-policy.yaml ├── k8s-external-load-balancer.yaml ├── k8s-service-account.yaml ├── k8s-service.yaml ├── k8s-sql-db-credentials.yaml └── k8s-stateful-set.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Repo specific files 15 | key*json 16 | *.DS_Store 17 | custom-settings.md 18 | install-bundle.tar.gz 19 | 20 | 21 | # Local .terraform directories 22 | **/.terraform/* 23 | 24 | # .tfstate files 25 | *.tfstate 26 | *.tfstate.* 27 | 28 | # Crash log files 29 | crash.log 30 | 31 | # Exclude all .tfvars files, which are likely to contain sentitive data, such as 32 | # password, private keys, and other secrets. These should not be part of version 33 | # control as they are data points which are potentially sensitive and subject 34 | # to change depending on the environment. 35 | # 36 | *.tfvars 37 | 38 | # Ignore override files as they are usually used to override resources locally and so 39 | # are not checked in 40 | override.tf 41 | override.tf.json 42 | *_override.tf 43 | *_override.tf.json 44 | 45 | # Include override files you do wish to add to version control using negated pattern 46 | # 47 | # !example_override.tf 48 | 49 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 50 | # example: *tfplan* 51 | 52 | # Ignore CLI configuration files 53 | .terraformrc 54 | terraform.rc 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Alex Bulankou 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Config Connector samples 2 | 3 | This repo contains a collection of infrastructure examples using [Kubernetes Config Connector](https://github.com/GoogleCloudPlatform/k8s-config-connector). Follow Config Connector Setup steps and then try any of the following examples: 4 | 5 | * [WordPress on K8s + GCP CloudSQL + Workload Identity Setup](src/wp-wi/README.md) 6 | * [WordPress on K8s + GCP Cloud SQL + WI + Gatekeeper + ACM](src/wp-acm/README.md) 7 | * [Multi-cluster ingress](src/mci/README.md) 8 | * [Gatekeeper integration](src/gatekeeper/README.md) 9 | * [Multiple Team Namespace-Project Configuration](src/multiteam/README.md) 10 | * [GCR image pull permissions from exernal K8s cluster with Config Connector](src/gcr-image/README.md) 11 | 12 | 13 | ## Config Connector Setup 14 | 15 | 1. Authenticate to GCP 16 | 17 | ```bash 18 | gcloud auth application-default login 19 | ``` 20 | 21 | 1. Create project and cluster with Config Connector enabled: 22 | 23 | ```bash 24 | cd ./tf-provision 25 | terraform apply -var="project=PROJECT_ID" \ 26 | -var="folder_id=FOLDER_ID" \ 27 | -var="billing_account=BILLING_ACCOUNT" 28 | cd .. 29 | ``` 30 | 31 | Note `project_id` output variable and use it in the next steps: 32 | 33 | ```bash 34 | PROJECT_ID=[project_id] 35 | 36 | 1. Set the context 37 | 38 | ```bash 39 | gcloud config set project $PROJECT_ID 40 | gcloud container clusters get-credentials cluster-1 --zone=us-central1-b 41 | ``` 42 | 43 | 1. Install Config Connector resource and annotate the namespace that you will use for Config Connector resources: 44 | 45 | ```bash 46 | # we need to ensure that only instance of config-connector resource exists per cluster 47 | kubectl delete configconnector.core.cnrm.cloud.google.com --all 48 | 49 | # customize and install with helm: 50 | helm install ./config-connector-resource/. --set projectID=$PROJECT_ID --generate-name 51 | kubectl annotate namespace default cnrm.cloud.google.com/project-id=$PROJECT_ID 52 | ``` 53 | 54 | 1. Verify that Config Connector is functional: 55 | 56 | ```bash 57 | kubectl wait -n cnrm-system --for=condition=Ready pod --all 58 | ``` 59 | -------------------------------------------------------------------------------- /provision.md: -------------------------------------------------------------------------------- 1 | 1. Replace with your project name and billing account: 2 | 3 | ```bash 4 | LC_CTYPE=C && find ./src/ -type f -exec sed -i '' 's/\[PROJECT_ID\]/your_project_id/g' {} \; 5 | LC_CTYPE=C && find ./src/ -type f -exec sed -i '' 's/\[BILLING_ACCOUNT\]/your_billing_account/g' {} \; 6 | ``` 7 | 1. Initialize project and cluster: 8 | 9 | ```bash 10 | bash src/provision.sh 11 | ``` -------------------------------------------------------------------------------- /src/config-connector-resource/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | name: config-connector-installation 3 | version: 0.1.0 4 | description: configures custom resource for GCP Config Connector -------------------------------------------------------------------------------- /src/config-connector-resource/templates/configconnector.yaml: -------------------------------------------------------------------------------- 1 | # configconnector.yaml 2 | apiVersion: core.cnrm.cloud.google.com/v1beta1 3 | kind: ConfigConnector 4 | metadata: 5 | # the name is restricted to ensure that there is only one 6 | # ConfigConnector instance installed in your cluster 7 | name: configconnector.core.cnrm.cloud.google.com 8 | spec: 9 | mode: cluster 10 | googleServiceAccount: "cnrmsa@{{ required "Project ID is required!" .Values.projectID }}.iam.gserviceaccount.com" -------------------------------------------------------------------------------- /src/config-connector-resource/values.yaml: -------------------------------------------------------------------------------- 1 | projectID: -------------------------------------------------------------------------------- /src/gatekeeper/README.md: -------------------------------------------------------------------------------- 1 | # Config Connector and Gatekeeper Integration 2 | 3 | This sample demonstrates how you create policies to verify Config Connector objects using Gatekeeper. 4 | 5 | 1. [Provision project, cluster and Config Connector](../../provision.md) 6 | 1. Intall Gatekeeper library: 7 | ```bash 8 | kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper/master/deploy/gatekeeper.yaml 9 | ``` 10 | 11 | 1. Install constraint template to `KCCAllowedResourceTypes`. It allows to specify what GCP types are allowed: 12 | ```bash 13 | kubectl apply -f templates/kccallowedresourcetypes_template.yaml 14 | ``` 15 | 16 | 1. Install constraint that only allows `PubSubTopic` and `ComputeNetwork`: 17 | ```bash 18 | kubectl apply -f constraints/only-allowed-gcp-types.yaml 19 | ``` 20 | 21 | 1. Try creating different objects to verify: 22 | 23 | - not allowed: resources/bad-service-account.yaml 24 | - allowed: resources/good-compute-network.yaml 25 | - allowed: resources/good-pubsubtopic.yaml 26 | - not checked: resources/not-checked-pod.yaml 27 | -------------------------------------------------------------------------------- /src/gatekeeper/constraints/only-allowed-gcp-types.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: constraints.gatekeeper.sh/v1beta1 2 | kind: KCCAllowedResourceTypes 3 | metadata: 4 | name: only-allowed-gcp-types 5 | spec: 6 | #match: 7 | #kinds: 8 | # - apiGroups: [""] 9 | # kinds: ["Pod"] 10 | #namespaces: 11 | # - "production" 12 | parameters: 13 | types: 14 | - "PubSubTopic" 15 | - "ComputeNetwork" 16 | -------------------------------------------------------------------------------- /src/gatekeeper/resources/bad-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: iamserviceaccount-example 7 | spec: 8 | displayName: Example Service Account 9 | -------------------------------------------------------------------------------- /src/gatekeeper/resources/good-compute-network.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeNetwork 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: computenetwork-sample 7 | spec: 8 | routingMode: REGIONAL 9 | autoCreateSubnetworks: true 10 | -------------------------------------------------------------------------------- /src/gatekeeper/resources/good-pubsubtopic.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 2 | kind: PubSubTopic 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: pubsubtopic-sample 7 | -------------------------------------------------------------------------------- /src/gatekeeper/resources/not-checked-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: myapp-pod 5 | labels: 6 | app: myapp 7 | spec: 8 | containers: 9 | - name: myapp-container 10 | image: busybox 11 | command: ['sh', '-c', 'echo Hello Kubernetes! && sleep 3600'] 12 | -------------------------------------------------------------------------------- /src/gatekeeper/templates/kccallowedresourcetypes_template.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: templates.gatekeeper.sh/v1beta1 2 | kind: ConstraintTemplate 3 | metadata: 4 | name: kccallowedresourcetypes 5 | spec: 6 | crd: 7 | spec: 8 | names: 9 | kind: KCCAllowedResourceTypes 10 | listKind: KCCAllowedResourceTypesList 11 | plural: KCCAllowedResourceTypes 12 | singular: KCCAllowedResourceTypes 13 | validation: 14 | # Schema for the `parameters` field 15 | openAPIV3Schema: 16 | properties: 17 | types: 18 | type: array 19 | items: 20 | type: string 21 | targets: 22 | - target: admission.k8s.gatekeeper.sh 23 | rego: | 24 | package kccallowedresourcetypes 25 | 26 | violation[{"msg": msg}] { 27 | type := input.review.object.kind 28 | apiVersion := input.review.object.apiVersion 29 | isKccType := contains(apiVersion, "cnrm.cloud.google.com") 30 | typeSatisfied := [good | allowedType = input.parameters.types[_] ; good = type == allowedType] 31 | isKccType; not any(typeSatisfied) 32 | msg := sprintf("using type <%v> is not allowed, allowed Config Connector types are %v", [type, input.parameters.types]) 33 | } 34 | -------------------------------------------------------------------------------- /src/gcr-image/README.md: -------------------------------------------------------------------------------- 1 | # GCR image pull permissions from exernal K8s cluster with Config Connector 2 | 3 | This sample demonstrates how you can provision a secret on external K8s clusters for GCR image using Config Connector 4 | 5 | 1. Update the files in this example to use your project name 6 | ``` 7 | LC_CTYPE=C && find ./resources/ -type f -exec sed -i '' 's/\[PROJECT_ID\]/your_project_id/g' {} \; 8 | ``` 9 | 10 | 1. Create cluster on your favorite non-GCP cloud provider. For example, [create Amazon EKS cluster](https://docs.aws.amazon.com/eks/latest/userguide/create-cluster.html) 11 | 1. Create GCP project and install Config Connector on your cluster: 12 | 13 | ``` 14 | export PROJECT_ID=[PROJECT_ID] 15 | export SA_EMAIL="cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com" 16 | 17 | # create project 18 | gcloud projects create $PROJECT_ID --name="$PROJECT_ID" 19 | gcloud config set project $PROJECT_ID 20 | 21 | # to provision Config Connector, create cnrm-system service account and export the tkey 22 | gcloud iam service-accounts create cnrm-system --project ${PROJECT_ID} 23 | gcloud projects add-iam-policy-binding ${PROJECT_ID} --member "serviceAccount:${SA_EMAIL}" --role roles/owner 24 | gcloud iam service-accounts keys create --iam-account "${SA_EMAIL}" ./key.json 25 | 26 | # install Config Connector 27 | kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account) 28 | curl -X GET -sLO --location-trusted https://us-central1-cnrm-eap.cloudfunctions.net/download/latest/infra/install-bundle.tar.gz 29 | rm -rf install-bundle 30 | tar zxvf install-bundle.tar.gz 31 | kubectl apply -f install-bundle/ 32 | 33 | # give cnrm-system namespace permissions to manage GCP 34 | kubectl create secret generic gcp-key --from-file ./key.json --namespace cnrm-system 35 | 36 | # annotate the namespace 37 | kubectl annotate namespace default "cnrm.cloud.google.com/project-id=${PROJECT_ID}" --overwrite 38 | ``` 39 | 1. Create your GCR bucket: 40 | ``` 41 | kubectl apply -f ./resources/gcr-bucket.yaml 42 | ``` 43 | 1. Upload the image to your GCR bucket. For example, using my name: 44 | ``` 45 | docker pull bulankou/node-hello-world:latest 46 | docker tag bulankou/node-hello-world gcr.io/[PROJECT_ID]/node-hello-world 47 | docker push gcr.io/[PROJECT_ID]/node-hello-world 48 | ``` 49 | 50 | 1. Create resources, including service account, service account key, IAM policy for the service account and GCR bucket. 51 | ``` 52 | kubectl apply -f ./resources/ 53 | ``` 54 | 55 | 1. Create `gcr-docker-key` secret to pull the image. It is using `gcr-sa-key` that was automatically created by Config Connector. 56 | ``` 57 | kubectl create secret docker-registry gcr-docker-key \ 58 | --docker-server=https://gcr.io \ 59 | --docker-username=_json_key \ 60 | --docker-email=user@example.com \ 61 | --docker-password="$(kubectl get secret gcr-sa-key -o go-template=$'{{index .data "key.json"}}' | base64 --decode)" 62 | ``` 63 | 1. Verify that pod is created: 64 | ``` 65 | kubectl get pods 66 | kubectl exec node-app-pod curl http://localhost:8080 67 | ``` 68 | -------------------------------------------------------------------------------- /src/gcr-image/resources/gcp-gcr-bucket-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: gcr-bucket-policy 5 | spec: 6 | resourceRef: 7 | apiVersion: storage.cnrm.cloud.google.com/v1beta1 8 | kind: StorageBucket 9 | name: artifacts.[PROJECT_ID].appspot.com 10 | bindings: 11 | - role: roles/storage.objectViewer 12 | members: 13 | - serviceAccount:gcr-sa@[PROJECT_ID].iam.gserviceaccount.com 14 | -------------------------------------------------------------------------------- /src/gcr-image/resources/gcp-gcr-service-account-key.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccountKey 3 | metadata: 4 | name: gcr-sa-key 5 | spec: 6 | publicKeyType: TYPE_X509_PEM_FILE 7 | keyAlgorithm: KEY_ALG_RSA_2048 8 | privateKeyType: TYPE_GOOGLE_CREDENTIALS_FILE 9 | serviceAccountRef: 10 | name: gcr-sa 11 | -------------------------------------------------------------------------------- /src/gcr-image/resources/gcp-gcr-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: gcr-sa 5 | spec: 6 | displayName: Service Account for GCR access 7 | -------------------------------------------------------------------------------- /src/gcr-image/resources/gcr-bucket.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.cnrm.cloud.google.com/v1beta1 2 | kind: StorageBucket 3 | metadata: 4 | name: artifacts.[PROJECT_ID].appspot.com 5 | -------------------------------------------------------------------------------- /src/gcr-image/resources/k8s-pod.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: node-app-pod 5 | spec: 6 | containers: 7 | - name: node-app-container 8 | image: gcr.io/[PROJECT_ID]/node-hello-world 9 | imagePullPolicy: Always 10 | env: 11 | - name: HELLO_MESSAGE 12 | value: "Hello from GCR!" 13 | ports: 14 | - containerPort: 8080 15 | imagePullSecrets: 16 | - name: gcr-docker-key 17 | -------------------------------------------------------------------------------- /src/helm1/README.md: -------------------------------------------------------------------------------- 1 | * Install HELM. [Instructions](https://helm.sh/docs/using_helm/#installing-helm) 2 | * Follow recommended init steps 3 | * Commands 4 | ```bash 5 | helm init --history-max 200 # start tiller 6 | helm list # list releases 7 | helm delete [release_name] # delete release 8 | ``` 9 | * 10 | -------------------------------------------------------------------------------- /src/helm1/mychart/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /src/helm1/mychart/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "1.0" 3 | description: A Helm chart for Kubernetes 4 | name: mychart 5 | version: 0.1.0 6 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if .Values.ingress.enabled }} 3 | {{- range $host := .Values.ingress.hosts }} 4 | {{- range .paths }} 5 | http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }} 6 | {{- end }} 7 | {{- end }} 8 | {{- else if contains "NodePort" .Values.service.type }} 9 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mychart.fullname" . }}) 10 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 11 | echo http://$NODE_IP:$NODE_PORT 12 | {{- else if contains "LoadBalancer" .Values.service.type }} 13 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 14 | You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mychart.fullname" . }}' 15 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mychart.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") 16 | echo http://$SERVICE_IP:{{ .Values.service.port }} 17 | {{- else if contains "ClusterIP" .Values.service.type }} 18 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mychart.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 19 | echo "Visit http://127.0.0.1:8080 to use your application" 20 | kubectl port-forward $POD_NAME 8080:80 21 | {{- end }} 22 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "mychart.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "mychart.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "mychart.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "mychart.labels" -}} 38 | app.kubernetes.io/name: {{ include "mychart.name" . }} 39 | helm.sh/chart: {{ include "mychart.chart" . }} 40 | app.kubernetes.io/instance: {{ .Release.Name }} 41 | {{- if .Chart.AppVersion }} 42 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 43 | {{- end }} 44 | app.kubernetes.io/managed-by: {{ .Release.Service }} 45 | {{- end -}} 46 | 47 | {{/* 48 | Create the name of the service account to use 49 | */}} 50 | {{- define "mychart.serviceAccountName" -}} 51 | {{- if .Values.serviceAccount.create -}} 52 | {{ default (include "mychart.fullname" .) .Values.serviceAccount.name }} 53 | {{- else -}} 54 | {{ default "default" .Values.serviceAccount.name }} 55 | {{- end -}} 56 | {{- end -}} 57 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: {{ .Release.Name }}-configmap 5 | data: 6 | myvalue: "Hello World" 7 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "mychart.fullname" . }} 5 | labels: 6 | {{ include "mychart.labels" . | indent 4 }} 7 | spec: 8 | replicas: {{ .Values.replicaCount }} 9 | selector: 10 | matchLabels: 11 | app.kubernetes.io/name: {{ include "mychart.name" . }} 12 | app.kubernetes.io/instance: {{ .Release.Name }} 13 | template: 14 | metadata: 15 | labels: 16 | app.kubernetes.io/name: {{ include "mychart.name" . }} 17 | app.kubernetes.io/instance: {{ .Release.Name }} 18 | spec: 19 | {{- with .Values.imagePullSecrets }} 20 | imagePullSecrets: 21 | {{- toYaml . | nindent 8 }} 22 | {{- end }} 23 | serviceAccountName: {{ template "mychart.serviceAccountName" . }} 24 | securityContext: 25 | {{- toYaml .Values.podSecurityContext | nindent 8 }} 26 | containers: 27 | - name: {{ .Chart.Name }} 28 | securityContext: 29 | {{- toYaml .Values.securityContext | nindent 12 }} 30 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 31 | imagePullPolicy: {{ .Values.image.pullPolicy }} 32 | ports: 33 | - name: http 34 | containerPort: 80 35 | protocol: TCP 36 | livenessProbe: 37 | httpGet: 38 | path: / 39 | port: http 40 | readinessProbe: 41 | httpGet: 42 | path: / 43 | port: http 44 | resources: 45 | {{- toYaml .Values.resources | nindent 12 }} 46 | {{- with .Values.nodeSelector }} 47 | nodeSelector: 48 | {{- toYaml . | nindent 8 }} 49 | {{- end }} 50 | {{- with .Values.affinity }} 51 | affinity: 52 | {{- toYaml . | nindent 8 }} 53 | {{- end }} 54 | {{- with .Values.tolerations }} 55 | tolerations: 56 | {{- toYaml . | nindent 8 }} 57 | {{- end }} 58 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "mychart.fullname" . -}} 3 | {{- $svcPort := .Values.service.port -}} 4 | {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 5 | apiVersion: networking.k8s.io/v1beta1 6 | {{- else -}} 7 | apiVersion: extensions/v1beta1 8 | {{- end }} 9 | kind: Ingress 10 | metadata: 11 | name: {{ $fullName }} 12 | labels: 13 | {{ include "mychart.labels" . | indent 4 }} 14 | {{- with .Values.ingress.annotations }} 15 | annotations: 16 | {{- toYaml . | nindent 4 }} 17 | {{- end }} 18 | spec: 19 | {{- if .Values.ingress.tls }} 20 | tls: 21 | {{- range .Values.ingress.tls }} 22 | - hosts: 23 | {{- range .hosts }} 24 | - {{ . | quote }} 25 | {{- end }} 26 | secretName: {{ .secretName }} 27 | {{- end }} 28 | {{- end }} 29 | rules: 30 | {{- range .Values.ingress.hosts }} 31 | - host: {{ .host | quote }} 32 | http: 33 | paths: 34 | {{- range .paths }} 35 | - path: {{ . }} 36 | backend: 37 | serviceName: {{ $fullName }} 38 | servicePort: {{ $svcPort }} 39 | {{- end }} 40 | {{- end }} 41 | {{- end }} 42 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "mychart.fullname" . }} 5 | labels: 6 | {{ include "mychart.labels" . | indent 4 }} 7 | spec: 8 | type: {{ .Values.service.type }} 9 | ports: 10 | - port: {{ .Values.service.port }} 11 | targetPort: http 12 | protocol: TCP 13 | name: http 14 | selector: 15 | app.kubernetes.io/name: {{ include "mychart.name" . }} 16 | app.kubernetes.io/instance: {{ .Release.Name }} 17 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "mychart.serviceAccountName" . }} 6 | labels: 7 | {{ include "mychart.labels" . | indent 4 }} 8 | {{- end -}} 9 | -------------------------------------------------------------------------------- /src/helm1/mychart/templates/tests/test-connection.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: "{{ include "mychart.fullname" . }}-test-connection" 5 | labels: 6 | {{ include "mychart.labels" . | indent 4 }} 7 | annotations: 8 | "helm.sh/hook": test-success 9 | spec: 10 | containers: 11 | - name: wget 12 | image: busybox 13 | command: ['wget'] 14 | args: ['{{ include "mychart.fullname" . }}:{{ .Values.service.port }}'] 15 | restartPolicy: Never 16 | -------------------------------------------------------------------------------- /src/helm1/mychart/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for mychart. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | image: 8 | repository: nginx 9 | tag: stable 10 | pullPolicy: IfNotPresent 11 | 12 | imagePullSecrets: [] 13 | nameOverride: "" 14 | fullnameOverride: "" 15 | 16 | serviceAccount: 17 | # Specifies whether a service account should be created 18 | create: true 19 | # The name of the service account to use. 20 | # If not set and create is true, a name is generated using the fullname template 21 | name: 22 | 23 | podSecurityContext: {} 24 | # fsGroup: 2000 25 | 26 | securityContext: {} 27 | # capabilities: 28 | # drop: 29 | # - ALL 30 | # readOnlyRootFilesystem: true 31 | # runAsNonRoot: true 32 | # runAsUser: 1000 33 | 34 | service: 35 | type: ClusterIP 36 | port: 80 37 | 38 | ingress: 39 | enabled: false 40 | annotations: {} 41 | # kubernetes.io/ingress.class: nginx 42 | # kubernetes.io/tls-acme: "true" 43 | hosts: 44 | - host: chart-example.local 45 | paths: [] 46 | 47 | tls: [] 48 | # - secretName: chart-example-tls 49 | # hosts: 50 | # - chart-example.local 51 | 52 | resources: {} 53 | # We usually recommend not to specify default resources and to leave this as a conscious 54 | # choice for the user. This also increases chances charts run on environments with little 55 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 56 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 57 | # limits: 58 | # cpu: 100m 59 | # memory: 128Mi 60 | # requests: 61 | # cpu: 100m 62 | # memory: 128Mi 63 | 64 | nodeSelector: {} 65 | 66 | tolerations: [] 67 | 68 | affinity: {} 69 | -------------------------------------------------------------------------------- /src/mci/README.md: -------------------------------------------------------------------------------- 1 | # Multi-Cluster Ingress 2 | 3 | 1. [Provision project, cluster and Config Connector](../../provision.md) 4 | 1. Deploy clusters and node pools for them: 5 | 6 | ```bash 7 | kubectl apply -f resources/clusters/gcp-clusters.yaml 8 | ``` 9 | 10 | 1. Switch the context to North American cluster and create a pod and a service: 11 | 12 | ```bash 13 | gcloud container clusters get-credentials cluster-na --zone=us-central1-a 14 | kubectl apply -f resources/na/ 15 | ``` 16 | 17 | 1. Switch the context to Europe cluster and create a pod and a service: 18 | 19 | ```bash 20 | gcloud container clusters get-credentials cluster-eu --zone=europe-west2-a 21 | kubectl apply -f resources/eu/ 22 | ``` 23 | 24 | 1. Get the links to newly created network endpoint groups: 25 | ```bash 26 | gcloud compute network-endpoint-groups list --format="value(uri())" 27 | ``` 28 | 29 | Update values in resources/lb/gcp-backend-service.yaml 30 | 1. Switch the context back the default cluster with Config Connector installed and create the load balancing resources 31 | 32 | ```bash 33 | gcloud container clusters get-credentials cluster-1 --zone=us-central1-b 34 | kubectl apply -f resources/lb/ 35 | ``` 36 | 37 | 1. Use `gcloud compute forwarding-rules list` to obtain forwarding rule address 38 | ```bash 39 | $ gcloud compute forwarding-rules list 40 | NAME REGION IP_ADDRESS IP_PROTOCOL TARGET 41 | node-app-fw-rule TCP node-app-target-proxy 42 | ``` 43 | and try curl'ing it: 44 | 45 | ```bash 46 | curl 47 | ``` 48 | Your should see "Hello from North America" or "Hello from Europe" dependending on what region is closer to your location. 49 | 50 | 1. Try changing one of the deployments to specify wrong image and then killing and redeploying it. If you continue curl'ing, then you will see 502 error codes, and then the service will recover and start sending the response from the region that is not closest to you. 51 | -------------------------------------------------------------------------------- /src/mci/resources/autoneg/autoneg.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: autoneg-system 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: Role 10 | metadata: 11 | name: autoneg-leader-election-role 12 | namespace: autoneg-system 13 | rules: 14 | - apiGroups: 15 | - "" 16 | resources: 17 | - configmaps 18 | verbs: 19 | - get 20 | - list 21 | - watch 22 | - create 23 | - update 24 | - patch 25 | - delete 26 | - apiGroups: 27 | - "" 28 | resources: 29 | - configmaps/status 30 | verbs: 31 | - get 32 | - update 33 | - patch 34 | - apiGroups: 35 | - "" 36 | resources: 37 | - events 38 | verbs: 39 | - create 40 | --- 41 | apiVersion: rbac.authorization.k8s.io/v1 42 | kind: ClusterRole 43 | metadata: 44 | creationTimestamp: null 45 | name: autoneg-manager-role 46 | rules: 47 | - apiGroups: 48 | - "" 49 | resources: 50 | - services 51 | verbs: 52 | - get 53 | - list 54 | - watch 55 | - create 56 | - update 57 | - patch 58 | - delete 59 | - apiGroups: 60 | - "" 61 | resources: 62 | - services/status 63 | verbs: 64 | - get 65 | - update 66 | - patch 67 | --- 68 | apiVersion: rbac.authorization.k8s.io/v1 69 | kind: ClusterRole 70 | metadata: 71 | name: autoneg-proxy-role 72 | rules: 73 | - apiGroups: 74 | - authentication.k8s.io 75 | resources: 76 | - tokenreviews 77 | verbs: 78 | - create 79 | - apiGroups: 80 | - authorization.k8s.io 81 | resources: 82 | - subjectaccessreviews 83 | verbs: 84 | - create 85 | --- 86 | apiVersion: rbac.authorization.k8s.io/v1 87 | kind: RoleBinding 88 | metadata: 89 | name: autoneg-leader-election-rolebinding 90 | namespace: autoneg-system 91 | roleRef: 92 | apiGroup: rbac.authorization.k8s.io 93 | kind: Role 94 | name: autoneg-leader-election-role 95 | subjects: 96 | - kind: ServiceAccount 97 | name: default 98 | namespace: autoneg-system 99 | --- 100 | apiVersion: rbac.authorization.k8s.io/v1 101 | kind: ClusterRoleBinding 102 | metadata: 103 | name: autoneg-manager-rolebinding 104 | roleRef: 105 | apiGroup: rbac.authorization.k8s.io 106 | kind: ClusterRole 107 | name: autoneg-manager-role 108 | subjects: 109 | - kind: ServiceAccount 110 | name: default 111 | namespace: autoneg-system 112 | --- 113 | apiVersion: rbac.authorization.k8s.io/v1 114 | kind: ClusterRoleBinding 115 | metadata: 116 | name: autoneg-proxy-rolebinding 117 | roleRef: 118 | apiGroup: rbac.authorization.k8s.io 119 | kind: ClusterRole 120 | name: autoneg-proxy-role 121 | subjects: 122 | - kind: ServiceAccount 123 | name: default 124 | namespace: autoneg-system 125 | --- 126 | apiVersion: v1 127 | kind: Service 128 | metadata: 129 | annotations: 130 | prometheus.io/port: "8443" 131 | prometheus.io/scheme: https 132 | prometheus.io/scrape: "true" 133 | labels: 134 | control-plane: controller-manager 135 | name: autoneg-controller-manager-metrics-service 136 | namespace: autoneg-system 137 | spec: 138 | ports: 139 | - name: https 140 | port: 8443 141 | targetPort: https 142 | selector: 143 | control-plane: controller-manager 144 | --- 145 | apiVersion: apps/v1 146 | kind: Deployment 147 | metadata: 148 | labels: 149 | control-plane: controller-manager 150 | name: autoneg-controller-manager 151 | namespace: autoneg-system 152 | spec: 153 | replicas: 1 154 | selector: 155 | matchLabels: 156 | control-plane: controller-manager 157 | template: 158 | metadata: 159 | labels: 160 | control-plane: controller-manager 161 | spec: 162 | containers: 163 | - args: 164 | - --secure-listen-address=0.0.0.0:8443 165 | - --upstream=http://127.0.0.1:8080/ 166 | - --logtostderr=true 167 | - --v=10 168 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0 169 | name: kube-rbac-proxy 170 | ports: 171 | - containerPort: 8443 172 | name: https 173 | - args: 174 | - --metrics-addr=127.0.0.1:8080 175 | - --enable-leader-election 176 | command: 177 | - /manager 178 | image: gcr.io/soell-labs/gke-autoneg-controller:latest 179 | name: manager 180 | resources: 181 | limits: 182 | cpu: 100m 183 | memory: 30Mi 184 | requests: 185 | cpu: 100m 186 | memory: 20Mi 187 | terminationGracePeriodSeconds: 10 188 | -------------------------------------------------------------------------------- /src/mci/resources/clusters/gcp-clusters.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: container.cnrm.cloud.google.com/v1beta1 2 | kind: ContainerCluster 3 | metadata: 4 | name: cluster-na 5 | spec: 6 | ipAllocationPolicy: 7 | useIpAliases: true 8 | location: us-central1-a 9 | initialNodeCount: 1 10 | removeDefaultNodePool: false 11 | loggingService: none 12 | nodeConfig: 13 | oauthScopes: 14 | - "https://www.googleapis.com/auth/cloud-platform" 15 | masterAuth: 16 | username: "user" 17 | password: "password12345678" 18 | clientCertificateConfig: 19 | issueClientCertificate: false 20 | --- 21 | apiVersion: container.cnrm.cloud.google.com/v1beta1 22 | kind: ContainerCluster 23 | metadata: 24 | name: cluster-eu 25 | spec: 26 | ipAllocationPolicy: 27 | useIpAliases: true 28 | location: europe-west2-a 29 | initialNodeCount: 1 30 | removeDefaultNodePool: false 31 | loggingService: none 32 | nodeConfig: 33 | oauthScopes: 34 | - "https://www.googleapis.com/auth/cloud-platform" 35 | masterAuth: 36 | username: "user" 37 | password: "password12345678" 38 | clientCertificateConfig: 39 | issueClientCertificate: false 40 | -------------------------------------------------------------------------------- /src/mci/resources/eu/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: node-app-eu-deployment 5 | labels: 6 | app: node-app 7 | spec: 8 | replicas: 3 9 | selector: 10 | matchLabels: 11 | app: node-app 12 | template: 13 | metadata: 14 | labels: 15 | app: node-app 16 | spec: 17 | containers: 18 | - name: node-app-container 19 | image: bulankou/node-hello-world 20 | env: 21 | - name: HELLO_MESSAGE 22 | value: "Hello from Europe!" 23 | ports: 24 | - containerPort: 8080 25 | readinessProbe: 26 | httpGet: 27 | path: / 28 | port: 8080 29 | initialDelaySeconds: 10 30 | periodSeconds: 10 31 | timeoutSeconds: 10 32 | failureThreshold: 10 33 | successThreshold: 1 34 | livenessProbe: 35 | httpGet: 36 | path: / 37 | port: 8080 38 | initialDelaySeconds: 10 39 | periodSeconds: 10 40 | timeoutSeconds: 10 41 | failureThreshold: 20 42 | successThreshold: 1 43 | -------------------------------------------------------------------------------- /src/mci/resources/eu/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: node-app-backend 5 | annotations: 6 | cloud.google.com/neg: '{"exposed_ports": {"80":{}}}' 7 | anthos.cft.dev/autoneg: '{"name":"node-app-backend-service", "max_rate_per_endpoint":100}' 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | app: node-app 12 | ports: 13 | - port: 80 14 | targetPort: 8080 15 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-backend-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeBackendService 3 | metadata: 4 | name: node-app-backend-service 5 | labels: 6 | retry: again 7 | spec: 8 | healthCheckRef: 9 | name: node-app-backend-healthcheck 10 | protocol: HTTP 11 | location: global 12 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-compute-firewall.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeFirewall 3 | metadata: 4 | name: fw-allow-mci-neg 5 | spec: 6 | allow: 7 | - protocol: tcp 8 | sourceRanges: 9 | - "130.211.0.0/22" 10 | - "35.191.0.0/16" 11 | networkRef: 12 | name: default 13 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-compute-forwarding-rule.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeForwardingRule 3 | metadata: 4 | name: node-app-forwarding-rule 5 | spec: 6 | target: 7 | targetHTTPProxyRef: 8 | name: node-app-target-proxy 9 | portRange: "80" 10 | ipProtocol: "TCP" 11 | ipVersion: "IPV4" 12 | location: global 13 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-compute-health-check.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeHealthCheck 3 | metadata: 4 | name: node-app-backend-healthcheck 5 | spec: 6 | checkIntervalSec: 10 7 | tcpHealthCheck: 8 | port: 8080 9 | location: global 10 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-default-network.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeNetwork 3 | metadata: 4 | name: default 5 | annotations: 6 | cnrm.cloud.google.com/deletion-policy: abandon 7 | spec: 8 | description: Default network for the project 9 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-target-http-proxy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeTargetHTTPProxy 3 | metadata: 4 | name: node-app-target-proxy 5 | spec: 6 | description: Proxy for node app 7 | urlMapRef: 8 | name: node-app-url-map 9 | location: global 10 | -------------------------------------------------------------------------------- /src/mci/resources/lb/gcp-url-map.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: compute.cnrm.cloud.google.com/v1beta1 2 | kind: ComputeURLMap 3 | metadata: 4 | name: node-app-url-map 5 | labels: 6 | retry: again 7 | spec: 8 | defaultService: 9 | backendServiceRef: 10 | name: node-app-backend-service 11 | location: global 12 | -------------------------------------------------------------------------------- /src/mci/resources/na/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: node-app-na-deployment 5 | labels: 6 | app: node-app 7 | spec: 8 | replicas: 3 9 | selector: 10 | matchLabels: 11 | app: node-app 12 | template: 13 | metadata: 14 | labels: 15 | app: node-app 16 | spec: 17 | containers: 18 | - name: node-app-container 19 | image: bulankou/node-hello-world 20 | env: 21 | - name: HELLO_MESSAGE 22 | value: "Hello from North America!" 23 | ports: 24 | - containerPort: 8080 25 | readinessProbe: 26 | httpGet: 27 | path: / 28 | port: 8080 29 | initialDelaySeconds: 10 30 | periodSeconds: 10 31 | timeoutSeconds: 10 32 | failureThreshold: 10 33 | successThreshold: 1 34 | livenessProbe: 35 | httpGet: 36 | path: / 37 | port: 8080 38 | initialDelaySeconds: 10 39 | periodSeconds: 10 40 | timeoutSeconds: 10 41 | failureThreshold: 20 42 | successThreshold: 1 43 | -------------------------------------------------------------------------------- /src/mci/resources/na/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: node-app-backend 5 | annotations: 6 | cloud.google.com/neg: '{"exposed_ports": {"80":{}}}' 7 | anthos.cft.dev/autoneg: '{"name":"node-app-backend-service", "max_rate_per_endpoint":100}' 8 | spec: 9 | type: ClusterIP 10 | selector: 11 | app: node-app 12 | ports: 13 | - port: 80 14 | targetPort: 8080 15 | -------------------------------------------------------------------------------- /src/multiteam/README.md: -------------------------------------------------------------------------------- 1 | # Namespace-Project Configuration for Multiple Teams 2 | 3 | We'll provision multiple Kubernetes namespaces, namespace per team. For each namespace: 4 | * There will be a associated GCP project for cloud infrastructure 5 | * K8s service account (KSA) linked to Google Service Account (GSA) on GCP side using Workload Identity 6 | * To illustrate this example, a pod communicating to Google Storage Bucket in GCP project 7 | 8 | 1. [Provision project, cluster and Config Connector](../../provision.md) 9 | 1. Create projects team-a and team-b dedicated for two teams. Give permissions to Config Connector to manage resources in these projects. You can aso uncomment part of the script to create groups and give them permissions to projects: 10 | 11 | ```bash 12 | bash ../provision-projects.sh 13 | ``` 14 | 15 | 1. Create K8s namespace dedicated to team-a resources: 16 | 17 | ```bash 18 | kubectl apply -f team-a/resources-admin/k8s-namespace.yaml 19 | ``` 20 | 21 | 1. Configure permissions for this team-a-user to edit standard resources in this namespace and also give them cnrm-manager role to edit Config Connector resoruces, such as buckets. 22 | 23 | ```bash 24 | kubectl apply -f team-a/resources-admin/k8s-namespace-permissions.yaml 25 | ``` 26 | 1. Repeat previous two steps for team-b-user: 27 | 28 | ```bash 29 | kubectl apply -f team-b/resources-admin/k8s-namespace.yaml 30 | kubectl apply -f team-b/resources-admin/k8s-namespace-permissions.yaml 31 | ``` 32 | 33 | 1. Verify that team-a-user can create pods or sqlinstances in team-a namespace, but cannot do this in team-b or default namespace: 34 | ```bash 35 | $ kubectl auth can-i create pods --namespace team-a --as=team-a-user 36 | yes 37 | $ kubectl auth can-i create sqlinstances --namespace team-a --as=team-a-user 38 | yes 39 | $ kubectl auth can-i create sqlinstances --namespace team-b --as=team-a-user 40 | no 41 | $ kubectl auth can-i create sqlinstances --as=team-a-user 42 | no 43 | ``` 44 | 1. Try to modify team-a namespace to point it to a different project. You will get an error: 45 | ``` 46 | error: namespaces "team-a" could not be patched: namespaces "team-a" is forbidden: User "team-a-user" cannot patch resource "namespaces" in API group "" in the namespace "team-a" 47 | ``` 48 | 1. Create `team-a` resources, impersonating team member: 49 | ```bash 50 | kubectl apply -f team-a/resources-team --as=team-a-user 51 | ``` 52 | 1. Verify that service account credentials are propagating automatically. Run a pod with `google/cloud-sdk` image: 53 | ``` 54 | kubectl run -it \ 55 | --generator=run-pod/v1 \ 56 | --image google/cloud-sdk \ 57 | --serviceaccount ksa-bucket-team-a\ 58 | --namespace team-a \ 59 | team-a-ksa-test --as=team-a-user 60 | ``` 61 | 1. Once on the pod, run the following to test your permissions: 62 | ```bash 63 | gcloud auth list # google service account should be listed 64 | # create file 65 | echo some text > f1.txt 66 | # copy to bucket, shoudl succeed 67 | gsutil cp f1.txt gs://alexbu-kcc-multiteam-team-a-bucket 68 | # list files in bucket, should succeed 69 | gsutil ls gs://alexbu-kcc-multiteam-team-a-bucket 70 | ``` 71 | 72 | 1. Replicate the same configuration for team-b: 73 | 74 | ```bash 75 | kubectl apply -f team-b/resources-team --as=team-b-user 76 | ``` 77 | -------------------------------------------------------------------------------- /src/multiteam/provision-projects.sh: -------------------------------------------------------------------------------- 1 | export PROJECT_ID=[PROJECT_ID] 2 | export SA_EMAIL="cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com" 3 | 4 | 5 | # configure project and permissions for team-a 6 | 7 | # create dedicated project for team-a 8 | gcloud projects create ${PROJECT_ID}-team-a --name=${PROJECT_ID}-team-a 9 | 10 | # give team-a viewer permission to the main project 11 | # gcloud projects add-iam-policy-binding ${PROJECT_ID} \ 12 | # --member group:team-a@googlegroups.com \ 13 | # --role roles/viewer 14 | 15 | # give team-a editor permission to the dedicated project 16 | # gcloud projects add-iam-policy-binding ${PROJECT_ID}-team-a \ 17 | # --member group:team-a@googlegroups.com \ 18 | # --role roles/editor 19 | 20 | # give cnrm system access to team-a dedicated project 21 | gcloud projects add-iam-policy-binding ${PROJECT_ID}-team-a --member "serviceAccount:${SA_EMAIL}" --role roles/owner 22 | gcloud projects add-iam-policy-binding ${PROJECT_ID}-team-a --member "serviceAccount:${SA_EMAIL}" --role roles/storage.admin 23 | 24 | # create dedicated project for team-b 25 | gcloud projects create ${PROJECT_ID}-team-b --name=${PROJECT_ID}-team-b 26 | 27 | # give team-b viewer permission to the main project 28 | # gcloud projects add-iam-policy-binding ${PROJECT_ID} \ 29 | # --member group:team-b@googlegroups.com \ 30 | # --role roles/viewer 31 | 32 | # give team-b editor permission to the dedicated project 33 | # gcloud projects add-iam-policy-binding ${PROJECT_ID}-team-b \ 34 | # --member group:team-b@googlegroups.com \ 35 | # --role roles/editor 36 | 37 | # give cnrm system access to team-b dedicated project 38 | gcloud projects add-iam-policy-binding ${PROJECT_ID}-team-b --member "serviceAccount:${SA_EMAIL}" --role roles/owner 39 | gcloud projects add-iam-policy-binding ${PROJECT_ID}-team-b --member "serviceAccount:${SA_EMAIL}" --role roles/storage.admin 40 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-admin/k8s-namespace-permissions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: team-a-role-binding 5 | namespace: team-a 6 | subjects: 7 | - kind: User 8 | name: team-a-user 9 | apiGroup: rbac.authorization.k8s.io 10 | roleRef: 11 | kind: ClusterRole 12 | name: edit 13 | apiGroup: rbac.authorization.k8s.io 14 | --- 15 | apiVersion: rbac.authorization.k8s.io/v1 16 | kind: RoleBinding 17 | metadata: 18 | name: team-a-cnrm-manager-role-binding 19 | namespace: team-a 20 | subjects: 21 | - kind: User 22 | name: team-a-user 23 | apiGroup: rbac.authorization.k8s.io 24 | roleRef: 25 | kind: ClusterRole 26 | name: cnrm-manager-role 27 | apiGroup: rbac.authorization.k8s.io 28 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-admin/k8s-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | annotations: 5 | cnrm.cloud.google.com/project-id: [PROJECT_ID]-team-a 6 | name: team-a 7 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-team/gcp-bucket-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: bucket-policy-team-a 5 | namespace: team-a 6 | spec: 7 | resourceRef: 8 | apiVersion: storage.cnrm.cloud.google.com/v1beta1 9 | kind: StorageBucket 10 | name: [PROJECT_ID]-team-a-bucket 11 | namespace: team-a 12 | bindings: 13 | - role: roles/storage.admin 14 | members: 15 | - serviceAccount:sa-bucket-team-a@[PROJECT_ID]-team-a.iam.gserviceaccount.com 16 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-team/gcp-bucket.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.cnrm.cloud.google.com/v1beta1 2 | kind: StorageBucket 3 | metadata: 4 | name: [PROJECT_ID]-team-a-bucket 5 | namespace: team-a 6 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-team/gcp-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: sa-bucket-team-a 5 | namespace: team-a 6 | spec: 7 | displayName: service account for bucket access 8 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-team/gcp-wi-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: sa-wi-policty-team-a 5 | namespace: team-a 6 | spec: 7 | resourceRef: 8 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 9 | kind: IAMServiceAccount 10 | name: sa-bucket-team-a 11 | namespace: team-a 12 | bindings: 13 | - role: roles/iam.workloadIdentityUser 14 | members: 15 | - serviceAccount:[PROJECT_ID].svc.id.goog[team-a/ksa-bucket-team-a] 16 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-team/k8s-pod.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /src/multiteam/team-a/resources-team/k8s-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: ksa-bucket-team-a 5 | namespace: team-a 6 | annotations: 7 | iam.gke.io/gcp-service-account: sa-bucket-team-a@[PROJECT_ID]-team-a.iam.gserviceaccount.com 8 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-admin/k8s-namespace-permissions.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: team-b-role-binding 5 | namespace: team-b 6 | subjects: 7 | - kind: User 8 | name: team-b-user 9 | apiGroup: rbac.authorization.k8s.io 10 | roleRef: 11 | kind: ClusterRole 12 | name: edit 13 | apiGroup: rbac.authorization.k8s.io 14 | --- 15 | apiVersion: rbac.authorization.k8s.io/v1 16 | kind: RoleBinding 17 | metadata: 18 | name: team-b-cnrm-manager-role-binding 19 | namespace: team-b 20 | subjects: 21 | - kind: User 22 | name: team-b-user 23 | apiGroup: rbac.authorization.k8s.io 24 | roleRef: 25 | kind: ClusterRole 26 | name: cnrm-manager-role 27 | apiGroup: rbac.authorization.k8s.io 28 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-admin/k8s-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | annotations: 5 | cnrm.cloud.google.com/project-id: [PROJECT_ID]-team-b 6 | name: team-b 7 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-team/gcp-bucket-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: bucket-policy-team-b 5 | namespace: team-b 6 | spec: 7 | resourceRef: 8 | apiVersion: storage.cnrm.cloud.google.com/v1beta1 9 | kind: StorageBucket 10 | name: [PROJECT_ID]-team-b-bucket 11 | namespace: team-b 12 | bindings: 13 | - role: roles/storage.admin 14 | members: 15 | - serviceAccount:sa-bucket-team-b@[PROJECT_ID]-team-b.iam.gserviceaccount.com 16 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-team/gcp-bucket.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: storage.cnrm.cloud.google.com/v1beta1 2 | kind: StorageBucket 3 | metadata: 4 | name: [PROJECT_ID]-team-b-bucket 5 | namespace: team-b 6 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-team/gcp-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: sa-bucket-team-b 5 | namespace: team-b 6 | spec: 7 | displayName: service account for bucket access 8 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-team/gcp-wi-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: sa-wi-policty-team-b 5 | namespace: team-b 6 | spec: 7 | resourceRef: 8 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 9 | kind: IAMServiceAccount 10 | name: sa-bucket-team-b 11 | namespace: team-b 12 | bindings: 13 | - role: roles/iam.workloadIdentityUser 14 | members: 15 | - serviceAccount:[PROJECT_ID].svc.id.goog[team-b/ksa-bucket-team-b] 16 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-team/k8s-pod.yaml: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /src/multiteam/team-b/resources-team/k8s-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: ksa-bucket-team-b 5 | namespace: team-b 6 | annotations: 7 | iam.gke.io/gcp-service-account: sa-bucket-team-b@[PROJECT_ID]-team-b.iam.gserviceaccount.com 8 | -------------------------------------------------------------------------------- /src/provision.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | export PROJECT_ID=[PROJECT_ID] 5 | export BILLING_ACCOUNT=[BILLING_ACCOUNT] 6 | export CLUSTER_ID=cluster-1 7 | export ZONE=us-central1-b 8 | export SA_EMAIL="cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com" 9 | export WP_SA_EMAIL=sql-wp-sa@${PROJECT_ID}.iam.gserviceaccount.com 10 | 11 | 12 | gcloud projects create $PROJECT_ID --name="$PROJECT_ID" --folder=[FOLDER_ID] 13 | gcloud alpha billing projects link $PROJECT_ID --billing-account $BILLING_ACCOUNT 14 | gcloud config set project $PROJECT_ID 15 | 16 | 17 | # provision project 18 | gcloud iam service-accounts create cnrm-system --project ${PROJECT_ID} 19 | gcloud projects add-iam-policy-binding ${PROJECT_ID} --member "serviceAccount:${SA_EMAIL}" --role roles/owner 20 | gcloud iam service-accounts keys create --iam-account "${SA_EMAIL}" ./key.json 21 | 22 | #gcloud services enable pubsub.googleapis.com --project ${PROJECT_ID} 23 | #gcloud services enable spanner.googleapis.com --project ${PROJECT_ID} 24 | #gcloud services enable sqladmin.googleapis.com --project ${PROJECT_ID} 25 | #gcloud services enable redis.googleapis.com --project ${PROJECT_ID} 26 | gcloud services enable cloudresourcemanager.googleapis.com --project ${PROJECT_ID} 27 | gcloud services enable container.googleapis.com --project ${PROJECT_ID} 28 | #gcloud services enable dns.googleapis.com --project ${PROJECT_ID} 29 | 30 | 31 | # for each cluster 32 | # Note: this creates a cluster with workload identity enabled, using Beta API 33 | gcloud container clusters create ${CLUSTER_ID} --workload-pool=${PROJECT_ID}.svc.id.goog --zone $ZONE 34 | gcloud container clusters get-credentials $CLUSTER_ID --zone=$ZONE 35 | 36 | # install KCC 37 | kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account) 38 | gsutil cp gs://cnrm/latest/release-bundle.tar.gz release-bundle.tar.gz 39 | rm -rf release-bundle 40 | tar zxvf release-bundle.tar.gz 41 | kubectl apply -f install-bundle-gcp-identity/ 42 | 43 | # give cnrm-system namespace permissions to manage GCP 44 | kubectl create secret generic gcp-key --from-file ./key.json --namespace cnrm-system 45 | 46 | # annotate the namespace 47 | kubectl annotate namespace default "cnrm.cloud.google.com/project-id=${PROJECT_ID}" --overwrite 48 | 49 | -------------------------------------------------------------------------------- /src/pubsub/keyless-iam.yaml: -------------------------------------------------------------------------------- 1 | # This set of resources is to create Google Cloud Pubsub, service account for access 2 | # and propagate permissions to K8s. 3 | # 4 | # Parameterize: 5 | # [PROJECT_ID] - your GCP project ID 6 | # [PUBSUB_TOPIC_NAME] - name of your Pubsub topic 7 | # [PUBSUB_SUBSCRIPTION_NAME] - name of your Pubsub subscription 8 | # [PUBSUB_POLICY] - pubsub policy name 9 | # [SA_NAME] - service account name 10 | # [SA_KEY] - service account key 11 | # 12 | # To use it in your pod, add to your container: 13 | # volumeMounts: 14 | # - name: [SA_KEY] 15 | # mountPath: /var/secrets/google 16 | # env: 17 | # - name: GOOGLE_APPLICATION_CREDENTIALS 18 | # value: /var/secrets/google/key.json 19 | 20 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 21 | kind: PubSubTopic 22 | metadata: 23 | name: [PUBSUB_TOPIC_NAME] 24 | --- 25 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 26 | kind: PubSubSubscription 27 | metadata: 28 | name: [PUBSUB_SUBSCRIPTION_NAME] 29 | spec: 30 | topicRef: 31 | name: [PUBSUB_TOPIC_NAME] 32 | --- 33 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 34 | kind: IAMPolicy 35 | metadata: 36 | name: [PUBSUB_POLICY] 37 | spec: 38 | resourceRef: 39 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 40 | kind: PubSubTopic 41 | name: [PUBSUB_TOPIC_NAME] 42 | bindings: 43 | - role: roles/editor 44 | members: 45 | - serviceAccount:[SA_NAME]@[PROJECT_ID].iam.gserviceaccount.com 46 | --- 47 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 48 | kind: IAMServiceAccount 49 | metadata: 50 | name: [SA_NAME] 51 | spec: 52 | displayName: Service account for Pubsub access 53 | --- 54 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 55 | kind: IAMServiceAccountKey 56 | metadata: 57 | name: [SA_KEY] 58 | spec: 59 | publicKeyType: TYPE_X509_PEM_FILE 60 | keyAlgorithm: KEY_ALG_RSA_2048 61 | privateKeyType: TYPE_GOOGLE_CREDENTIALS_FILE 62 | serviceAccountRef: 63 | name: [SA_NAME] 64 | -------------------------------------------------------------------------------- /src/pubsub/wi-iam.yaml: -------------------------------------------------------------------------------- 1 | # This set of resources is to create Google Cloud Pubsub, service account for access 2 | # and propagate permissions to K8s using Workload identity 3 | # 4 | # Parameterize: 5 | # [PROJECT_ID] - your GCP project ID 6 | # [PUBSUB_TOPIC_NAME] - name of your Pubsub topic 7 | # [PUBSUB_SUBSCRIPTION_NAME] - name of your Pubsub subscription 8 | # [PUBSUB_POLICY] - pubsub policy name 9 | # [SA_NAME] - service account name 10 | # [KSA_NAME] - K8s service account name 11 | # [KNS] - K8s namespace 12 | # To use it in your pod, add to your pod spec: 13 | # serviceAccountName: [KSA_NAME] 14 | 15 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 16 | kind: PubSubTopic 17 | metadata: 18 | name: [PUBSUB_TOPIC_NAME] 19 | --- 20 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 21 | kind: PubSubSubscription 22 | metadata: 23 | name: [PUBSUB_SUBSCRIPTION_NAME] 24 | spec: 25 | topicRef: 26 | name: [PUBSUB_TOPIC_NAME] 27 | --- 28 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 29 | kind: IAMPolicy 30 | metadata: 31 | name: [PUBSUB_POLICY] 32 | spec: 33 | resourceRef: 34 | apiVersion: pubsub.cnrm.cloud.google.com/v1beta1 35 | kind: PubSubTopic 36 | name: [PUBSUB_TOPIC_NAME] 37 | bindings: 38 | - role: roles/editor 39 | members: 40 | - serviceAccount:[SA_NAME]@[PROJECT_ID].iam.gserviceaccount.com 41 | --- 42 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 43 | kind: IAMServiceAccount 44 | metadata: 45 | name: [SA_NAME] 46 | spec: 47 | displayName: Service account for Pubsub access 48 | --- 49 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 50 | kind: IAMPolicy 51 | metadata: 52 | name: [WI_POLICY] 53 | spec: 54 | resourceRef: 55 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 56 | kind: IAMServiceAccount 57 | name: [SA_NAME] 58 | bindings: 59 | - role: roles/iam.workloadIdentityUser 60 | members: 61 | - serviceAccount:[PROJECT_ID].svc.id.goog[[KNS]/[KSA_NAME]] 62 | --- 63 | apiVersion: v1 64 | kind: ServiceAccount 65 | metadata: 66 | name: [KSA_NAME] 67 | namespace: [KNS] 68 | annotations: 69 | iam.gke.io/gcp-service-account: [SA_NAME]@[PROJECT_ID].iam.gserviceaccount.com 70 | -------------------------------------------------------------------------------- /src/tf-provision/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/google" { 5 | version = "3.50.0" 6 | hashes = [ 7 | "h1:8X2o8seUN+DfP9GvQUVJCttKnc3BAGFpC9SdmNWoRwI=", 8 | "zh:194e113ccc1429d324d20a2709b979a993bbc0543ccaf3bad4b385f896323b2b", 9 | "zh:230ac07c6a4726ad64e16603e95f58d47bbd71dbca8709673c006fd4df14abaa", 10 | "zh:35483e5b8dda4c520d8f9f77592eb5b501ce8b04be47c5a93698e4d0eb67203e", 11 | "zh:39a1ac35e2f224c1ebb392c3c21e62a2547ef41f726c66ef53a574226b6556f7", 12 | "zh:a12de427187c4d36a8ed4655ee82fb466510413726424c8433a450c4b744590a", 13 | "zh:a1e4da62d8e4c77185c62d6e16deb4b20601773c065ad0b796e6d02777db6fe7", 14 | "zh:b25aaa7d72ea1b22450eb3ad202ba90d6aa36bab2d6537a302b37a205348220b", 15 | "zh:b7f649f96624a6d74c0fd45e06f051f6ae49130e643c61354f0ca83eeb604b41", 16 | "zh:df8ced9ad951b4acaf17fe8a7a6c2093bca2a8f4a347685e3baa6a08766fe3b0", 17 | "zh:e6556c2a4af6bed8fb3545f1f07a47fb6f2ebf91bc639cbee54981f1e5162184", 18 | ] 19 | } 20 | 21 | provider "registry.terraform.io/hashicorp/google-beta" { 22 | version = "3.50.0" 23 | hashes = [ 24 | "h1:9tOWshXi6FWD4L1DMqP0ndxl8ray+A0pOEPyUnkiex4=", 25 | "zh:003b42d6a0d91a05422c356ceb65b9491b6e2e4fc58ab1e8b00e714f93a7d2d7", 26 | "zh:06df1688347c8e0f7e2f302f1dd52ec493a24c7673e2bffd41135d506c24afcf", 27 | "zh:0ea50e386abd028b14fed901b44aefd56b6baa72aeb1dd598184b47219370541", 28 | "zh:3056434e0c1eccd123fdd676624d5461199968e0250a315b9ca0b750d17a060b", 29 | "zh:4697b1ab2c9a5142183b357451f940a3ea5162b6412cd5ba92a22c266562ff67", 30 | "zh:5d51dd6d514cf491310987291f2d4ec9f0d8b87d7570da977b1274e79d612338", 31 | "zh:641b969de0a7ed37af1e3d13e80943245b454b68a9e3506f4e016553609dd3d2", 32 | "zh:753f9bcf7becc4a9b2e1fd6c2224c5a7a8fbebd5bfda5818365c35d47cbedf2b", 33 | "zh:7a7c81445cfab905e289324c5515e3c268849879dfadb22bb56850a351cd2ce4", 34 | "zh:d4ab5338441c072a53aebc6a75d65d26897c1bcf965373157b0cdc7e789ef5c5", 35 | ] 36 | } 37 | 38 | provider "registry.terraform.io/hashicorp/random" { 39 | version = "3.0.0" 40 | hashes = [ 41 | "h1:yhHJpb4IfQQfuio7qjUXuUFTU/s+ensuEpm23A+VWz0=", 42 | "zh:0fcb00ff8b87dcac1b0ee10831e47e0203a6c46aafd76cb140ba2bab81f02c6b", 43 | "zh:123c984c0e04bad910c421028d18aa2ca4af25a153264aef747521f4e7c36a17", 44 | "zh:287443bc6fd7fa9a4341dec235589293cbcc6e467a042ae225fd5d161e4e68dc", 45 | "zh:2c1be5596dd3cca4859466885eaedf0345c8e7628503872610629e275d71b0d2", 46 | "zh:684a2ef6f415287944a3d966c4c8cee82c20e393e096e2f7cdcb4b2528407f6b", 47 | "zh:7625ccbc6ff17c2d5360ff2af7f9261c3f213765642dcd84e84ae02a3768fd51", 48 | "zh:9a60811ab9e6a5bfa6352fbb943bb530acb6198282a49373283a8fa3aa2b43fc", 49 | "zh:c73e0eaeea6c65b1cf5098b101d51a2789b054201ce7986a6d206a9e2dacaefd", 50 | "zh:e8f9ed41ac83dbe407de9f0206ef1148204a0d51ba240318af801ffb3ee5f578", 51 | "zh:fbdd0684e62563d3ac33425b0ac9439d543a3942465f4b26582bcfabcb149515", 52 | ] 53 | } 54 | -------------------------------------------------------------------------------- /src/tf-provision/provision.tf: -------------------------------------------------------------------------------- 1 | variable "project" {} 2 | variable "folder_id" {} 3 | variable "billing_account" {} 4 | 5 | locals { 6 | region = "us-central1" 7 | zone = "us-central1-b" 8 | } 9 | 10 | provider "google-beta" { 11 | region = local.region 12 | zone = local.zone 13 | } 14 | 15 | resource "random_id" "id" { 16 | byte_length = 4 17 | prefix = "${var.project}-" 18 | } 19 | 20 | 21 | 22 | resource "google_project" "root_project" { 23 | name = random_id.id.hex 24 | project_id = random_id.id.hex 25 | folder_id = var.folder_id 26 | billing_account = var.billing_account 27 | } 28 | 29 | resource "google_project_service" "crmservice" { 30 | project = google_project.root_project.project_id 31 | service = "cloudresourcemanager.googleapis.com" 32 | 33 | disable_dependent_services = true 34 | } 35 | 36 | resource "google_project_service" "containerservice" { 37 | project = google_project.root_project.project_id 38 | service = "container.googleapis.com" 39 | 40 | disable_dependent_services = true 41 | 42 | depends_on = [ 43 | google_project_service.crmservice 44 | ] 45 | } 46 | 47 | resource "google_container_cluster" "primary" { 48 | provider = google-beta 49 | name = "cluster-1" 50 | project = google_project.root_project.project_id 51 | location = local.zone 52 | 53 | # We can't create a cluster with no node pool defined, but we want to only use 54 | # separately managed node pools. So we create the smallest possible default 55 | # node pool and immediately delete it. 56 | remove_default_node_pool = true 57 | initial_node_count = 1 58 | 59 | master_auth { 60 | username = "" 61 | password = "" 62 | 63 | client_certificate_config { 64 | issue_client_certificate = false 65 | } 66 | } 67 | 68 | workload_identity_config { 69 | identity_namespace = "${google_project.root_project.project_id}.svc.id.goog" 70 | } 71 | 72 | addons_config { 73 | config_connector_config { 74 | enabled = true 75 | } 76 | } 77 | 78 | depends_on = [ 79 | google_project_service.containerservice 80 | ] 81 | } 82 | 83 | resource "google_container_node_pool" "primary_preemptible_nodes" { 84 | name = "primary-node-pool" 85 | project = google_project.root_project.project_id 86 | cluster = google_container_cluster.primary.name 87 | location = local.zone 88 | node_count = 3 89 | 90 | node_config { 91 | preemptible = true 92 | machine_type = "e2-medium" 93 | 94 | metadata = { 95 | disable-legacy-endpoints = "true" 96 | } 97 | 98 | oauth_scopes = [ 99 | "https://www.googleapis.com/auth/cloud-platform" 100 | ] 101 | } 102 | } 103 | 104 | resource "google_service_account" "cnrmsa" { 105 | account_id = "cnrmsa" 106 | project = google_project.root_project.project_id 107 | display_name = "IAM service account used by Config Connector" 108 | } 109 | 110 | resource "google_project_iam_binding" "project" { 111 | project = google_project.root_project.project_id 112 | role = "roles/owner" 113 | 114 | members = [ 115 | "serviceAccount:${google_service_account.cnrmsa.email}", 116 | ] 117 | 118 | depends_on = [ 119 | google_service_account.cnrmsa 120 | ] 121 | } 122 | 123 | resource "google_service_account_iam_binding" "admin-account-iam" { 124 | service_account_id = google_service_account.cnrmsa.name 125 | role = "roles/iam.workloadIdentityUser" 126 | 127 | members = [ 128 | "serviceAccount:${google_project.root_project.project_id}.svc.id.goog[cnrm-system/cnrm-controller-manager]", 129 | ] 130 | 131 | depends_on = [ 132 | google_container_cluster.primary, 133 | google_service_account.cnrmsa 134 | ] 135 | } 136 | 137 | output "project_id" { 138 | value = google_project.root_project.project_id 139 | description = "Created project id" 140 | } -------------------------------------------------------------------------------- /src/wp-acm/README.md: -------------------------------------------------------------------------------- 1 | # WordPress on K8s + GCP Cloud SQL + WI + Gatekeeper + ACM 2 | 3 | This sample shows how Anthos Config Management can be used together with Config Connector to manage GCP infrastructure through Git worklflow. 4 | 5 | 1. Fork [this repo](https://github.com/AlexBulankou/gcp-kcc-samples) 6 | 1. Clone your fork 7 | ```bash 8 | git clone https://github.com/[my-git-user-name]/gcp-kcc-samples.git repo-name/ 9 | ``` 10 | 11 | ... and continue working with your fork. 12 | 13 | 1. Provision the project 14 | ```bash 15 | export PROJECT_ID=[PROJECT_ID] 16 | export BILLING_ACCOUNT=[BILLING_ACCOUNT] 17 | export SA_EMAIL="cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com" 18 | 19 | gcloud projects create ${PROJECT_ID} --name=${PROJECT_ID} 20 | gcloud alpha billing projects link ${PROJECT_ID} --billing-account $BILLING_ACCOUNT 21 | gcloud config set project $PROJECT_ID 22 | 23 | # enable container API 24 | gcloud services enable container.googleapis.com --project ${PROJECT_ID} 25 | 26 | # enable SQL admin API 27 | gcloud services enable sqladmin.googleapis.com --project ${PROJECT_ID} 28 | 29 | # create service account that will be used for Config Connector 30 | gcloud iam service-accounts create cnrm-system --project ${PROJECT_ID} 31 | gcloud projects add-iam-policy-binding ${PROJECT_ID} --member "serviceAccount:${SA_EMAIL}" --role roles/owner 32 | gcloud iam service-accounts keys create --iam-account "${SA_EMAIL}" ./key.json 33 | ``` 34 | 35 | 36 | 1. Create two additional projects that will be holding the resources for Production and Dev environments 37 | 38 | ```bash 39 | export PROJECT_ID=[PROJECT_ID] 40 | export SA_EMAIL="cnrm-system@${PROJECT_ID}.iam.gserviceaccount.com" 41 | export BILLING_ACCOUNT=[BILLING_ACCOUNT] 42 | 43 | gcloud projects create ${PROJECT_ID}-dev --name=${PROJECT_ID}-dev 44 | gcloud alpha billing projects link ${PROJECT_ID}-dev --billing-account $BILLING_ACCOUNT 45 | gcloud projects add-iam-policy-binding ${PROJECT_ID}-dev --member "serviceAccount:${SA_EMAIL}" --role roles/owner 46 | gcloud services enable sqladmin.googleapis.com --project ${PROJECT_ID}-dev 47 | 48 | gcloud projects create ${PROJECT_ID}-prod --name=${PROJECT_ID}-prod 49 | gcloud alpha billing projects link ${PROJECT_ID}-prod --billing-account $BILLING_ACCOUNT 50 | gcloud projects add-iam-policy-binding ${PROJECT_ID}-prod --member "serviceAccount:${SA_EMAIL}" --role roles/owner 51 | gcloud services enable sqladmin.googleapis.com --project ${PROJECT_ID}-prod 52 | ``` 53 | 1. Create GKE cluster. We are using beta API to enable Workload Identity feature. 54 | ```bash 55 | gcloud beta container clusters create ${CLUSTER_ID} --identity-namespace=${PROJECT_ID}.svc.id.goog --zone $ZONE 56 | gcloud container clusters get-credentials $CLUSTER_ID --zone=$ZONE 57 | ``` 58 | 59 | 1. Install ACM operator 60 | 61 | ```bash 62 | # download 63 | gsutil cp gs://config-management-release/released/1.1.0/config-management-operator.yaml config-management-operator.yaml 64 | # apply CRD 65 | kubectl apply -f config-management-operator.yaml 66 | ``` 67 | 68 | 1. Your local fork should contain changes where PROJECT_ID template was applied. Check in these changes into Git. 69 | 70 | 1. Update ./config-management file with your custom repo name. Then apply ACM config on the cluster: 71 | 72 | ```bash 73 | kubectl apply -f ./config-management.yaml 74 | ``` 75 | 76 | This should create all the objects. 77 | 78 | 1. Run `kubectl describe configmanagement` and verify that the status is `Healthy`. 79 | 80 | 1. Wait some time for resources to sync. 81 | 82 | 1. Once `cnrm-system` namespace is created, propagate permissions from Config Connector service account to cnrm-system namespace, where we have Config Connector pod 83 | 84 | ```bash 85 | # give cnrm-system namespace permissions to manage GCP 86 | kubectl create secret generic gcp-key --from-file ./key.json --namespace cnrm-system 87 | ``` 88 | This enables Config Connector resources to initialize. 89 | 90 | 1. Wait some time for resources to sync. Verify that service account resources are created: 91 | 92 | ```bash 93 | kubectl describe iamserviceaccount --all-namespaces 94 | ``` 95 | 96 | Last update on both should say: `The resource is up to date`. 97 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/cluster/ns-must-have-cost-center.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: constraints.gatekeeper.sh/v1beta1 2 | kind: K8sRequiredLabels 3 | metadata: 4 | name: ns-must-have-cost-center 5 | spec: 6 | match: 7 | kinds: 8 | - apiGroups: [""] 9 | kinds: ["Namespace"] 10 | parameters: 11 | message: "All namespaces must have an `cost-center` label that points to your cost-center" 12 | labels: 13 | - key: cost-center 14 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/gcp-sql-db.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLDatabase 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: wordpress 7 | spec: 8 | charset: utf8 9 | instanceRef: 10 | name: wp-db 11 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/gcp-sql-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: sql-wp-sa 5 | spec: 6 | displayName: Service Account for WP access 7 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/k8s-external-load-balancer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: wordpress-external 5 | labels: 6 | app: wordpress 7 | spec: 8 | type: LoadBalancer 9 | ports: 10 | - port: 80 11 | name: web 12 | targetPort: 80 13 | protocol: TCP 14 | selector: 15 | app: wordpress 16 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/k8s-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: wordpress 5 | labels: 6 | app: wordpress 7 | spec: 8 | ports: 9 | - port: 80 10 | name: web 11 | clusterIP: None 12 | selector: 13 | app: wordpress 14 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/k8s-stateful-set.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: wordpress 5 | spec: 6 | selector: 7 | matchLabels: 8 | app: wordpress 9 | serviceName: "wordpress" 10 | replicas: 1 11 | template: 12 | metadata: 13 | labels: 14 | app: wordpress 15 | spec: 16 | terminationGracePeriodSeconds: 30 17 | serviceAccountName: sql-wp-ksa-wi 18 | containers: 19 | - name: wordpress 20 | resources: 21 | limits: 22 | cpu: "200m" 23 | memory: "100Mi" 24 | image: wordpress:5.2.2-apache 25 | imagePullPolicy: IfNotPresent 26 | env: 27 | - name: WORDPRESS_DB_HOST 28 | value: 127.0.0.1:3306 29 | - name: WORDPRESS_DB_USER 30 | valueFrom: 31 | secretKeyRef: 32 | name: wordpress-cloudsql-db-credentials 33 | key: username 34 | - name: WORDPRESS_DB_PASSWORD 35 | valueFrom: 36 | secretKeyRef: 37 | name: wordpress-cloudsql-db-credentials 38 | key: password 39 | ports: 40 | - containerPort: 80 41 | volumeMounts: 42 | - name: wordpress-volume-2 43 | mountPath: /var/www/html 44 | readinessProbe: 45 | httpGet: 46 | path: / 47 | port: 80 48 | initialDelaySeconds: 180 49 | periodSeconds: 10 50 | timeoutSeconds: 10 51 | failureThreshold: 10 52 | successThreshold: 1 53 | livenessProbe: 54 | httpGet: 55 | path: / 56 | port: 80 57 | initialDelaySeconds: 30 58 | periodSeconds: 10 59 | timeoutSeconds: 10 60 | failureThreshold: 20 61 | successThreshold: 1 62 | - name: cloudsql-proxy 63 | resources: 64 | limits: 65 | cpu: "200m" 66 | memory: "100Mi" 67 | image: gcr.io/cloudsql-docker/gce-proxy:1.11 68 | env: 69 | - name: CONNECTION_NAME 70 | valueFrom: 71 | secretKeyRef: 72 | name: wordpress-cloudsql-db-credentials 73 | key: connectionName 74 | - name: PROJECT_ID 75 | valueFrom: 76 | secretKeyRef: 77 | name: wordpress-cloudsql-db-credentials 78 | key: projectId 79 | command: ["/cloud_sql_proxy", 80 | "-instances=$(PROJECT_ID):$(CONNECTION_NAME)=tcp:3306"] 81 | volumeClaimTemplates: 82 | - metadata: 83 | name: wordpress-volume-2 84 | spec: 85 | accessModes: [ "ReadWriteOnce" ] 86 | resources: 87 | requests: 88 | storage: 10Gi 89 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/gcp-sql-instance.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLInstance 3 | metadata: 4 | name: wp-db 5 | spec: 6 | databaseVersion: MYSQL_5_7 7 | region: us-central1 8 | settings: 9 | tier: db-f1-micro 10 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/gcp-sql-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicyMember 3 | metadata: 4 | name: sql-sa-policy 5 | spec: 6 | member: serviceAccount:sql-wp-sa@[PROJECT_ID]-dev.iam.gserviceaccount.com 7 | role: roles/cloudsql.client 8 | resourceRef: 9 | kind: Project 10 | apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 11 | external: [PROJECT_ID] 12 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/gcp-sql-user.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLUser 3 | metadata: 4 | name: wordpress 5 | spec: 6 | instanceRef: 7 | name: wp-db 8 | host: "%" 9 | password: change-me-dev 10 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/gcp-wi-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: sql-wp-sa-wi-policy 5 | spec: 6 | resourceRef: 7 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 8 | kind: IAMServiceAccount 9 | name: sql-wp-sa 10 | bindings: 11 | - role: roles/iam.workloadIdentityUser 12 | members: 13 | - serviceAccount:[PROJECT_ID].svc.id.goog[wp-dev/sql-wp-ksa-wi] 14 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/k8s-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: sql-wp-ksa-wi 5 | annotations: 6 | iam.gke.io/gcp-service-account: sql-wp-sa@[PROJECT_ID]-dev.iam.gserviceaccount.com 7 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/k8s-sql-db-credentials.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: wordpress-cloudsql-db-credentials 5 | stringData: 6 | projectId: [PROJECT_ID]-dev 7 | username: wordpress 8 | password: change-me-dev 9 | connectionName: us-central1:wp-db 10 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-dev/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | annotations: 5 | cnrm.cloud.google.com/project-id: [PROJECT_ID]-dev 6 | labels: 7 | cost-center: wp 8 | name: wp-dev 9 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/gcp-sql-instance.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLInstance 3 | metadata: 4 | name: wp-db 5 | spec: 6 | databaseVersion: MYSQL_5_7 7 | region: us-central1 8 | settings: 9 | tier: db-g1-small 10 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/gcp-sql-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicyMember 3 | metadata: 4 | name: sql-sa-policy 5 | spec: 6 | member: serviceAccount:sql-wp-sa@[PROJECT_ID]-prod.iam.gserviceaccount.com 7 | role: roles/cloudsql.client 8 | resourceRef: 9 | kind: Project 10 | apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 11 | external: [PROJECT_ID] 12 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/gcp-sql-user.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLUser 3 | metadata: 4 | name: wordpress 5 | spec: 6 | instanceRef: 7 | name: wp-db 8 | host: "%" 9 | password: change-me-prod 10 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/gcp-wi-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: sql-wp-sa-wi-policy 5 | spec: 6 | resourceRef: 7 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 8 | kind: IAMServiceAccount 9 | name: sql-wp-sa 10 | bindings: 11 | - role: roles/iam.workloadIdentityUser 12 | members: 13 | - serviceAccount:[PROJECT_ID].svc.id.goog[wp-prod/sql-wp-ksa-wi] 14 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/k8s-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: sql-wp-ksa-wi 5 | annotations: 6 | iam.gke.io/gcp-service-account: sql-wp-sa@[PROJECT_ID]-prod.iam.gserviceaccount.com 7 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/k8s-sql-db-credentials.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: wordpress-cloudsql-db-credentials 5 | stringData: 6 | projectId: [PROJECT_ID]-prod 7 | username: wordpress 8 | password: change-me-prod 9 | connectionName: us-central1:wp-db 10 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/namespaces/online/wp/wp-prod/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | annotations: 5 | cnrm.cloud.google.com/project-id: [PROJECT_ID]-prod 6 | labels: 7 | cost-center: wp 8 | name: wp-prod 9 | -------------------------------------------------------------------------------- /src/wp-acm/acm-root/system/repo.yaml: -------------------------------------------------------------------------------- 1 | kind: Repo 2 | apiVersion: configmanagement.gke.io/v1 3 | metadata: 4 | name: repo 5 | spec: 6 | version: "1.0.0" 7 | -------------------------------------------------------------------------------- /src/wp-acm/config-management.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: configmanagement.gke.io/v1 2 | kind: ConfigManagement 3 | metadata: 4 | name: config-management 5 | spec: 6 | clusterName: cluster-1 7 | git: 8 | syncRepo: https://github.com/[your-git-user-name]/gcp-kcc-samples.git 9 | syncBranch: master 10 | secretType: none 11 | policyDir: "src/wp-acm/acm-root" 12 | configConnector: 13 | enabled: true 14 | policyController: 15 | enabled: true 16 | -------------------------------------------------------------------------------- /src/wp-simple/README.md: -------------------------------------------------------------------------------- 1 | # WordPress on K8s + GCP CloudSQL Setup 2 | 3 | This example shows how you can deploy WordPress to your Kubernetes cluster, backed by Google CloudSQL database. Once Kubernetes Config Connector is enabled, the whole installation takes a single kubectl command to run. 4 | 5 | One limitation of Config Connector today is that sql_user resource is referencing credentials in clear text. Once secretRef's are supported for sensitive fields, it will be possible to reference database credentials from Kubernetes secrets. 6 | 7 | 1. [Provision project, cluster and Config Connector](../../provision.md) 8 | 1. Deploy: 9 | 10 | ```bash 11 | kubectl apply -f resources/ 12 | ``` 13 | 1. Run one additonal temporary step required, as this sample is using cnrm-system key. This will be soon replaced by using service account object created by Config Connector. 14 | 15 | ```bash 16 | kubectl create secret generic gcp-key --from-file ./key.json 17 | ``` 18 | 19 | ## Clean up: 20 | ``` bash 21 | kubectl delete -f resources/ 22 | kubectl delete pvc wordpress-volume-wordpress-0 23 | ``` 24 | -------------------------------------------------------------------------------- /src/wp-simple/resources/external-load-balancer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | namespace: default 5 | name: wordpress-external 6 | labels: 7 | app: wordpress 8 | spec: 9 | type: LoadBalancer 10 | ports: 11 | - port: 80 12 | name: web 13 | targetPort: 80 14 | protocol: TCP 15 | selector: 16 | app: wordpress 17 | -------------------------------------------------------------------------------- /src/wp-simple/resources/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | namespace: default 5 | name: wordpress 6 | labels: 7 | app: wordpress 8 | spec: 9 | ports: 10 | - port: 80 11 | name: web 12 | clusterIP: None 13 | selector: 14 | app: wordpress 15 | -------------------------------------------------------------------------------- /src/wp-simple/resources/sql-db-credentials.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: wordpress-cloudsql-db-credentials 5 | stringData: 6 | projectId: [PROJECT_ID] 7 | username: wordpress 8 | password: change-me 9 | connectionName: us-central1:wp-db 10 | -------------------------------------------------------------------------------- /src/wp-simple/resources/sql-db.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLDatabase 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: wordpress 7 | spec: 8 | charset: utf8 9 | instanceRef: 10 | name: wp-db 11 | -------------------------------------------------------------------------------- /src/wp-simple/resources/sql-instance.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLInstance 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: wp-db 7 | spec: 8 | databaseVersion: MYSQL_5_7 9 | region: us-central1 10 | settings: 11 | tier: db-f1-micro 12 | -------------------------------------------------------------------------------- /src/wp-simple/resources/sql-user.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLUser 3 | metadata: 4 | name: wordpress 5 | spec: 6 | instanceRef: 7 | name: wp-db 8 | host: "%" 9 | password: change-me 10 | -------------------------------------------------------------------------------- /src/wp-simple/resources/stateful-set.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | namespace: default 5 | name: wordpress 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: wordpress 10 | serviceName: "wordpress" 11 | replicas: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: wordpress 16 | spec: 17 | terminationGracePeriodSeconds: 30 18 | containers: 19 | - name: wordpress 20 | image: wordpress:4.9.8-apache 21 | imagePullPolicy: IfNotPresent 22 | env: 23 | - name: WORDPRESS_DB_HOST 24 | value: 127.0.0.1:3306 25 | - name: WORDPRESS_DB_USER 26 | valueFrom: 27 | secretKeyRef: 28 | name: wordpress-cloudsql-db-credentials 29 | key: username 30 | - name: WORDPRESS_DB_PASSWORD 31 | valueFrom: 32 | secretKeyRef: 33 | name: wordpress-cloudsql-db-credentials 34 | key: password 35 | ports: 36 | - containerPort: 80 37 | volumeMounts: 38 | - name: wordpress-volume 39 | mountPath: /var/www/html 40 | readinessProbe: 41 | httpGet: 42 | path: / 43 | port: 80 44 | initialDelaySeconds: 180 45 | periodSeconds: 10 46 | timeoutSeconds: 10 47 | failureThreshold: 10 48 | successThreshold: 1 49 | livenessProbe: 50 | httpGet: 51 | path: / 52 | port: 80 53 | initialDelaySeconds: 30 54 | periodSeconds: 10 55 | timeoutSeconds: 10 56 | failureThreshold: 20 57 | successThreshold: 1 58 | - name: cloudsql-proxy 59 | image: gcr.io/cloudsql-docker/gce-proxy:1.11 60 | env: 61 | - name: CONNECTION_NAME 62 | valueFrom: 63 | secretKeyRef: 64 | name: wordpress-cloudsql-db-credentials 65 | key: connectionName 66 | - name: PROJECT_ID 67 | valueFrom: 68 | secretKeyRef: 69 | name: wordpress-cloudsql-db-credentials 70 | key: projectId 71 | command: ["/cloud_sql_proxy", 72 | "-instances=$(PROJECT_ID):$(CONNECTION_NAME)=tcp:3306", 73 | "-credential_file=/secrets/cloudsql/key.json"] 74 | volumeMounts: 75 | - name: gcp-key 76 | mountPath: /secrets/cloudsql 77 | readOnly: true 78 | volumes: 79 | - name: gcp-key 80 | secret: 81 | secretName: gcp-key 82 | volumeClaimTemplates: 83 | - metadata: 84 | name: wordpress-volume 85 | spec: 86 | accessModes: [ "ReadWriteOnce" ] 87 | resources: 88 | requests: 89 | storage: 100Gi 90 | -------------------------------------------------------------------------------- /src/wp-wi/README.md: -------------------------------------------------------------------------------- 1 | # WordPress on K8s + GCP CloudSQL + Workload Identity Setup 2 | 3 | This extends the previous example by enabling Workload Identity integration. This requires 4 additional resources: 4 | * [Google service account (GSA)](resources/gcp-sql-service-account.yaml) 5 | * [Sqlclient permission for GSA](deploy.sh). Currently this step is done via gcloud command, however soon it will be possible to configure individual binding declaratively. 6 | * [Kubernetes service account (KSA)](resources/k8s-service-account.yaml) annotated with GSA 7 | * [Workload identity permission for GSA](resources/gcp-wi-policy.yaml) that links GSA and KSA 8 | 9 | In this sample there's no longer needed to mount keys in the [pod configuration](resources/stateful-set.yaml) as SQL client permissions are propagated through Kubernetes service account. Note: don't forget serviceAccountName field in pod config. 10 | 11 | 1. [Provision project, cluster and Config Connector](../../provision.md) 12 | 1. Deploy: 13 | 14 | ```bash 15 | kubectl apply -f resources/ 16 | ``` 17 | 18 | 1. Wait for sql instance to be ready 19 | ```bash 20 | # Note that you can wait on the proxy resources too 21 | kubectl wait --for=condition=Ready sqlinstance/wp-db --timeout=30m 22 | kubectl wait --for=condition=Ready sqluser/wordpress --timeout=30m 23 | 24 | # But ultimately you need to wait on the pod to be created 25 | kubectl wait --for=condition=Ready pods/wordpress-0 --timeout=30m 26 | ``` 27 | 28 | ## Enable GateKeeper: 29 | 30 | As an additional extension, this example demonstrates the use of gatekeeper. First it applies the release version of gatekeeper, then applies constraint template. 31 | 32 | ```bash 33 | kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper/master/deploy/gatekeeper.yaml 34 | kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper/master/demo/agilebank/templates/k8scontainterlimits_template.yaml 35 | ``` 36 | 37 | ## Clean up: 38 | ``` bash 39 | kubectl delete -f resources/ 40 | kubectl delete pvc wordpress-volume-2-wordpress-0 41 | bash undeploy.sh 42 | ``` 43 | -------------------------------------------------------------------------------- /src/wp-wi/constraints/containers_must_be_limited.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: constraints.gatekeeper.sh/v1beta1 2 | kind: K8sContainerLimits 3 | metadata: 4 | name: container-must-have-limits 5 | spec: 6 | match: 7 | kinds: 8 | - apiGroups: [""] 9 | kinds: ["Pod"] 10 | parameters: 11 | cpu: "200m" 12 | memory: "1Gi" 13 | -------------------------------------------------------------------------------- /src/wp-wi/resources/gcp-sql-db.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLDatabase 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: wordpress 7 | spec: 8 | charset: utf8 9 | instanceRef: 10 | name: wp-db 11 | -------------------------------------------------------------------------------- /src/wp-wi/resources/gcp-sql-instance.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLInstance 3 | metadata: 4 | labels: 5 | label-one: "value-one" 6 | name: wp-db 7 | spec: 8 | databaseVersion: MYSQL_5_7 9 | region: us-central1 10 | settings: 11 | tier: db-f1-micro 12 | -------------------------------------------------------------------------------- /src/wp-wi/resources/gcp-sql-policy-member.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicyMember 3 | metadata: 4 | name: sql-sa-policy 5 | spec: 6 | member: serviceAccount:sql-wp-sa@[PROJECT_ID].iam.gserviceaccount.com 7 | role: roles/cloudsql.client 8 | resourceRef: 9 | kind: Project 10 | apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 11 | external: [PROJECT_ID] 12 | -------------------------------------------------------------------------------- /src/wp-wi/resources/gcp-sql-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMServiceAccount 3 | metadata: 4 | name: sql-wp-sa 5 | spec: 6 | displayName: Service Account for WP access 7 | -------------------------------------------------------------------------------- /src/wp-wi/resources/gcp-sql-user.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: sql.cnrm.cloud.google.com/v1beta1 2 | kind: SQLUser 3 | metadata: 4 | name: wordpress 5 | spec: 6 | instanceRef: 7 | name: wp-db 8 | host: "%" 9 | password: 10 | valueFrom: 11 | secretKeyRef: 12 | name: wordpress-cloudsql-db-credentials 13 | key: password 14 | -------------------------------------------------------------------------------- /src/wp-wi/resources/gcp-wi-policy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 2 | kind: IAMPolicy 3 | metadata: 4 | name: sql-wp-sa-wi-policy 5 | spec: 6 | resourceRef: 7 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 8 | kind: IAMServiceAccount 9 | name: sql-wp-sa 10 | bindings: 11 | - role: roles/iam.workloadIdentityUser 12 | members: 13 | - serviceAccount:[PROJECT_ID].svc.id.goog[default/sql-wp-ksa-wi] 14 | -------------------------------------------------------------------------------- /src/wp-wi/resources/k8s-external-load-balancer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | namespace: default 5 | name: wordpress-external 6 | labels: 7 | app: wordpress 8 | spec: 9 | type: LoadBalancer 10 | ports: 11 | - port: 80 12 | name: web 13 | targetPort: 80 14 | protocol: TCP 15 | selector: 16 | app: wordpress 17 | -------------------------------------------------------------------------------- /src/wp-wi/resources/k8s-service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: sql-wp-ksa-wi 5 | annotations: 6 | iam.gke.io/gcp-service-account: sql-wp-sa@[PROJECT_ID].iam.gserviceaccount.com 7 | -------------------------------------------------------------------------------- /src/wp-wi/resources/k8s-service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | namespace: default 5 | name: wordpress 6 | labels: 7 | app: wordpress 8 | spec: 9 | ports: 10 | - port: 80 11 | name: web 12 | clusterIP: None 13 | selector: 14 | app: wordpress 15 | -------------------------------------------------------------------------------- /src/wp-wi/resources/k8s-sql-db-credentials.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: wordpress-cloudsql-db-credentials 5 | stringData: 6 | projectId: [PROJECT_ID] 7 | username: wordpress 8 | password: change-me 9 | connectionName: us-central1:wp-db 10 | -------------------------------------------------------------------------------- /src/wp-wi/resources/k8s-stateful-set.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | namespace: default 5 | name: wordpress 6 | spec: 7 | selector: 8 | matchLabels: 9 | app: wordpress 10 | serviceName: "wordpress" 11 | replicas: 1 12 | template: 13 | metadata: 14 | labels: 15 | app: wordpress 16 | spec: 17 | terminationGracePeriodSeconds: 30 18 | serviceAccountName: sql-wp-ksa-wi 19 | containers: 20 | - name: wordpress 21 | resources: 22 | limits: 23 | cpu: "200m" 24 | memory: "100Mi" 25 | image: wordpress:5.2.2-apache 26 | imagePullPolicy: IfNotPresent 27 | env: 28 | - name: WORDPRESS_DB_HOST 29 | value: 127.0.0.1:3306 30 | - name: WORDPRESS_DB_USER 31 | valueFrom: 32 | secretKeyRef: 33 | name: wordpress-cloudsql-db-credentials 34 | key: username 35 | - name: WORDPRESS_DB_PASSWORD 36 | valueFrom: 37 | secretKeyRef: 38 | name: wordpress-cloudsql-db-credentials 39 | key: password 40 | ports: 41 | - containerPort: 80 42 | volumeMounts: 43 | - name: wordpress-volume-2 44 | mountPath: /var/www/html 45 | readinessProbe: 46 | httpGet: 47 | path: / 48 | port: 80 49 | initialDelaySeconds: 180 50 | periodSeconds: 10 51 | timeoutSeconds: 10 52 | failureThreshold: 10 53 | successThreshold: 1 54 | livenessProbe: 55 | httpGet: 56 | path: / 57 | port: 80 58 | initialDelaySeconds: 30 59 | periodSeconds: 10 60 | timeoutSeconds: 10 61 | failureThreshold: 20 62 | successThreshold: 1 63 | - name: cloudsql-proxy 64 | resources: 65 | limits: 66 | cpu: "200m" 67 | memory: "100Mi" 68 | image: gcr.io/cloudsql-docker/gce-proxy:1.11 69 | env: 70 | - name: CONNECTION_NAME 71 | valueFrom: 72 | secretKeyRef: 73 | name: wordpress-cloudsql-db-credentials 74 | key: connectionName 75 | - name: PROJECT_ID 76 | valueFrom: 77 | secretKeyRef: 78 | name: wordpress-cloudsql-db-credentials 79 | key: projectId 80 | command: ["/cloud_sql_proxy", 81 | "-instances=$(PROJECT_ID):$(CONNECTION_NAME)=tcp:3306"] 82 | volumeClaimTemplates: 83 | - metadata: 84 | name: wordpress-volume-2 85 | spec: 86 | accessModes: [ "ReadWriteOnce" ] 87 | resources: 88 | requests: 89 | storage: 10Gi 90 | --------------------------------------------------------------------------------