├── .github └── workflows │ └── release.yaml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── analytics ├── Dockerfile ├── go.mod ├── go.sum ├── manifests │ ├── deployment.yaml │ ├── ingress.yaml │ ├── loadbalancer.yaml │ ├── mc.yaml │ └── nodeport.yaml ├── server.go ├── skaffold.yaml └── testRequest.json ├── assets ├── demo.gif ├── invalid-function-on-destroy.png ├── logo-2048.png ├── logo-256.png ├── logo-512.png └── private-cluster.svg ├── cli ├── LICENSE ├── Makefile ├── README.md ├── cmd │ ├── apply.go │ ├── connect.go │ ├── delete.go │ ├── init.go │ └── root.go ├── go.mod ├── go.sum ├── main.go └── pkg │ ├── analytics │ └── client.go │ ├── cli_init │ ├── clusters │ │ └── main.tf │ ├── fleet │ │ └── main.tf │ ├── init.go │ ├── network │ │ └── main.tf │ ├── samples │ │ ├── default-config.yaml │ │ ├── fleet-full.yaml │ │ ├── fleet-shared-vpc.yaml │ │ ├── module-bypass-multi-clusters-networking-acm-standalone-vpc.yaml │ │ ├── multi-clusters-acm-shared-vpc.yaml │ │ ├── multi-clusters-acm-standalone-vpc.yaml │ │ ├── multi-clusters-networking-acm-shared-vpc.yaml │ │ ├── multi-clusters-networking-acm-standalone-vpc.yaml │ │ ├── multi-clusters-shared-vpc.yaml │ │ └── multi-clusters-standalone-vpc.yaml │ └── templates │ │ ├── cluster_config.tmpl │ │ ├── terraform.tfvars.tmpl │ │ └── terraform_backend.tf.tmpl │ ├── config │ ├── config.go │ ├── tf_state.go │ └── tfvars_generator.go │ └── lifecycle │ ├── get_credentials.go │ ├── tf_delete.go │ └── tf_deploy.go ├── demos └── fleets │ ├── README.md │ ├── config.yaml │ ├── default-configs │ ├── asm-gcp-gateways │ │ ├── backendconfig.yaml │ │ ├── cloud-armor-backendpolicy.yaml │ │ ├── default-httproute-redirect.yaml │ │ ├── default-httproute.yaml │ │ ├── frontend-gateway.yaml │ │ └── ingress-gateway-healthcheck.yaml │ ├── asm-ingress-gateway │ │ ├── namespace.yaml │ │ ├── pubic-gw-deployment.yaml │ │ ├── public-gw.yaml │ │ ├── role.yaml │ │ ├── service.yaml │ │ └── svc_export.yaml │ ├── buffer │ │ ├── capacity-res-deployment.yaml │ │ └── priorityclasses.yaml │ ├── observability │ │ ├── cloudops-metrics-adapter.yaml │ │ └── prom-frontend.yaml │ ├── roles │ │ ├── gateway-cluster-role.yaml │ │ ├── istio-cluster-role.yaml │ │ └── monitoring-cluster-role.yaml │ ├── tenants │ │ ├── llm │ │ │ ├── backendconfig.yaml │ │ │ ├── export-global.yaml │ │ │ ├── gateway.yaml │ │ │ ├── healthcheck.yaml │ │ │ ├── hpa-custom-metrics.yaml │ │ │ ├── inference │ │ │ │ ├── inference-ns-selector.yaml │ │ │ │ └── inference-repo-sync.yaml │ │ │ └── service.yaml │ │ └── whereami │ │ │ ├── backend │ │ │ ├── backend-ns-selector.yaml │ │ │ └── backend-repo-sync.yaml │ │ │ ├── frontend │ │ │ ├── frontend-ns-selector.yaml │ │ │ └── frontend-repo-sync.yaml │ │ │ ├── network-policy.yaml │ │ │ ├── team-resource-quota.yaml │ │ │ └── team-selector.yaml │ └── tools │ │ └── secret-provider-class.yaml │ └── teams │ ├── llm │ └── inference │ │ ├── export-us-central.yaml │ │ ├── export-us-east.yaml │ │ ├── export-us-east4.yaml │ │ ├── hpa-custom-metrics.yaml │ │ ├── monitoring.yaml │ │ ├── service.yaml │ │ ├── tgi-2b-it-1.1-central.yaml │ │ ├── tgi-2b-it-1.1-east.yaml │ │ ├── tgi-2b-it-1.1-west.yaml │ │ ├── tgi-2b-it-1.1.yaml │ │ └── tgi-kn.yaml │ └── whereami │ ├── backend │ ├── app.yaml │ ├── whereami-backend-dr.yaml │ └── whereami-backend-vs.yaml │ └── frontend │ ├── app.yaml │ ├── whereami-frontend-dr.yaml │ └── whereami-frontend-vs.yaml ├── docs ├── analytics.md ├── architecture.md ├── building-demos.md ├── configuration.md └── frequently-asked-questions.md ├── terraform └── modules │ ├── clusters │ ├── clusters.tf │ ├── outputs.tf │ ├── provider.tf │ └── variables.tf │ ├── fleet │ ├── admin-cluster.tf │ ├── feature-defaults.tf │ ├── iam.tf │ ├── mcg.tf │ ├── outputs.tf │ ├── provider.tf │ ├── services.tf │ └── variables.tf │ └── network │ ├── main.tf │ ├── network.tf │ ├── provider.tf │ ├── shared_vpc.tf │ └── variables.tf └── test ├── README.md └── e2e ├── cleanup-cloud-run ├── Dockerfile ├── README.md └── delete_old_projects.sh ├── cloudbuild-default-config.yaml └── custom-builder ├── Dockerfile └── README.md /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: publish 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | publish: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 1 15 | - uses: actions/setup-go@v2 16 | with: 17 | go-version: '^1.22.0' 18 | - name: Make all 19 | run: cd cli && make all 20 | - name: Release Darwin 21 | uses: softprops/action-gh-release@v1 22 | with: 23 | files: ./cli/bin/gkekitctl-darwin 24 | - name: Release Linux 25 | uses: softprops/action-gh-release@v1 26 | with: 27 | files: ./cli/bin/gkekitctl-amd64 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # OSX leaves these everywhere on SMB shares 2 | ._* 3 | 4 | # OSX trash 5 | .DS_Store 6 | 7 | # Emacs save files 8 | *~ 9 | \#*\# 10 | .\#* 11 | 12 | # Vim-related files 13 | [._]*.s[a-w][a-z] 14 | [._]s[a-w][a-z] 15 | *.un~ 16 | Session.vim 17 | .netrwhist 18 | 19 | # ignore local CLI binary 20 | cli/gkekitctl 21 | 22 | ### https://raw.github.com/github/gitignore/90f149de451a5433aebd94d02d11b0e28843a1af/Terraform.gitignore 23 | 24 | # Local .terraform directories 25 | **/.terraform/* 26 | 27 | # .tfstate files 28 | *.tfstate 29 | *.tfstate.* 30 | 31 | # kubectl config 32 | scripts/kubeconfig 33 | 34 | # Crash log files 35 | crash.log 36 | 37 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 38 | # .tfvars files are managed as part of configuration and so should be included in 39 | # version control. 40 | # 41 | # example.tfvars 42 | 43 | terraform.tfvars 44 | backend.tf 45 | # gcloud service account credentials 46 | credentials.json 47 | creds/ 48 | # postgres db name 49 | .instance 50 | 51 | # terraform locks 52 | .terraform.lock.hcl 53 | 54 | # local bash command file 55 | _local.sh 56 | 57 | # Ignore cluster_config vars file 58 | cluster_config 59 | github.key 60 | 61 | # Binaries for programs and plugins 62 | *.exe 63 | *.exe~ 64 | *.dll 65 | *.so 66 | *.dylib 67 | 68 | # Test binary, built with `go test -c` 69 | *.test 70 | 71 | # Output of the go coverage tool, specifically when used with LiteIDE 72 | *.out 73 | 74 | # cli binary 75 | gkekitctl 76 | 77 | # Vscode dot files 78 | .vscode/ 79 | 80 | # Cred folder for rbac demo 81 | creds/ 82 | 83 | # Dependency directories (remove the comment below to include it) 84 | # vendor/ 85 | 86 | # ssh 87 | *id_rsa* 88 | 89 | # Cloud Source Repos 90 | *gke-poc-config-sync* 91 | 92 | # Ignore GKE Toolkit init files 93 | cli/cluster_build/* 94 | cli/samples/* 95 | cli/shared_vpc/* 96 | cli/templates/* 97 | 98 | # Ignore yaml files in root of cli 99 | cli/*.yaml 100 | 101 | # Ignore users' kubeconfigs 102 | *kubeconfig* 103 | 104 | .idea/ 105 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Dependencies 7 | 8 | The following dependencies must be installed in Cloud Shell or on the development system: 9 | 10 | * [Terraform >= 0.13](https://www.terraform.io/downloads.html) 11 | * [Google Cloud SDK version >= 325.0.0](https://cloud.google.com/sdk/docs/downloads-versioned-archives) 12 | * [kubectl matching the latest GKE version](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 13 | * bash or bash compatible shell 14 | 15 | ## Contributor License Agreement 16 | 17 | Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. Head over to to see your current agreements on file or to sign a new one. 18 | 19 | You generally only need to submit a CLA once, so if you've already submitted one (even if it was for a different project), you probably don't need to do it again. 20 | 21 | ## Code Submission 22 | 23 | * All community contributions must be linked to an existing issue. If an issue does not exist, please create one descibing the issue resolved by the associated pull request. Instructions on how to link a pull request to an issue can be found in the GitHub following doc - [Linking a pull request to an issue](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue). 24 | 25 | * All submissions, including submissions by project members, will be reviewed before merge. We use GitHub pull requests for this purpose. Consult [GitHub help](https://help.github.com/articles/about-pull-requests/) for more information on using pull requests. All pull requests are linted and integration tests performed to maintain a standard of quality. 26 | 27 | ## Community Guidelines 28 | 29 | This project follows [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GKE PoC Toolkit 2 | 3 | ![release](https://img.shields.io/github/v/release/googlecloudplatform/gke-poc-toolkit) ![stars](https://img.shields.io/github/stars/GoogleCloudPlatform/gke-poc-toolkit) ![license](https://img.shields.io/github/license/GoogleCloudPlatform/gke-poc-toolkit) 4 | 5 | 6 | ![logo](assets/logo-256.png) 7 | 8 | The GKE Proof of Concept (PoC) Toolkit is a demo generator for [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine). 9 | 10 | ![demo-gif](assets/demo.gif) 11 | 12 | ## Quickstart 13 | 14 | 1. **[Create a Google Cloud Project](https://cloud.google.com/resource-manager/docs/creating-managing-projects)** and connect it to an existing Billing account. 15 | 2. **Open a bash-compatible shell** (eg. [Google Cloud Shell](https://cloud.google.com/shell)) and ensure you have the following tools installed: 16 | 17 | * [Google Cloud SDK version >= 325.0.0](https://cloud.google.com/sdk/docs/downloads-versioned-archives) 18 | * * [Terraform >= 0.13](https://www.terraform.io/downloads.html) 19 | * [kubectl](https://kubernetes.io/docs/tasks/tools/) ( >= v1.20) 20 | 21 | 3. **Set your Project ID environment variable and operating system.** 22 | 23 | ```bash 24 | export PROJECT_ID= 25 | export OS="darwin" # choice of darwin or amd64 26 | ``` 27 | 28 | 4. **Set up local authentication to your project.** 29 | 30 | ``` 31 | gcloud config set project $PROJECT_ID 32 | gcloud auth login 33 | gcloud auth application-default login 34 | ``` 35 | 36 | 5. **Download the GKE PoC Toolkit binary.** 37 | 38 | ```shell 39 | mkdir gke-poc-toolkit && cd "$_" 40 | VERSION=$(curl -s https://api.github.com/repos/GoogleCloudPlatform/gke-poc-toolkit/releases/latest | grep browser_download_url | cut -d "/" -f 8 | tail -1) 41 | curl -sLSf -o ./gkekitctl https://github.com/GoogleCloudPlatform/gke-poc-toolkit/releases/download/${VERSION}/gkekitctl-${OS} && chmod +x ./gkekitctl 42 | ``` 43 | 44 | 6. **Initialize the cli:** 45 | ```bash 46 | ./gkekitctl init 47 | ``` 48 | 49 | 7. **Run `gkekitctl apply` to run the Toolkit.** By default, this command sets up a single-cluster GKE environment. ([Configuration here](cli/pkg/cli_init/samples/default-config.yaml)). Enter your project ID when prompted. 50 | 51 | ```shell 52 | ./gkekitctl apply 53 | ``` 54 | ```shell 55 | # expected output 56 | INFO[0000] ☸️ ----- GKE POC TOOLKIT ----- 🛠 57 | INFO[0000] Enter your Google Cloud Project ID: 58 | ``` 59 | 60 | This command takes about **10 minutes** to run; when it completes, you will have a full GKE demo environment ready to explore and deploy applications to. 61 | 62 | ```bash 63 | # Expected output on successful run 64 | Apply complete! Resources: 61 added, 0 changed, 0 destroyed. 65 | time="2022-02-04T21:57:59Z" level=info msg="🔄 Finishing ACM install..." 66 | time="2022-02-04T21:57:59Z" level=info msg="☸️ Generating Kubeconfig..." 67 | time="2022-02-04T21:57:59Z" level=info msg="Clusters Project ID is gpt-e2etest-020422-214428" 68 | time="2022-02-04T21:58:00Z" level=info msg="Connecting to cluster: gke_gpt-e2etest-020422-214428_us-central1_gke-central," 69 | time="2022-02-04T21:58:00Z" level=info msg="✔️ Kubeconfig generated: &{Kind:Config APIVersion:v1 Preferences:{Colors:false Extensions:map[]} Clusters:map[gke_gpt-e2etest-020422-214428_us-central1_gke-central:0xc000844900] AuthInfos:map[gke_gpt-e2etest-020422-214428_us-central1_gke-central:0xc0008a23c0] Contexts:map[gke_gpt-e2etest-020422-214428_us-central1_gke-central:0xc0012bad20] CurrentContext: Extensions:map[]}" 70 | time="2022-02-04T21:58:00Z" level=info msg="☸️ Verifying Kubernetes API access for all clusters..." 71 | time="2022-02-04T21:58:00Z" level=info msg="🌎 5 Namespaces found in cluster=gke_gpt-e2etest-020422-214428_us-central1_gke-central" 72 | ``` 73 | 74 | ## Update 75 | 76 | If you want to update your environment change the config file and re-run the apply command. This is a great way to add or remove clusters. 77 | ```bash 78 | ## Local tf state 79 | ./gkekitctl apply --config <"config file name"> 80 | 81 | ## If you are using remote TF state 82 | ./gkekitctl apply --config <"config file name"> --gkestate <"bucket name used for state file"> --vpcstate <"if using sharevpc, bucket name for shared vpc state file"> 83 | ``` 84 | 85 | ## Clean up 86 | 87 | ```bash 88 | ./gkekitctl delete 89 | ``` 90 | 91 | ## Learn More 92 | 93 | - [🤔 FAQ](/docs/frequently-asked-questions.md) 94 | - [✏️ Configuration](/docs/configuration.md): how to customize your Toolkit environment 95 | - [📦 Building Demos with the Toolkit](/docs/building-demos.md) 96 | - [🗺 Architecture](/docs/architecture.md) 97 | - [📊 Analytics](/docs/analytics.md) 98 | -------------------------------------------------------------------------------- /analytics/Dockerfile: -------------------------------------------------------------------------------- 1 | # Source https://github.com/GoogleCloudPlatform/golang-samples/blob/6c46053696035e0b5d210806f005c43da9bcb6ee/cloudsql/postgres/database-sql/Dockerfile 2 | # us-central1-docker.pkg.dev/gkepoctoolkit/analytics-server/analytics-server:v0.0.1 3 | 4 | FROM golang:1.16-buster as builder 5 | 6 | # Create and change to the app directory. 7 | WORKDIR /app 8 | 9 | # Retrieve application dependencies. 10 | # This allows the container build to reuse cached dependencies. 11 | # Expecting to copy go.mod and if present go.sum. 12 | COPY go.* ./ 13 | RUN go mod download 14 | 15 | # Copy local code to the container image. 16 | COPY . ./ 17 | 18 | # Build the binary. 19 | RUN go build -v -o server 20 | 21 | # Use the official Debian slim image for a lean production container. 22 | # https://hub.docker.com/_/debian 23 | # https://docs.docker.com/develop/develop-images/multistage-build/#use-multi-stage-builds 24 | FROM debian:buster-slim 25 | RUN set -x && apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ 26 | ca-certificates && \ 27 | rm -rf /var/lib/apt/lists/* 28 | 29 | # Copy the binary to the production image from the builder stage. 30 | COPY --from=builder /app/server /app/server 31 | 32 | # Run the web service on container startup. 33 | WORKDIR /app 34 | EXPOSE 8000 35 | CMD ["/app/server"] 36 | -------------------------------------------------------------------------------- /analytics/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/googlecloudplatform/gke-poc-toolkit/analytics 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/gofrs/uuid v4.1.0+incompatible 7 | github.com/gorilla/mux v1.8.0 8 | github.com/jackc/pgx/v4 v4.13.0 9 | github.com/sirupsen/logrus v1.8.1 10 | ) 11 | -------------------------------------------------------------------------------- /analytics/manifests/deployment.yaml: -------------------------------------------------------------------------------- 1 | # Source: https://cloud.google.com/sql/docs/postgres/quickstart-kubernetes-engine#gcloud_3 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: analytics-server 6 | namespace: analytics 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: analytics-server 11 | template: 12 | metadata: 13 | labels: 14 | app: analytics-server 15 | spec: 16 | serviceAccountName: analytics-ksa 17 | containers: 18 | - name: analytics-server 19 | image: analytics-server 20 | ports: 21 | - containerPort: 8000 22 | readinessProbe: 23 | initialDelaySeconds: 10 24 | httpGet: 25 | path: "/_healthz" 26 | port: 8000 27 | livenessProbe: 28 | initialDelaySeconds: 10 29 | httpGet: 30 | path: "/_healthz" 31 | port: 8000 32 | env: 33 | - name: PORT 34 | value: "8000" 35 | - name: DB_HOST 36 | value: "127.0.0.1" 37 | - name: DB_PORT 38 | value: "5432" 39 | - name: DB_USER 40 | valueFrom: 41 | secretKeyRef: 42 | name: cloudsql-user 43 | key: username 44 | - name: DB_PASS 45 | valueFrom: 46 | secretKeyRef: 47 | name: cloudsql-user 48 | key: password 49 | - name: DB_NAME 50 | valueFrom: 51 | secretKeyRef: 52 | name: cloudsql-user 53 | key: database 54 | - name: cloud-sql-proxy 55 | image: gcr.io/cloudsql-docker/gce-proxy:1.27.0 56 | command: 57 | - "/cloud_sql_proxy" 58 | - "-instances=gkepoctoolkit:us-central1:gkepoc-metrics=tcp:5432" 59 | securityContext: 60 | runAsNonRoot: true 61 | resources: 62 | requests: 63 | memory: "2Gi" 64 | cpu: "1" 65 | -------------------------------------------------------------------------------- /analytics/manifests/ingress.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.k8s.io/v1 2 | kind: Ingress 3 | metadata: 4 | name: managed-cert-ingress 5 | namespace: analytics 6 | annotations: 7 | kubernetes.io/ingress.global-static-ip-name: analytics3 8 | networking.gke.io/managed-certificates: managed-cert 9 | kubernetes.io/ingress.class: "gce" 10 | # kubernetes.io/ingress.allow-http: "false" 11 | spec: 12 | defaultBackend: 13 | service: 14 | name: analytics-server 15 | port: 16 | number: 80 17 | -------------------------------------------------------------------------------- /analytics/manifests/loadbalancer.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: analytics-server-lb 5 | namespace: analytics 6 | spec: 7 | selector: 8 | app: analytics-server 9 | type: LoadBalancer 10 | ports: 11 | - port: 80 12 | targetPort: 8000 13 | -------------------------------------------------------------------------------- /analytics/manifests/mc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.gke.io/v1 2 | kind: ManagedCertificate 3 | metadata: 4 | name: managed-cert 5 | namespace: analytics 6 | spec: 7 | domains: 8 | - analytics.gkepoctoolkit.dev 9 | -------------------------------------------------------------------------------- /analytics/manifests/nodeport.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: analytics-server 5 | namespace: analytics 6 | spec: 7 | selector: 8 | app: analytics-server 9 | type: NodePort 10 | ports: 11 | - protocol: TCP 12 | port: 80 13 | targetPort: 8000 14 | -------------------------------------------------------------------------------- /analytics/skaffold.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: skaffold/v2beta18 2 | kind: Config 3 | metadata: 4 | name: analytics 5 | build: 6 | artifacts: 7 | - image: analytics-server 8 | context: . 9 | deploy: 10 | kubectl: 11 | manifests: 12 | - manifests/deployment.yaml 13 | -------------------------------------------------------------------------------- /analytics/testRequest.json: -------------------------------------------------------------------------------- 1 | { 2 | "create_id": "12345", 3 | "cluster_id": "12345", 4 | "timestamp": "2021-11-12T11:45:26.371Z", 5 | "os": "darwin_x64", 6 | "terraformState": "local", 7 | "region": "us-central1", 8 | "enableWorkloadIdentity": false, 9 | "enablePreemptibleNodepool": false, 10 | "defaultNodepoolOS": "cos", 11 | "privateEndpoint": false, 12 | "enableConfigSync": true, 13 | "enablePolicyController": true, 14 | "vpcType": "standalone", 15 | "clusterIndex": 0, 16 | "clusterNumNodes": 3, 17 | "clusterType": "public", 18 | "clusterMachineType": "e2-standard-4", 19 | "clusterRegion": "us-central1", 20 | "clusterZone": "us-central1-b" 21 | } 22 | -------------------------------------------------------------------------------- /assets/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-poc-toolkit/2cff69f938487e19f3d860e6d53b2173f5a91f8b/assets/demo.gif -------------------------------------------------------------------------------- /assets/invalid-function-on-destroy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-poc-toolkit/2cff69f938487e19f3d860e6d53b2173f5a91f8b/assets/invalid-function-on-destroy.png -------------------------------------------------------------------------------- /assets/logo-2048.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-poc-toolkit/2cff69f938487e19f3d860e6d53b2173f5a91f8b/assets/logo-2048.png -------------------------------------------------------------------------------- /assets/logo-256.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-poc-toolkit/2cff69f938487e19f3d860e6d53b2173f5a91f8b/assets/logo-256.png -------------------------------------------------------------------------------- /assets/logo-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-poc-toolkit/2cff69f938487e19f3d860e6d53b2173f5a91f8b/assets/logo-512.png -------------------------------------------------------------------------------- /cli/Makefile: -------------------------------------------------------------------------------- 1 | Version := $(shell git describe --tags --dirty) 2 | # Version := "dev" 3 | GitCommit := $(shell git rev-parse HEAD) 4 | LDFLAGS := "-s -w -X gkekitctl/cmd.Version=$(Version) -X gkekitctl/cmd.GitCommit=$(GitCommit)" 5 | 6 | .PHONY: all 7 | all: gofmt dist 8 | 9 | # .PHONY: test 10 | # test: 11 | # go test -v ./... 12 | 13 | .PHONY: gofmt 14 | gofmt: 15 | @test -z $(shell gofmt -l ./ | tee /dev/stderr) || (echo "[WARN] Fix formatting issues with 'make fmt'" && exit 1) 16 | 17 | .PHONY: dist 18 | dist: 19 | mkdir -p bin/ 20 | CGO_ENABLED=0 GOOS=linux go build -mod=mod -a -ldflags $(LDFLAGS) -installsuffix cgo -o bin/gkekitctl-amd64 21 | CGO_ENABLED=0 GOOS=darwin go build -mod=mod -a -ldflags $(LDFLAGS) -installsuffix cgo -o bin/gkekitctl-darwin 22 | GOARM=7 GOARCH=arm CGO_ENABLED=0 GOOS=linux go build -mod=vendor -a -ldflags $(LDFLAGS) -installsuffix cgo -o bin/gkekitctl-arm -mod=mod 23 | GOARCH=arm64 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -a -ldflags $(LDFLAGS) -installsuffix cgo -o bin/gkekitctl-arm64 -mod=mod 24 | # GOOS=windows CGO_ENABLED=0 go build -mod=vendor -a -ldflags $(LDFLAGS) -installsuffix cgo -o bin/gkekitctl.exe -------------------------------------------------------------------------------- /cli/README.md: -------------------------------------------------------------------------------- 1 | # GKE POC Toolkit - Command Line Interface 2 | 3 | This subdirectory contains code and sample config for the GKE POC Toolkit CLI. This Golang command-line tool wraps the Terraform scripts used to set up a GKE environment consisting of one or more clusters. 4 | 5 | 6 | ## Configuration 7 | 8 | | Name | Type | Valid Values | Default Value | 9 | | --------------------------- | ----------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------ | 10 | | `region` | string | any Google Cloud region | `us-central1` | 11 | | `terraformState` | string | `local,cloud` | `local` | 12 | | `configSync` | bool | | | 13 | | `clustersProjectId` | string | | (you are prompted for your GCP Project ID by the tool) | 14 | | `governanceProjectId` | string | | (you are prompted for your GCP Project ID by the tool) | 15 | | `privateEndpoint` | string | | false | 16 | | `enableWorkloadIdentity` | bool | | | 17 | | `enableWindowsNodepool` | bool | | | 18 | | `enablePreemptibleNodepool` | bool | | | 19 | | `defaultNodepoolOS` | string | https://cloud.google.com/kubernetes-engine/docs/concepts/node-images#available_node_images | `cos` | 20 | | `vpcConfig.vpcName` | string | any non-empty string | `"default"` | 21 | | `vpcConfig.vpcType` | string | `shared,standalone` | `standalone` | 22 | | `vpcConfig.vpcProjectId` | string | | (you are prompted for your GCP Project ID by the tool) | 23 | | `clustersConfig` | `[]ClusterConfig` | `ClusterConfig` values below! | | 24 | 25 | ### ClusterConfig 26 | 27 | 28 | | Name | Type | Valid Values | Default Value | 29 | | ------------- | ------ | --------------------------------------------------- | ----------------------------------------------------- | 30 | | `projectId` | string | | (you are prompted for your GCP Project ID on startup) | 31 | | `nodeSize` | int | 1-100 | 3 | 32 | | `machineType` | string | https://cloud.google.com/compute/docs/machine-types | `e2-standard-4` | 33 | | `clusterType` | string | `private,public` | `public` | 34 | | `authCIDR` | string | | | 35 | | `region` | string | https://cloud.google.com/compute/docs/regions-zones | | 36 | | `zone` | string | https://cloud.google.com/compute/docs/regions-zones | | 37 | | `subnetName` | string | any non-empty string | `"default"` | 38 | | `podCIDRName` | string | | | 39 | | `svcCIDRName` | string | | | 40 | -------------------------------------------------------------------------------- /cli/cmd/apply.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package cmd 17 | 18 | import ( 19 | "gkekitctl/pkg/analytics" 20 | "gkekitctl/pkg/config" 21 | "gkekitctl/pkg/lifecycle" 22 | 23 | log "github.com/sirupsen/logrus" 24 | 25 | "github.com/spf13/cobra" 26 | ) 27 | 28 | // createCmd represents the create command 29 | var applyCmd = &cobra.Command{ 30 | Use: "apply", 31 | Short: "Create or Update GKE Demo Environment", 32 | Example: ` gkekitctl apply 33 | gkekitctl apply --config 34 | gkekitctl apply --config -g -v -f `, 35 | Run: func(cmd *cobra.Command, args []string) { 36 | bucketNameNetwork, err := cmd.Flags().GetString("vpcstate") 37 | if err != nil { 38 | log.Errorf("🚨Failed to get bucketNameSharedVPC value from flag: %s", err) 39 | } 40 | if bucketNameNetwork != "" { 41 | log.Infof("✅ Terraform state storage bucket for shared VPC is %s", bucketNameNetwork) 42 | } 43 | bucketNameFleet, err := cmd.Flags().GetString("fleetstate") 44 | if err != nil { 45 | log.Errorf("🚨Failed to get bucketNameFleet value from flag: %s", err) 46 | } 47 | if bucketNameFleet != "" { 48 | log.Infof("✅ Terraform state storage bucket for Fleet is %s", bucketNameFleet) 49 | } 50 | bucketNameClusters, err := cmd.Flags().GetString("gkestate") 51 | if err != nil { 52 | log.Errorf("🚨Failed to get bucketNameClusters value from flag: %s", err) 53 | } 54 | if bucketNameClusters != "" { 55 | log.Infof("✅ Terraform state storage bucket for clusters is %s", bucketNameClusters) 56 | } 57 | 58 | log.Info("👟 Started config validation...") 59 | conf := config.InitConf(cfgFile) 60 | 61 | // Send user analytics - async 62 | if conf.SendAnalytics { 63 | go analytics.SendAnalytics(conf, Version, GitCommit) 64 | } 65 | 66 | log.Info("👟 Started generating TFVars...") 67 | config.GenerateTfvars(conf) 68 | 69 | log.Info("👟 Started configuring TF State...") 70 | err = config.CheckTfStateType(conf, bucketNameNetwork, bucketNameFleet, bucketNameClusters) 71 | 72 | if err != nil { 73 | log.Errorf("🚨 Failed setting up TF state: %s", err) 74 | } else { 75 | log.Info("✅ TF state configured successfully.") 76 | } 77 | 78 | lifecycle.InitTF("network") 79 | lifecycle.ApplyTF("network") 80 | lifecycle.InitTF("fleet") 81 | lifecycle.ApplyTF("fleet") 82 | lifecycle.InitTF("clusters") 83 | lifecycle.ApplyTF("clusters") 84 | 85 | // Authenticate Kubernetes client-go to all clusters 86 | log.Info("☸️ Generating Kubeconfig...") 87 | kc, err := lifecycle.GenerateKubeConfig(conf.FleetProjectID) 88 | if err != nil { 89 | log.Errorf("🚨 Failed to generate kube config: %s", err) 90 | } else { 91 | log.Infof("✅ Kubeconfig generated: %+v", kc) 92 | } 93 | }, 94 | } 95 | 96 | func init() { 97 | var bucketNameClusters string 98 | var bucketNameNetwork string 99 | var bucketNameFleet string 100 | 101 | rootCmd.AddCommand(applyCmd) 102 | applyCmd.Flags().StringVarP(&bucketNameClusters, "gkestate", "g", "", "GKE Tf State bucket name") 103 | applyCmd.Flags().StringVarP(&bucketNameNetwork, "vpcstate", "v", "", "Network Tf State bucket name") 104 | applyCmd.Flags().StringVarP(&bucketNameFleet, "fleetstate", "f", "", "Fleet Tf State bucket name") 105 | } 106 | -------------------------------------------------------------------------------- /cli/cmd/connect.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package cmd 17 | 18 | import ( 19 | "gkekitctl/pkg/lifecycle" 20 | 21 | log "github.com/sirupsen/logrus" 22 | 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | // connectCmd represents the connect command 27 | var connectCmd = &cobra.Command{ 28 | Use: "connect", 29 | Short: "connect for all clusters in GKE Demo Environment", 30 | Example: ` gkekitctl connect -p `, 31 | 32 | Run: func(cmd *cobra.Command, args []string) { 33 | log.Println("Getting credentials...") 34 | // conf := config.InitConf(cfgFile) 35 | 36 | log.Info("☸️ Generating Kubeconfig...") 37 | kc, err := lifecycle.GenerateKubeConfig(fleetProjectId) 38 | if err != nil { 39 | log.Errorf("🚨 Failed to generate kube config: %s", err) 40 | } else { 41 | log.Infof("✅ Kubeconfig generated: %+v", kc) 42 | } 43 | }, 44 | } 45 | 46 | func init() { 47 | rootCmd.AddCommand(connectCmd) 48 | } 49 | -------------------------------------------------------------------------------- /cli/cmd/delete.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package cmd 17 | 18 | import ( 19 | "gkekitctl/pkg/lifecycle" 20 | 21 | log "github.com/sirupsen/logrus" 22 | 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | // deleteCmd represents the delete command 27 | var deleteCmd = &cobra.Command{ 28 | Use: "delete", 29 | Short: "delete GKE Demo Environment", 30 | Example: ` gkekitctl delete`, 31 | 32 | Run: func(cmd *cobra.Command, args []string) { 33 | log.Println("Starting delete...") 34 | lifecycle.DestroyTF("clusters") 35 | lifecycle.DestroyTF("fleet") 36 | lifecycle.DestroyTF("network") 37 | 38 | }, 39 | } 40 | 41 | func init() { 42 | rootCmd.AddCommand(deleteCmd) 43 | } 44 | -------------------------------------------------------------------------------- /cli/cmd/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package cmd 17 | 18 | import ( 19 | "gkekitctl/pkg/cli_init" 20 | 21 | log "github.com/sirupsen/logrus" 22 | "github.com/spf13/cobra" 23 | ) 24 | 25 | // createCmd represents the create command 26 | var initCmd = &cobra.Command{ 27 | Use: "init", 28 | Short: "Initialize local environment for the cli", 29 | Example: ` gkekitctl init`, 30 | Run: func(cmd *cobra.Command, args []string) { 31 | folders := []string{"samples", "templates", "clusters", "network", "fleet"} 32 | err := cli_init.InitFlatFiles(folders) 33 | if err != nil { 34 | log.Errorf("🚨 Failed to initialize gkekitctl: %v", err) 35 | } 36 | // Run opt-in analytics prompt 37 | err = cli_init.OptInAnalytics() 38 | if err != nil { 39 | log.Warnf("Failed to initialize user analytics: %v", err) 40 | } 41 | log.Info("🎉 GKE PoC Toolkit has been initialized! You're ready to run gkekitctl create.") 42 | }, 43 | } 44 | 45 | func init() { 46 | rootCmd.AddCommand(initCmd) 47 | } 48 | -------------------------------------------------------------------------------- /cli/cmd/root.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package cmd 17 | 18 | import ( 19 | log "github.com/sirupsen/logrus" 20 | 21 | "github.com/spf13/cobra" 22 | "github.com/spf13/viper" 23 | ) 24 | 25 | var cfgFile string 26 | var fleetProjectId string 27 | 28 | var ( 29 | Version string 30 | GitCommit string 31 | ) 32 | 33 | // rootCmd represents the base command when called without any subcommands 34 | var rootCmd = &cobra.Command{ 35 | Use: "gkekitctl", 36 | Version: Version + " : Git Commit : " + GitCommit, 37 | Short: "Tool to quickly deploy some pretty dope GKE demos", 38 | Example: ` gkekitctl create 39 | gkectl create --config 40 | gkekitctl delete`, 41 | } 42 | 43 | // Execute adds all child commands to the root command and sets flags appropriately. 44 | // This is called by main.main(). It only needs to happen once to the rootCmd. 45 | func Execute() { 46 | cobra.CheckErr(rootCmd.Execute()) 47 | } 48 | 49 | func init() { 50 | cobra.OnInitialize(initConfig) 51 | 52 | // Here you will define your flags and configuration settings. 53 | // Cobra supports persistent flags, which, if defined here, 54 | // will be global for your application. 55 | 56 | rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file (default is /.gkekitctl.yaml)") 57 | rootCmd.PersistentFlags().StringVarP(&fleetProjectId, "fleet-project-id", "p", "", "Fleet project ID") 58 | 59 | // Cobra also supports local flags, which will only run 60 | // when this action is called directly. 61 | rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") 62 | 63 | log.Printf("☸️ ----- GKE POC TOOLKIT ----- 🛠\n") 64 | 65 | } 66 | 67 | // initConfig reads in config file and ENV variables if set. 68 | func initConfig() { 69 | if cfgFile != "" { 70 | // Use config file from the flag. 71 | viper.SetConfigFile(cfgFile) 72 | } else { 73 | viper.AddConfigPath(".") 74 | viper.SetConfigType("yaml") 75 | viper.SetConfigName(".gkekitctl") 76 | } 77 | 78 | viper.AutomaticEnv() // read in environment variables that match 79 | 80 | // If a config file is found, read it in. 81 | if err := viper.ReadInConfig(); err == nil { 82 | log.Printf("Using config file: %s", viper.ConfigFileUsed()) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /cli/go.mod: -------------------------------------------------------------------------------- 1 | module gkekitctl 2 | 3 | go 1.22 4 | 5 | require ( 6 | cloud.google.com/go/compute v1.28.1 7 | cloud.google.com/go/serviceusage v1.9.1 8 | cloud.google.com/go/storage v1.43.0 9 | github.com/gofrs/uuid v4.1.0+incompatible 10 | github.com/hashicorp/terraform-exec v0.15.0 11 | github.com/manifoldco/promptui v0.9.0 12 | github.com/sirupsen/logrus v1.8.1 13 | github.com/spf13/cobra v1.2.1 14 | github.com/spf13/viper v1.9.0 15 | github.com/thanhpk/randstr v1.0.4 16 | google.golang.org/api v0.196.0 17 | google.golang.org/genproto v0.0.0-20240924160255-9d4c2d233b61 18 | google.golang.org/genproto/googleapis/api/serviceusage v0.0.0-20240930140551-af27646dc61f 19 | k8s.io/apimachinery v0.22.4 20 | k8s.io/client-go v0.22.4 21 | ) 22 | 23 | require ( 24 | cloud.google.com/go/auth v0.9.3 // indirect 25 | cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect 26 | cloud.google.com/go/compute/metadata v0.5.0 // indirect 27 | cloud.google.com/go/longrunning v0.6.1 // indirect 28 | github.com/felixge/httpsnoop v1.0.4 // indirect 29 | github.com/go-logr/stdr v1.2.2 // indirect 30 | github.com/google/s2a-go v0.1.8 // indirect 31 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect 32 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect 33 | go.opentelemetry.io/otel v1.29.0 // indirect 34 | go.opentelemetry.io/otel/metric v1.29.0 // indirect 35 | go.opentelemetry.io/otel/trace v1.29.0 // indirect 36 | golang.org/x/sync v0.8.0 // indirect 37 | google.golang.org/genproto/googleapis/api v0.0.0-20240924160255-9d4c2d233b61 // indirect 38 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect 39 | ) 40 | 41 | require ( 42 | cloud.google.com/go v0.115.1 // indirect 43 | cloud.google.com/go/gkeconnect v0.11.1 44 | cloud.google.com/go/iam v1.2.1 // indirect 45 | github.com/aws/aws-sdk-go v1.44.122 // indirect 46 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect 47 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect 48 | github.com/davecgh/go-spew v1.1.1 // indirect 49 | github.com/fsnotify/fsnotify v1.5.1 // indirect 50 | github.com/go-logr/logr v1.4.2 // indirect 51 | github.com/gogo/protobuf v1.3.2 // indirect 52 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 53 | github.com/golang/protobuf v1.5.4 // indirect 54 | github.com/google/go-cmp v0.6.0 // indirect 55 | github.com/google/gofuzz v1.1.0 // indirect 56 | github.com/google/uuid v1.6.0 // indirect 57 | github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect 58 | github.com/googleapis/gax-go/v2 v2.13.0 // indirect 59 | github.com/googleapis/gnostic v0.5.5 // indirect 60 | github.com/hashicorp/go-checkpoint v0.5.0 // indirect 61 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 62 | github.com/hashicorp/go-getter v1.7.1 // indirect 63 | github.com/hashicorp/go-safetemp v1.0.0 // indirect 64 | github.com/hashicorp/go-uuid v1.0.1 // indirect 65 | github.com/hashicorp/go-version v1.6.0 // indirect 66 | github.com/hashicorp/hcl v1.0.0 // indirect 67 | github.com/hashicorp/terraform-json v0.13.0 // indirect 68 | github.com/imdario/mergo v0.3.12 // indirect 69 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 70 | github.com/jmespath/go-jmespath v0.4.0 // indirect 71 | github.com/json-iterator/go v1.1.11 // indirect 72 | github.com/klauspost/compress v1.15.11 // indirect 73 | github.com/magiconair/properties v1.8.5 // indirect 74 | github.com/mitchellh/go-homedir v1.1.0 // indirect 75 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 76 | github.com/mitchellh/mapstructure v1.4.2 // indirect 77 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 78 | github.com/modern-go/reflect2 v1.0.1 // indirect 79 | github.com/pelletier/go-toml v1.9.4 // indirect 80 | github.com/spf13/afero v1.6.0 // indirect 81 | github.com/spf13/cast v1.4.1 // indirect 82 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 83 | github.com/spf13/pflag v1.0.5 // indirect 84 | github.com/subosito/gotenv v1.2.0 // indirect 85 | github.com/ulikunitz/xz v0.5.10 // indirect 86 | github.com/zclconf/go-cty v1.9.1 // indirect 87 | go.opencensus.io v0.24.0 // indirect 88 | golang.org/x/crypto v0.27.0 // indirect 89 | golang.org/x/net v0.29.0 // indirect 90 | golang.org/x/oauth2 v0.22.0 // indirect 91 | golang.org/x/sys v0.25.0 // indirect 92 | golang.org/x/term v0.24.0 // indirect 93 | golang.org/x/text v0.18.0 // indirect 94 | golang.org/x/time v0.6.0 // indirect 95 | google.golang.org/grpc v1.67.0 // indirect 96 | google.golang.org/protobuf v1.34.2 // indirect 97 | gopkg.in/inf.v0 v0.9.1 // indirect 98 | gopkg.in/ini.v1 v1.63.2 // indirect 99 | gopkg.in/yaml.v2 v2.4.0 // indirect 100 | gopkg.in/yaml.v3 v3.0.1 // indirect 101 | k8s.io/api v0.22.4 // indirect 102 | k8s.io/klog/v2 v2.130.1 // indirect 103 | k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a // indirect 104 | sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect 105 | sigs.k8s.io/yaml v1.2.0 // indirect 106 | ) 107 | -------------------------------------------------------------------------------- /cli/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2021 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "gkekitctl/cmd" 21 | ) 22 | 23 | func main() { 24 | cmd.Execute() 25 | } 26 | -------------------------------------------------------------------------------- /cli/pkg/analytics/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package analytics 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "gkekitctl/pkg/config" 23 | 24 | "net/http" 25 | "runtime" 26 | "time" 27 | 28 | "github.com/gofrs/uuid" 29 | log "github.com/sirupsen/logrus" 30 | ) 31 | 32 | // SendMetrics runs on gkekitctl create, after config validation and before TF apply. It gets the user's OS, scrubs their project ID data, and sends their create configuration to the metrics server. 33 | 34 | // This is an Analytics-only object representing 1 GKE cluster created with gkekitctl. 35 | type Cluster struct { 36 | ClusterId string `json:"clusterId"` 37 | CreateId string `json:"createId"` 38 | Version string `json:"version"` 39 | GitCommit string `json:"gitCommit"` 40 | Timestamp string `json:"timestamp"` 41 | OS string `json:"os"` 42 | TerraformState string `json:"terraformState"` 43 | Region string `json:"region"` 44 | EnablePreemptibleNodepool bool `json:"enablePreemptibleNodepool"` 45 | DefaultNodepoolOS string `json:"defaultNodepoolOS"` 46 | PrivateEndpoint bool `json:"privateEndpoint"` 47 | EnableConfigSync bool `json:"enableConfigSync"` 48 | EnablePolicyController bool `json:"enablePolicyController"` 49 | AnthosServiceMesh bool `json:"anthosServiceMesh"` 50 | MultiClusterGateway bool `json:"multiClusterGateway"` 51 | VPCType string `json:"vpcType"` 52 | ClusterIndex int `json:"clusterIndex"` 53 | ClusterType string `json:"clusterType"` 54 | ClusterMachineType string `json:"clusterMachineType"` 55 | ClusterRegion string `json:"clusterRegion"` 56 | ClusterZones []string `json:"[clusterZone]"` 57 | } 58 | 59 | func SendAnalytics(conf *config.Config, version string, gitCommit string) { 60 | // Generate timestamp. Format: 2006-01-02T15:04:05.000Z 61 | now := time.Now() 62 | timestamp := now.Format("2006-01-02T15:04:05.000Z") 63 | 64 | // Generate CreateId used by all clusters 65 | 66 | // Assign UUIDs to create run and cluster 67 | createId, err := uuid.NewV4() 68 | if err != nil { 69 | log.Warn(err) 70 | return 71 | } 72 | 73 | for i, cluster := range conf.ClustersConfig { 74 | // Cluster Id 75 | clusterId, err := uuid.NewV4() 76 | if err != nil { 77 | log.Warn(err) 78 | return 79 | } 80 | 81 | // Create Cluster Object, omitting any user-identified data (Project IDs) 82 | // NOTE - ClusterId and CreateId are generated server-side before DB insert to avoid HTTP POST anti-patterns 83 | sendObject := Cluster{ 84 | ClusterId: clusterId.String(), 85 | CreateId: createId.String(), 86 | Version: version, 87 | GitCommit: gitCommit, 88 | Timestamp: timestamp, 89 | OS: runtime.GOOS, 90 | TerraformState: conf.TerraformState, 91 | DefaultNodepoolOS: conf.DefaultNodepoolOS, 92 | PrivateEndpoint: conf.PrivateEndpoint, 93 | VPCType: conf.VpcConfig.VpcType, 94 | ClusterIndex: i, 95 | ClusterType: cluster.ClusterType, 96 | ClusterMachineType: cluster.MachineType, 97 | ClusterRegion: cluster.Region, 98 | ClusterZones: cluster.Zones, 99 | } 100 | // Encode as JSON 101 | json, err := json.Marshal(sendObject) 102 | if err != nil { 103 | log.Warn(err) 104 | return 105 | } 106 | err = PostToAnalyticsServer(json) 107 | if err != nil { 108 | log.Warn(err) 109 | return 110 | } 111 | } 112 | log.Info("✅ Sent cluster info to analytics server") 113 | return 114 | } 115 | 116 | func PostToAnalyticsServer(json []byte) error { 117 | url := "https://analytics.gkepoctoolkit.dev" 118 | req, err := http.NewRequest("POST", url, bytes.NewBuffer(json)) 119 | if err != nil { 120 | return err 121 | } 122 | req.Header.Set("Content-Type", "application/json") 123 | 124 | client := &http.Client{} 125 | resp, err := client.Do(req) 126 | if err != nil { 127 | return err 128 | } 129 | defer resp.Body.Close() 130 | return nil 131 | } 132 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/clusters/main.tf: -------------------------------------------------------------------------------- 1 | module "clusters" { 2 | source = "{{.TFModuleRepo}}clusters?ref={{.TFModuleBranch}}" 3 | project_id = var.project_id 4 | fleet_project = var.fleet_project 5 | regional_clusters = var.regional_clusters 6 | shared_vpc = var.shared_vpc 7 | vpc_name = var.vpc_name 8 | vpc_project_id = var.vpc_project_id 9 | vpc_ip_range_pods_name = var.vpc_ip_range_pods_name 10 | vpc_ip_range_services_name = var.vpc_ip_range_services_name 11 | release_channel = var.release_channel 12 | initial_node_count = var.initial_node_count 13 | min_node_count = var.min_node_count 14 | max_node_count = var.max_node_count 15 | linux_machine_type = var.linux_machine_type 16 | private_endpoint = var.private_endpoint 17 | authenticator_security_group = var.authenticator_security_group 18 | cluster_config = var.cluster_config 19 | } 20 | 21 | variable "project_id" { 22 | type = string 23 | description = "The project ID to host the cluster in" 24 | } 25 | 26 | variable "fleet_project" { 27 | description = "(Optional) Register the cluster with the fleet in this project." 28 | type = string 29 | default = null 30 | } 31 | 32 | variable "regional_clusters" { 33 | type = bool 34 | description = "Enable regional control plane." 35 | default = true 36 | } 37 | 38 | variable "region" { 39 | type = string 40 | description = "The region to host the cluster in" 41 | default = "us-central1" 42 | } 43 | 44 | variable "shared_vpc" { 45 | type = bool 46 | description = "boolean value for determining whether to create Standalone VPC or use a preexisting Shared VPC" 47 | default = false 48 | } 49 | 50 | variable "vpc_name" { 51 | type = string 52 | description = "The name of the network being created to host the cluster in" 53 | default = "gke-toolkit-network" 54 | } 55 | 56 | 57 | variable "vpc_project_id" { 58 | type = string 59 | description = "The Share VPC Project ID - This is optional and only valid if a Shared VPC is used" 60 | default = "" 61 | } 62 | 63 | variable "vpc_ip_range_pods_name" { 64 | type = string 65 | description = "The secondary ip range to use for pods in the shared vpc - This is optional and only valid if a Shared VPC is used" 66 | default = "" 67 | } 68 | 69 | variable "vpc_ip_range_services_name" { 70 | type = string 71 | description = "The secondary ip range to use for services in the shared vpc - This is optional and only valid if a Shared VPC is used" 72 | default = "" 73 | } 74 | 75 | variable "release_channel" { 76 | type = string 77 | default = "regular" 78 | } 79 | 80 | variable "node_pool" { 81 | type = string 82 | default = "gke-toolkit-pool" 83 | } 84 | 85 | variable "initial_node_count" { 86 | type = number 87 | default = 4 88 | } 89 | 90 | variable "min_node_count" { 91 | type = number 92 | default = 4 93 | } 94 | 95 | variable "max_node_count" { 96 | type = number 97 | default = 10 98 | } 99 | 100 | variable "linux_machine_type" { 101 | type = string 102 | default = "n1-standard-4" 103 | } 104 | 105 | variable "private_endpoint" { 106 | type = bool 107 | default = false 108 | } 109 | 110 | variable "authenticator_security_group" { 111 | type = string 112 | description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" 113 | default = null 114 | } 115 | 116 | variable "cluster_config" { 117 | description = "For each cluster, create an object that contain the required fields" 118 | default = {} 119 | } 120 | 121 | variable "auth_cidr" { 122 | type = string 123 | default = "172.16.100.16/28" 124 | } -------------------------------------------------------------------------------- /cli/pkg/cli_init/fleet/main.tf: -------------------------------------------------------------------------------- 1 | module "fleet" { 2 | source = "{{.TFModuleRepo}}fleet?ref={{.TFModuleBranch}}" 3 | project_id = var.project_id 4 | fleet_project = var.fleet_project 5 | config_sync_repo = var.config_sync_repo 6 | config_sync_repo_branch = var.config_sync_repo_branch 7 | config_sync_repo_dir = var.config_sync_repo_dir 8 | vpc_name = var.vpc_name 9 | vpc_project_id = var.vpc_project_id 10 | release_channel = var.release_channel 11 | authenticator_security_group = var.authenticator_security_group 12 | cluster_config = var.cluster_config 13 | shared_vpc = var.shared_vpc 14 | vpc_ip_range_pods_name = var.vpc_ip_range_pods_name 15 | vpc_ip_range_services_name = var.vpc_ip_range_services_name 16 | } 17 | 18 | variable "project_id" { 19 | type = string 20 | description = "The project ID to host the cluster in" 21 | } 22 | 23 | variable "fleet_project" { 24 | type = string 25 | description = "(Optional) Register the cluster with the fleet in this project." 26 | } 27 | 28 | variable "vpc_project_id" { 29 | type = string 30 | description = "Shared VPC project needed for setting MCI and MCS RBAC." 31 | } 32 | 33 | variable "config_sync_repo" { 34 | description = "Git repo used as the default config sync repo for your fleet." 35 | type = string 36 | default = null 37 | } 38 | 39 | variable "config_sync_repo_branch" { 40 | description = "Git repo branch used as the default config sync repo for your fleet." 41 | type = string 42 | default = null 43 | } 44 | 45 | variable "config_sync_repo_dir" { 46 | description = "Git repo directory used as the default config sync repo for your fleet." 47 | type = string 48 | default = null 49 | } 50 | 51 | variable "shared_vpc" { 52 | type = bool 53 | description = "Determines whether to create a standalone VPC or use an existing Shared VPC" 54 | default = false 55 | } 56 | 57 | variable "vpc_ip_range_pods_name" { 58 | type = string 59 | description = "The secondary IP range to use for pods in the shared VPC" 60 | default = "" 61 | } 62 | 63 | variable "vpc_ip_range_services_name" { 64 | type = string 65 | description = "The secondary IP range to use for services in the shared VPC" 66 | default = "" 67 | } 68 | 69 | variable "release_channel" { 70 | type = string 71 | description = "The release channel of the cluster" 72 | default = "regular" 73 | } 74 | 75 | variable "authenticator_security_group" { 76 | type = string 77 | description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC." 78 | default = null 79 | } 80 | 81 | variable "vpc_name" { 82 | type = string 83 | description = "The name of the VPC - used for shared or local VPC" 84 | default = "" 85 | } 86 | 87 | variable "cluster_config" { 88 | type = map(object({ 89 | subnet_name = string 90 | region = string 91 | })) 92 | description = "For each cluster, create an object that contains the required fields" 93 | default = {} 94 | } 95 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/init.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cli_init 18 | 19 | import ( 20 | "bytes" 21 | "embed" 22 | "fmt" 23 | "os" 24 | "strings" 25 | 26 | "github.com/manifoldco/promptui" 27 | 28 | log "github.com/sirupsen/logrus" 29 | ) 30 | 31 | // embeding flatfiles and setting them as a file system variable, embed.FS. 32 | // embed.FS can be treated like io.FS. 33 | 34 | //go:embed templates/* samples/* clusters/* network/* fleet/* 35 | var templates embed.FS 36 | 37 | func InitFlatFiles(folders []string) error { 38 | log.Info("🔄 Initializing flat files for gkekitctl...") 39 | 40 | // Range over embedded folders of flat files 41 | for _, folder := range folders { 42 | files, err := templates.ReadDir(folder) 43 | if err != nil { 44 | return err 45 | } 46 | var buf bytes.Buffer 47 | 48 | // Range over embed files in folder and write them out to the directory gkekitctl is running inside 49 | for _, file := range files { 50 | b, err := templates.ReadFile(folder + "/" + file.Name()) 51 | if err != nil { 52 | return err 53 | } 54 | if _, err := os.Stat(folder); os.IsNotExist(err) { 55 | os.MkdirAll(folder, 0700) 56 | } 57 | buf.Write(b) 58 | err = os.WriteFile(folder+"/"+file.Name(), buf.Bytes(), 0644) 59 | if err != nil { 60 | return err 61 | } 62 | buf.Reset() 63 | } 64 | } 65 | return nil 66 | } 67 | 68 | // Helper function to create a list of files from a folder 69 | func CreateFileList(dir string) []string { 70 | files_out := []string{} 71 | folder, err := os.Open(dir) 72 | if err != nil { 73 | log.Fatal(err) 74 | } 75 | files, err := folder.Readdir(-1) 76 | folder.Close() 77 | if err != nil { 78 | log.Fatal(err) 79 | } 80 | for _, file := range files { 81 | files_out = append(files_out, file.Name()) 82 | } 83 | return files_out 84 | } 85 | 86 | // Prompt user to opt into anonymous analytics 87 | func OptInAnalytics() error { 88 | log.Info("📊 Send anonymous analytics to GKE PoC Toolkit maintainers?") 89 | sendAnalytics := yesNo() 90 | if !sendAnalytics { 91 | return nil 92 | } 93 | // Write opt-in to all config files 94 | files, err := os.ReadDir("./samples") 95 | if err != nil { 96 | return err 97 | } 98 | 99 | for _, f := range files { 100 | log.Infof("Processing file: %s", f.Name()) 101 | err := addOptInAnalyticsToConfigFile(fmt.Sprintf("samples/%s", f.Name())) 102 | if err != nil { 103 | return err 104 | } 105 | } 106 | return nil 107 | } 108 | 109 | func yesNo() bool { 110 | prompt := promptui.Select{ 111 | Label: "Select[Yes/No]", 112 | Items: []string{"Yes", "No"}, 113 | } 114 | _, result, err := prompt.Run() 115 | if err != nil { 116 | log.Warnf("Prompt failed %v\n", err) 117 | } 118 | result = strings.ToUpper(result) 119 | return result == "YES" 120 | } 121 | 122 | func addOptInAnalyticsToConfigFile(f string) error { 123 | input, err := os.ReadFile(f) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | lines := strings.Split(string(input), "\n") 129 | 130 | for i, line := range lines { 131 | if strings.Contains(line, "sendAnalytics") { 132 | lines[i] = "sendAnalytics: true" 133 | } 134 | } 135 | output := strings.Join(lines, "\n") 136 | err = os.WriteFile(f, []byte(output), 0644) 137 | if err != nil { 138 | return err 139 | } 140 | return nil 141 | } 142 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/network/main.tf: -------------------------------------------------------------------------------- 1 | module "network" { 2 | source = "{{.TFModuleRepo}}network?ref={{.TFModuleBranch}}" 3 | project_id = var.project_id 4 | vpc_project_id = var.vpc_project_id 5 | vpc_name = var.vpc_name 6 | vpc_ip_range_pods_name = var.vpc_ip_range_pods_name 7 | vpc_ip_range_services_name = var.vpc_ip_range_services_name 8 | cluster_config = var.cluster_config 9 | shared_vpc = var.shared_vpc 10 | region = var.region 11 | } 12 | 13 | variable "project_id" { 14 | type = string 15 | description = "The project ID to host the cluster in" 16 | } 17 | 18 | variable "vpc_project_id" { 19 | type = string 20 | description = "The Share VPC Project ID - This is optional and only valid if a Shared VPC is used" 21 | default = "" 22 | } 23 | 24 | variable "shared_vpc" { 25 | type = bool 26 | description = "Determines whether to create a standalone VPC or use an existing Shared VPC" 27 | default = false 28 | } 29 | 30 | variable "region" { 31 | type = string 32 | description = "The region to host the cluster in" 33 | default = "us-central1" 34 | } 35 | 36 | variable "vpc_name" { 37 | type = string 38 | description = "The name of the Shared VPC - This is optional and only valid if a Shared VPC is used" 39 | default = "" 40 | } 41 | 42 | variable "vpc_ip_range_pods_name" { 43 | type = string 44 | description = "The secondary ip range to use for pods in the shared vpc - This is optional and only valid if a Shared VPC is used" 45 | default = "" 46 | } 47 | 48 | variable "vpc_ip_range_services_name" { 49 | type = string 50 | description = "The secondary ip range to use for services in the shared vpc - This is optional and only valid if a Shared VPC is used" 51 | default = "" 52 | } 53 | 54 | variable "cluster_config" { 55 | description = "For each cluster, create an object that contain the required fields" 56 | default = {} 57 | } -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/default-config.yaml: -------------------------------------------------------------------------------- 1 | # ------ DO NOT EDIT ------------------ 2 | # These are the default values used for `gkekitctl create` when 3 | # the user does not supply their own config.yml. Note that the user 4 | # *does* have to provide their GCP project ID at runtime. 5 | 6 | ## Global 7 | sendAnalytics: false 8 | terraformState: cloud # local, cloud 9 | 10 | ## VPC Build 11 | vpcConfig: 12 | vpcName: "prod" 13 | vpcType: "standalone" # standalone, shared 14 | vpcProjectId: "my-project" 15 | vpcPodCIDRName: "default" 16 | vpcSvcCIDRName: "default" 17 | 18 | ## FleetBuild 19 | configSyncRepo: "default-config-sync-repo" 20 | configSyncBranch: "main" 21 | configSyncDir: "/" 22 | 23 | ## Cluster Build 24 | authenticatorSecurityGroup: "gke-security-groups@" 25 | clustersProjectId: "my-project" 26 | fleetProjectId: "my-project" 27 | privateEndpoint: true 28 | releaseChannel: REGULAR 29 | initialNodeCount: 1 # for Autopilot this has to be 1 30 | minNodeCount: 1 31 | maxNodeCount: 10 32 | defaultNodepoolOS: cos 33 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 34 | tfModuleBranch: "main" 35 | 36 | clustersConfig: # a list of one or more clusters, each with their own config 37 | - clusterName: "gke-ap-central-00" 38 | machineType: "e2-standard-4" 39 | region: "us-central1" 40 | zones: ["us-central1-a"] 41 | subnetName: "us-central1" -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/fleet-full.yaml: -------------------------------------------------------------------------------- 1 | ## Global 2 | sendAnalytics: true 3 | terraformState: cloud # local, cloud 4 | 5 | ## VPC Build 6 | vpcConfig: 7 | vpcName: "prod-3" 8 | vpcType: "standalone" # standalone, shared 9 | vpcProjectId: "app-team-2-bu-1" 10 | vpcPodCIDRName: "mypodcidr" 11 | vpcSvcCIDRName: "mysvccidr" 12 | 13 | ## FleetBuild 14 | configSyncRepo: "https://github.com/knee-berts/fleets-next24-demo" 15 | configSyncBranch: "dev" 16 | configSyncDir: "default-configs" 17 | 18 | ## Cluster Build 19 | regionalClusters: true # Control plane availability 20 | authenticatorSecurityGroup: "gke-security-groups@gkedemos.joonix.net" 21 | clustersProjectId: "app-team-2-bu-1" 22 | fleetProjectId: "app-team-2-bu-1" 23 | privateEndpoint: true 24 | releaseChannel: REGULAR 25 | initialNodeCount: 10 26 | minNodeCount: 1 27 | maxNodeCount: 10 28 | defaultNodepoolOS: cos 29 | tfModuleRepo: "github.com/knee-berts/gke-poc-toolkit//terraform/modules/" 30 | tfModuleBranch: "main" 31 | 32 | clustersConfig: # a list of one or more clusters, each with their own config 33 | - clusterName: "gke-ap-central-03" 34 | machineType: "e2-standard-4" 35 | region: "us-central1" 36 | zones: ["us-central1-b"] 37 | subnetName: "us-central1" 38 | - clusterName: "gke-ap-east-03" 39 | machineType: "e2-standard-4" 40 | region: "us-east1" 41 | zones: ["us-east1-b"] 42 | subnetName: "us-east1" 43 | # - clusterName: "gke-west" 44 | # machineType: "e2-standard-4" 45 | # region: "us-west1" 46 | # zones: ["us-west1-b"] 47 | # subnetName: "us-west1" 48 | # - clusterName: "gke-eu-north" 49 | # machineType: "e2-standard-4" 50 | # region: "europe-north1" 51 | # zones: ["europe-north1-c"] 52 | # subnetName: "europe-north1" 53 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/fleet-shared-vpc.yaml: -------------------------------------------------------------------------------- 1 | ## Global 2 | sendAnalytics: true 3 | terraformState: cloud # local, cloud 4 | 5 | ## VPC Build 6 | vpcConfig: 7 | vpcName: "prod" 8 | vpcType: "shared" # standalone, shared 9 | vpcProjectId: "shared-vpc-prod-00" 10 | vpcPodCIDRName: "mypodcidr" 11 | vpcSvcCIDRName: "mysvccidr" 12 | 13 | ## FleetBuild 14 | configSyncRepo: "https://github.com/knee-berts/fleets-next24-demo" 15 | configSyncBranch: "dev" 16 | configSyncDir: "default-configs" 17 | 18 | ## Cluster Build 19 | regionalClusters: true # Control plane availability 20 | authenticatorSecurityGroup: "gke-security-groups@gkedemos.joonix.net" 21 | clustersProjectId: "fleet-prod-2" 22 | fleetProjectId: "fleet-prod-2" 23 | privateEndpoint: true 24 | releaseChannel: REGULAR 25 | initialNodeCount: 10 26 | minNodeCount: 1 27 | maxNodeCount: 10 28 | defaultNodepoolOS: cos 29 | tfModuleRepo: "github.com/knee-berts/gke-poc-toolkit//terraform/modules/" 30 | tfModuleBranch: "main" 31 | 32 | clustersConfig: # a list of one or more clusters, each with their own config 33 | - clusterName: "gke-std-central-00" 34 | machineType: "e2-standard-4" 35 | region: "us-central1" 36 | zones: ["us-central1-b"] 37 | subnetName: "us-central1" 38 | - clusterName: "gke-std-east-00" 39 | machineType: "e2-standard-4" 40 | region: "us-east1" 41 | zones: ["us-east1-b"] 42 | subnetName: "us-east1" 43 | # - clusterName: "gke-west" 44 | # machineType: "e2-standard-4" 45 | # region: "us-west1" 46 | # zones: ["us-west1-b"] 47 | # subnetName: "us-west1" 48 | # - clusterName: "gke-eu-north" 49 | # machineType: "e2-standard-4" 50 | # region: "europe-north1" 51 | # zones: ["europe-north1-c"] 52 | # subnetName: "europe-north1" 53 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/module-bypass-multi-clusters-networking-acm-standalone-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: false # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: true 17 | multiClusterGateway: true 18 | anthosServiceMesh: true 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: true 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "standalone" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" 51 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/multi-clusters-acm-shared-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: true # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: true 17 | multiClusterGateway: false 18 | anthosServiceMesh: false 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: false 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "shared" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" 51 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/multi-clusters-acm-standalone-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: true # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: true 17 | multiClusterGateway: false 18 | anthosServiceMesh: false 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: false 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "standalone" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" 51 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/multi-clusters-networking-acm-shared-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: true # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: true 17 | multiClusterGateway: true 18 | anthosServiceMesh: true 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: false 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "shared" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" 51 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/multi-clusters-networking-acm-standalone-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: true # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: true 17 | multiClusterGateway: true 18 | anthosServiceMesh: true 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: false 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "standalone" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" 51 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/multi-clusters-shared-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: true # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: false 17 | multiClusterGateway: false 18 | anthosServiceMesh: false 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: false 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "shared" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" 51 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/samples/multi-clusters-standalone-vpc.yaml: -------------------------------------------------------------------------------- 1 | terraformState: cloud # local, cloud 2 | clustersProjectId: "my-project" 3 | governanceProjectId: "my-project" 4 | regionalClusters: true # Control plane availability 5 | region: "us-east1" # Region for resources aside from GKE clusters 6 | enableWindowsNodepool: false 7 | enablePreemptibleNodepool: false # Enforced on Linux Node pools only 8 | privateEndpoint: false 9 | releaseChannel: REGULAR 10 | defaultNodepoolOS: cos 11 | initialNodeCount: 10 12 | maxNodeCount: 10 13 | minNodeCount: 1 14 | configSync: true 15 | configSyncRepo: "config-sync-repo" 16 | policyController: false 17 | multiClusterGateway: false 18 | anthosServiceMesh: false 19 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 20 | tfModuleBranch: "main" 21 | gkeModuleBypass: false 22 | sendAnalytics: false 23 | vpcConfig: 24 | vpcName: "gke-poc-toolkit" 25 | vpcType: "standalone" # standalone, shared 26 | vpcProjectId: "my-host-project" 27 | podCIDRName: "mypodcidr" 28 | svcCIDRName: "mysvccidr" 29 | authCIDR: "0.0.0.0/0" # Change to your workstation public IP 30 | clustersConfig: # a list of one or more clusters, each with their own config 31 | - clusterName: "gke-central" 32 | machineType: "e2-standard-4" 33 | region: "us-central1" 34 | zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-east" 37 | machineType: "e2-standard-4" 38 | region: "us-east1" 39 | zones: ["us-east1-b"] 40 | subnetName: "us-east1" 41 | - clusterName: "gke-west" 42 | machineType: "e2-standard-4" 43 | region: "us-west1" 44 | zones: ["us-west1-b"] 45 | subnetName: "us-west1" 46 | - clusterName: "gke-eu-north" 47 | machineType: "e2-standard-4" 48 | region: "europe-north1" 49 | zones: ["europe-north1-c"] 50 | subnetName: "europe-north1" -------------------------------------------------------------------------------- /cli/pkg/cli_init/templates/cluster_config.tmpl: -------------------------------------------------------------------------------- 1 | {{"\t"}}{{.ClusterName}} = { 2 | {{"\t"}} region = "{{.Region}}" 3 | {{"\t"}} zones = ["{{.Zones}}"] 4 | {{"\t"}} subnet_name = "{{.SubnetName}}" 5 | {{"\t"}} linux_machine_type = "{{.MachineType}}" 6 | {{"\t"}}} 7 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/templates/terraform.tfvars.tmpl: -------------------------------------------------------------------------------- 1 | ## VPC Config 2 | shared_vpc = "{{.VpcType}}" 3 | vpc_name = "{{.VpcName}}" 4 | vpc_project_id = "{{.VpcProjectId}}" 5 | vpc_ip_range_pods_name = "{{.VpcPodCidrName}}" 6 | vpc_ip_range_services_name = "{{.VpcSvcCidrName}}" 7 | 8 | ## Fleet Config 9 | config_sync_repo = "{{.ConfigSyncRepo}}" 10 | config_sync_repo_branch = "{{.ConfigSyncRepoBranch}}" 11 | config_sync_repo_dir = "{{.ConfigSyncRepoDir}}" 12 | 13 | ## Clusters Base Config 14 | regional_clusters = "{{.RegionalClusters}}" 15 | authenticator_security_group = "{{.AuthenticatorSecurityGroup}}" 16 | project_id = "{{.ClustersProjectId}}" 17 | fleet_project = "{{.FleetProjectId}}" 18 | private_endpoint = "{{.PrivateEndpoint}}" 19 | release_channel = "{{.ReleaseChannel}}" 20 | initial_node_count = "{{.InitialNodeCount}}" 21 | min_node_count = "{{.MinNodeCount}}" 22 | max_node_count = "{{.MaxNodeCount}}" 23 | default_nodepool_os = "{{.DefaultNodepoolOs}}" 24 | tf_module_repo = "{{.TFModuleRepo}}" 25 | tf_module_branch = "{{.TFModuleBranch}}" 26 | 27 | ## Clusters Config 28 | cluster_config = { 29 | -------------------------------------------------------------------------------- /cli/pkg/cli_init/templates/terraform_backend.tf.tmpl: -------------------------------------------------------------------------------- 1 | terraform { 2 | backend "gcs" { 3 | bucket = "{{.TfStateBucket}}" 4 | prefix = "terraform/state" 5 | } 6 | } -------------------------------------------------------------------------------- /cli/pkg/config/tf_state.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "context" 21 | "html/template" 22 | "os" 23 | "strings" 24 | 25 | "cloud.google.com/go/storage" 26 | log "github.com/sirupsen/logrus" 27 | "github.com/thanhpk/randstr" 28 | ) 29 | 30 | func CheckTfStateType(conf *Config, bucketNameNetwork string, bucketNameFleet string, bucketNameClusters string) error { 31 | if conf.TerraformState == "cloud" { 32 | if bucketNameNetwork == "" { 33 | bucketNameNetwork = "tf-state-network-" + strings.ToLower(randstr.String(6)) 34 | err := createTfStorage(conf.ClustersProjectID, bucketNameNetwork) 35 | if err != nil { 36 | return err 37 | } 38 | log.Infof("✅ Created a bucket for the Network TF State: %s", bucketNameNetwork) 39 | } 40 | err := createTfBackend(bucketNameNetwork, "network/backend.tf") 41 | if err != nil { 42 | return err 43 | } 44 | if bucketNameFleet == "" { 45 | bucketNameFleet = "tf-state-fleet-" + strings.ToLower(randstr.String(6)) 46 | err := createTfStorage(conf.ClustersProjectID, bucketNameFleet) 47 | if err != nil { 48 | return err 49 | } 50 | log.Infof("✅ Created a bucket for the Fleet TF State: %s", bucketNameFleet) 51 | } 52 | err = createTfBackend(bucketNameFleet, "fleet/backend.tf") 53 | if err != nil { 54 | return err 55 | } 56 | if bucketNameClusters == "" { 57 | bucketNameClusters = "tf-state-clusters-" + strings.ToLower(randstr.String(6)) 58 | err := createTfStorage(conf.ClustersProjectID, bucketNameClusters) 59 | if err != nil { 60 | return err 61 | } 62 | log.Infof("✅ Created a bucket for the Clusters TF State: %s", bucketNameFleet) 63 | } 64 | err = createTfBackend(bucketNameClusters, "clusters/backend.tf") 65 | if err != nil { 66 | return err 67 | } 68 | log.Infof("✅ Created Cluster TF State backend file in bucket: %s", bucketNameClusters) 69 | return nil 70 | } 71 | return nil 72 | } 73 | 74 | // func CreateTfStateBucket(projectId string, bucketName string) error { 75 | func createTfStorage(projectId string, bucketName string) error { 76 | ctx := context.Background() 77 | c, err := storage.NewClient(ctx) 78 | if err != nil { 79 | return err 80 | } 81 | err = c.Bucket(bucketName).Create(ctx, projectId, nil) 82 | if err != nil { 83 | log.Fatalf("error creating storage bucket: %s", err) 84 | return err 85 | } 86 | log.Infof("✅ Created storage bucket: %s ", bucketName) 87 | return err 88 | } 89 | 90 | func createTfBackend(bucketName string, fileLocation string) error { 91 | vars := make(map[string]interface{}) 92 | vars["TfStateBucket"] = bucketName 93 | tmpl, err := template.ParseFiles("templates/terraform_backend.tf.tmpl") 94 | if err != nil { 95 | log.Fatalf("error parsing template: %s", err) 96 | return err 97 | } 98 | file, err := os.Create(fileLocation) 99 | if err != nil { 100 | log.Fatalf("error creating file: %s", err) 101 | return err 102 | } 103 | defer file.Close() 104 | err = tmpl.Execute(file, vars) 105 | if err != nil { 106 | log.Fatalf("error executing tffavs template merge: %s", err) 107 | return err 108 | } 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /cli/pkg/config/tfvars_generator.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "bytes" 21 | "os" 22 | "strings" 23 | "text/template" 24 | 25 | log "github.com/sirupsen/logrus" 26 | ) 27 | 28 | func GenerateTfvars(conf *Config) { 29 | vars := make(map[string]interface{}) 30 | 31 | // Set base config vars 32 | vars["RegionalClusters"] = conf.RegionalClusters 33 | vars["AuthenticatorSecurityGroup"] = conf.AuthenticatorSecurityGroup 34 | vars["ClustersProjectId"] = conf.ClustersProjectID 35 | vars["FleetProjectId"] = conf.FleetProjectID 36 | vars["PrivateEndpoint"] = conf.PrivateEndpoint 37 | vars["ReleaseChannel"] = conf.ReleaseChannel 38 | vars["InitialNodeCount"] = conf.InitialNodeCount 39 | vars["MinNodeCount"] = conf.MinNodeCount 40 | vars["MaxNodeCount"] = conf.MaxNodeCount 41 | vars["DefaultNodepoolOS"] = conf.DefaultNodepoolOS 42 | vars["TFModuleRepo"] = conf.TFModuleRepo 43 | vars["TFModuleBranch"] = conf.TFModuleBranch 44 | vars["ConfigSyncRepo"] = conf.ConfigSyncRepo 45 | vars["ConfigSyncRepoBranch"] = conf.ConfigSyncRepoBranch 46 | vars["ConfigSyncRepoDir"] = conf.ConfigSyncRepoDir 47 | 48 | // Set vpc config vars 49 | if conf.VpcConfig.VpcType == "standalone" { 50 | vars["VpcType"] = false 51 | } else { 52 | vars["VpcType"] = true 53 | } 54 | vars["VpcName"] = conf.VpcConfig.VpcName 55 | vars["VpcProjectId"] = conf.VpcConfig.VpcProjectID 56 | vars["VpcPodCidrName"] = conf.VpcConfig.VpcPodCIDRName 57 | vars["VpcSvcCidrName"] = conf.VpcConfig.VpcSvcCIDRName 58 | 59 | // First phase of templating tfvars (base and VPC configs) 60 | tmpl, err := template.ParseFiles("templates/terraform.tfvars.tmpl") 61 | if err != nil { 62 | log.Fatalf("error parsing tfvars template: %s", err) 63 | } 64 | file, err := os.Create("terraform.tfvars") 65 | if err != nil { 66 | log.Fatalf("error creating tfvars file: %s", err) 67 | } 68 | defer file.Close() 69 | err = tmpl.Execute(file, vars) 70 | if err != nil { 71 | log.Fatalf("error executing tffavs template merge: %s", err) 72 | } 73 | 74 | // Set Cluster config vars 75 | for cc := range conf.ClustersConfig { 76 | clusterVars := make(map[string]interface{}) 77 | clusterVars["ClusterName"] = conf.ClustersConfig[cc].ClusterName 78 | clusterVars["Region"] = conf.ClustersConfig[cc].Region 79 | clusterVars["Zones"] = strings.Join(conf.ClustersConfig[cc].Zones, ",") 80 | clusterVars["SubnetName"] = conf.ClustersConfig[cc].SubnetName 81 | clusterVars["MachineType"] = conf.ClustersConfig[cc].MachineType 82 | tmpl, err := template.ParseFiles("templates/cluster_config.tmpl") 83 | if err != nil { 84 | log.Fatalf("error parsing cluster_config template: %s", err) 85 | } 86 | file, err := os.Create("clusters.tf") 87 | if err != nil { 88 | log.Fatalf("error creating clusters tf file: %s", err) 89 | } 90 | defer file.Close() 91 | err = tmpl.Execute(file, clusterVars) 92 | if err != nil { 93 | log.Fatalf("error executing clusters template merge: %s", err) 94 | } 95 | files := []string{"terraform.tfvars", "clusters.tf"} 96 | var buf bytes.Buffer 97 | for _, file := range files { 98 | b, err := os.ReadFile(file) 99 | if err != nil { 100 | log.Fatalf("error reading %s: %s", file, err) 101 | } 102 | buf.Write(b) 103 | err = os.WriteFile("terraform.tfvars", buf.Bytes(), 0644) 104 | if err != nil { 105 | log.Fatalf("error writing to %s: %s", file, err) 106 | } 107 | } 108 | } 109 | err = os.Remove("clusters.tf") 110 | if err != nil { 111 | log.Fatalf("error deleting clusters.tf file: %s", err) 112 | } 113 | f, err := os.OpenFile("terraform.tfvars", 114 | os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 115 | if err != nil { 116 | log.Fatalf("error opening tfvars file for final insert: %s", err) 117 | } 118 | defer f.Close() 119 | if _, err := f.WriteString("}\n"); err != nil { 120 | log.Fatalf("error appending } to the tfvars file: %s", err) 121 | } 122 | log.Info("✅ TFVars generated successfully.") 123 | 124 | } 125 | -------------------------------------------------------------------------------- /cli/pkg/lifecycle/get_credentials.go: -------------------------------------------------------------------------------- 1 | package lifecycle 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | gateway "cloud.google.com/go/gkeconnect/gateway/apiv1" 9 | gatewaypb "cloud.google.com/go/gkeconnect/gateway/apiv1/gatewaypb" 10 | log "github.com/sirupsen/logrus" 11 | "google.golang.org/api/cloudresourcemanager/v1" 12 | "google.golang.org/api/gkehub/v1" 13 | "google.golang.org/api/option" 14 | "k8s.io/client-go/tools/clientcmd" 15 | clientcmdapi "k8s.io/client-go/tools/clientcmd/api" 16 | ) 17 | 18 | // GenerateKubeConfig generates a kubeconfig with contexts for all clusters in the GKE Demo Environment. 19 | func GenerateKubeConfig(fleetProjectId string) (*clientcmdapi.Config, error) { 20 | 21 | // Create a GKE Hub client. 22 | ctx := context.Background() 23 | hubClient, err := gkehub.NewService(ctx, option.WithEndpoint("https://gkehub.googleapis.com/v1")) 24 | if err != nil { 25 | log.Errorf("Failed to create GKE Hub client: %v", err) 26 | return nil, err 27 | } 28 | 29 | // Get a list of all GKE Fleet memberships in the project. 30 | parent := fmt.Sprintf("projects/%s/locations/-", fleetProjectId) 31 | req := hubClient.Projects.Locations.Memberships.List(parent) 32 | memberships, err := req.Do() 33 | if err != nil { 34 | log.Errorf("Failed to list memberships: %v", err) 35 | return nil, err 36 | } 37 | 38 | // Get the project number for the fleet project needed later in the generate credentials request 39 | projectNumber, err := getProjectNumber(fleetProjectId) 40 | if err != nil { 41 | log.Errorf("Failed to get project number: %v", err) 42 | return nil, err 43 | } 44 | 45 | // Create a new kubeconfig. 46 | config := clientcmdapi.NewConfig() 47 | 48 | // Keep track of failed memberships 49 | failedMemberships := []string{} 50 | 51 | // Generate credentials for each membership 52 | for _, membership := range memberships.Resources { 53 | membershipName := membership.Name 54 | membershipLocation := extractLocation(membershipName) 55 | 56 | // Create a Gateway Control Client. 57 | endpoint := "connectgateway.googleapis.com" 58 | if isRegionalMembership(membershipName) { 59 | endpoint = membershipLocation + "-" + endpoint // Use regional endpoint 60 | } else { 61 | endpoint = "connectgateway.googleapis.com" // Use global endpoint 62 | } 63 | 64 | // Create a Gateway Control Client with the correct endpoint 65 | ctx2 := context.Background() 66 | gcc, err := gateway.NewGatewayControlRESTClient(ctx2, option.WithEndpoint(endpoint)) 67 | if err != nil { 68 | log.Errorf("Failed to create gateway control client for %s: %v", membershipName, err) 69 | failedMemberships = append(failedMemberships, membershipName) 70 | continue // Skip to the next membership 71 | } 72 | defer gcc.Close() 73 | 74 | log.Infof("Generating credentials for membership: %s", membershipName) 75 | 76 | // Construct the correct membership name with project number 77 | membershipName = fmt.Sprintf("projects/%s/locations/%s/memberships/%s", 78 | projectNumber, membershipLocation, extractMembershipID(membership.Name)) 79 | 80 | // Generate credentials for each membership 81 | req := &gatewaypb.GenerateCredentialsRequest{ 82 | Name: membershipName, 83 | } 84 | resp, err := gcc.GenerateCredentials(ctx, req) 85 | if err != nil { 86 | log.Errorf("Failed to generate credentials for membership %s: Endpoint: %s, Error: %v", membershipName, endpoint, err) 87 | failedMemberships = append(failedMemberships, membershipName) 88 | continue // Skip to the next membership 89 | } 90 | 91 | // Get the kubeconfig from the response 92 | kc := resp.GetKubeconfig() 93 | 94 | // Parse the kubeconfig 95 | parsedConfig, err := clientcmd.Load(kc) 96 | if err != nil { 97 | log.Errorf("Failed to load kubeconfig for membership %s: %v", membershipName, err) 98 | return nil, err 99 | } 100 | 101 | // Merge the parsed config into the main config 102 | for key, context := range parsedConfig.Contexts { 103 | config.Contexts[key] = context 104 | } 105 | for key, cluster := range parsedConfig.Clusters { 106 | config.Clusters[key] = cluster 107 | } 108 | for key, authInfo := range parsedConfig.AuthInfos { 109 | config.AuthInfos[key] = authInfo 110 | } 111 | } 112 | 113 | // Write the kubeconfig to a file. 114 | err = clientcmd.WriteToFile(*config, "kubeconfig") 115 | if err != nil { 116 | log.Errorf("Failed to write kubeconfig: %v", err) 117 | return nil, err 118 | } 119 | if len(failedMemberships) > 0 { 120 | log.Warnf("Failed to generate credentials for the following memberships: %v", failedMemberships) 121 | } else { 122 | log.Info("Kubeconfig generated successfully.") 123 | } 124 | return config, err 125 | } 126 | 127 | func isRegionalMembership(membership string) bool { 128 | // A membership is regional if the membership.name string does not contain "locations/global" 129 | return !strings.Contains(membership, "locations/global") 130 | } 131 | 132 | func extractLocation(path string) string { 133 | parts := strings.Split(path, "/") 134 | for i, part := range parts { 135 | if part == "locations" && i+1 < len(parts) { 136 | return parts[i+1] 137 | } 138 | } 139 | return "" 140 | } 141 | 142 | func extractMembershipID(membershipName string) string { 143 | parts := strings.Split(membershipName, "/") 144 | return parts[len(parts)-1] 145 | } 146 | 147 | func getProjectNumber(projectID string) (string, error) { 148 | ctx := context.Background() 149 | crmService, err := cloudresourcemanager.NewService(ctx) 150 | if err != nil { 151 | return "", fmt.Errorf("failed to create Resource Manager client: %v", err) 152 | } 153 | 154 | project, err := crmService.Projects.Get(projectID).Do() 155 | if err != nil { 156 | return "", fmt.Errorf("failed to get project: %v", err) 157 | } 158 | 159 | return fmt.Sprintf("%d", project.ProjectNumber), nil 160 | } 161 | -------------------------------------------------------------------------------- /cli/pkg/lifecycle/tf_delete.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package lifecycle 18 | 19 | import ( 20 | "context" 21 | "log" 22 | "os" 23 | 24 | "github.com/hashicorp/terraform-exec/tfexec" 25 | "github.com/hashicorp/terraform-exec/tfinstall" 26 | ) 27 | 28 | func DestroyTF(tfDir string) { 29 | tmpDir, err := os.MkdirTemp("", "tfinstall") 30 | if err != nil { 31 | log.Fatalf("error creating temp dir: %s", err) 32 | } 33 | defer os.RemoveAll(tmpDir) 34 | 35 | execPath, err := tfinstall.Find(context.Background(), tfinstall.ExactVersion("1.9.5", tmpDir)) 36 | if err != nil { 37 | log.Fatalf("error locating Terraform binary: %s", err) 38 | } 39 | 40 | tf, err := tfexec.NewTerraform(tfDir, execPath) 41 | if err != nil { 42 | log.Fatalf("error running NewTerraform: %s", err) 43 | } 44 | 45 | tf.SetStdout(os.Stdout) 46 | 47 | err = tf.Destroy(context.Background(), tfexec.VarFile("../terraform.tfvars")) 48 | if err != nil { 49 | log.Fatalf("error running Destroy: %s", err) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /cli/pkg/lifecycle/tf_deploy.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright © 2020 Google Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package lifecycle 18 | 19 | import ( 20 | "context" 21 | "os" 22 | 23 | log "github.com/sirupsen/logrus" 24 | 25 | "github.com/hashicorp/terraform-exec/tfexec" 26 | "github.com/hashicorp/terraform-exec/tfinstall" 27 | ) 28 | 29 | func InitTF(tfDir string) { 30 | tmpDir, err := os.MkdirTemp("", "tfinstall") 31 | if err != nil { 32 | log.Fatalf("error creating temp dir: %s", err) 33 | } 34 | defer os.RemoveAll(tmpDir) 35 | 36 | execPath, err := tfinstall.Find(context.Background(), tfinstall.ExactVersion("1.9.5", tmpDir)) 37 | if err != nil { 38 | log.Fatalf("error locating Terraform binary: %s", err) 39 | } 40 | 41 | tf, err := tfexec.NewTerraform(tfDir, execPath) 42 | if err != nil { 43 | log.Fatalf("error running NewTerraform: %s", err) 44 | } 45 | 46 | tf.SetStdout(log.StandardLogger().Out) 47 | 48 | err = tf.Init(context.Background(), tfexec.Upgrade(true)) 49 | if err != nil { 50 | log.Fatalf("error running Init: %s", err) 51 | } 52 | 53 | state, err := tf.Show(context.Background()) 54 | if err != nil { 55 | log.Fatalf("error running Show: %s", err) 56 | } 57 | 58 | log.Println(state.FormatVersion) // "0.1" 59 | 60 | plan, err := tf.Plan(context.Background(), tfexec.VarFile("../terraform.tfvars")) 61 | if err != nil { 62 | log.Fatalf("error running Plan: %s", err) 63 | } 64 | log.Println(plan) 65 | } 66 | 67 | func ApplyTF(tfDir string) { 68 | tmpDir, err := os.MkdirTemp("", "tfinstall") 69 | if err != nil { 70 | log.Fatalf("error creating temp dir: %s", err) 71 | } 72 | defer os.RemoveAll(tmpDir) 73 | 74 | execPath, err := tfinstall.Find(context.Background(), tfinstall.ExactVersion("1.9.5", tmpDir)) 75 | if err != nil { 76 | log.Fatalf("error locating Terraform binary: %s", err) 77 | } 78 | 79 | tf, err := tfexec.NewTerraform(tfDir, execPath) 80 | if err != nil { 81 | log.Fatalf("error running NewTerraform: %s", err) 82 | } 83 | 84 | tf.SetStdout(os.Stdout) 85 | 86 | err = tf.Apply(context.Background(), tfexec.VarFile("../terraform.tfvars")) 87 | if err != nil { 88 | log.Fatalf("error running Apply: %s", err) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /demos/fleets/config.yaml: -------------------------------------------------------------------------------- 1 | ## Global 2 | sendAnalytics: true 3 | terraformState: cloud # local, cloud 4 | 5 | ## VPC Build 6 | vpcConfig: 7 | vpcName: "prod-3" 8 | vpcType: "standalone" # standalone, shared 9 | vpcProjectId: "MYPROJECT" 10 | vpcPodCIDRName: "mypodcidr" 11 | vpcSvcCIDRName: "mysvccidr" 12 | 13 | ## FleetBuild 14 | configSyncRepo: "default-config-sync-repo" 15 | configSyncBranch: "main" 16 | configSyncDir: "/default-configs" 17 | 18 | ## Cluster Build 19 | authenticatorSecurityGroup: "gke-security-groups@MYCLOUDIDENTITYGROUP" 20 | clustersProjectId: "MYPROJECT" 21 | fleetProjectId: "MYPROJECT" 22 | privateEndpoint: true 23 | releaseChannel: REGULAR 24 | initialNodeCount: 1 25 | minNodeCount: 1 26 | maxNodeCount: 10 27 | defaultNodepoolOS: cos 28 | tfModuleRepo: "github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 29 | tfModuleBranch: "main" 30 | 31 | clustersConfig: # a list of one or more clusters, each with their own config 32 | - clusterName: "gke-ap-central-00" 33 | region: "us-central1" 34 | # zones: ["us-central1-b"] 35 | subnetName: "us-central1" 36 | - clusterName: "gke-ap-east-00" 37 | region: "us-east1" 38 | # zones: ["us-east1-b"] 39 | subnetName: "us-east1" 40 | # - clusterName: "gke-west" 41 | # region: "us-west1" 42 | # zones: ["us-west1-b"] 43 | # subnetName: "us-west1" 44 | # - clusterName: "gke-eu-north" 45 | # region: "europe-north1" 46 | # zones: ["europe-north1-c"] 47 | # subnetName: "europe-north1" 48 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-gcp-gateways/backendconfig.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cloud.google.com/v1 2 | kind: BackendConfig 3 | metadata: 4 | name: asm-ingress-xlb-config 5 | namespace: asm-gateways 6 | spec: 7 | healthCheck: 8 | requestPath: /healthz/ready 9 | port: 15021 10 | type: HTTP -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-gcp-gateways/cloud-armor-backendpolicy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.gke.io/v1 2 | kind: GCPBackendPolicy 3 | metadata: 4 | name: cloud-armor-backendpolicy 5 | namespace: asm-gateways 6 | spec: 7 | default: 8 | securityPolicy: edge-fw-policy 9 | targetRef: 10 | group: net.gke.io 11 | kind: ServiceImport 12 | name: asm-ingress-gateway-xlb 13 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-gcp-gateways/default-httproute-redirect.yaml: -------------------------------------------------------------------------------- 1 | kind: HTTPRoute 2 | apiVersion: gateway.networking.k8s.io/v1beta1 3 | metadata: 4 | name: http-to-https-redirect-httproute 5 | namespace: asm-gateways 6 | annotations: 7 | configsync.gke.io/cluster-name-selector: us-central1_gke-ap-admin-cp-00 8 | spec: 9 | parentRefs: 10 | - name: external-http 11 | namespace: asm-gateways 12 | sectionName: http 13 | rules: 14 | - filters: 15 | - type: RequestRedirect 16 | requestRedirect: 17 | scheme: https 18 | statusCode: 301 19 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-gcp-gateways/default-httproute.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: gateway.networking.k8s.io/v1beta1 2 | kind: HTTPRoute 3 | metadata: 4 | name: default-httproute 5 | namespace: asm-gateways 6 | annotations: 7 | configsync.gke.io/cluster-name-selector: us-central1_gke-ap-admin-cp-00 8 | spec: 9 | parentRefs: 10 | - name: external-http 11 | namespace: asm-gateways 12 | sectionName: https 13 | rules: 14 | - backendRefs: 15 | - group: net.gke.io 16 | kind: ServiceImport 17 | name: asm-ingress-gateway-xlb 18 | port: 443 19 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-gcp-gateways/frontend-gateway.yaml: -------------------------------------------------------------------------------- 1 | kind: Gateway 2 | apiVersion: gateway.networking.k8s.io/v1beta1 3 | metadata: 4 | name: external-http 5 | namespace: asm-gateways 6 | annotations: 7 | networking.gke.io/certmap: mcg-cert-map 8 | configsync.gke.io/cluster-name-selector: us-central1_gke-ap-admin-cp-00 9 | spec: 10 | gatewayClassName: gke-l7-global-external-managed-mc 11 | listeners: 12 | - name: http # list the port only so we can redirect any incoming http requests to https 13 | protocol: HTTP 14 | port: 80 15 | - name: https 16 | protocol: HTTPS 17 | port: 443 18 | allowedRoutes: 19 | kinds: 20 | - kind: HTTPRoute 21 | addresses: 22 | - type: NamedAddress 23 | value: whereami-ip 24 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-gcp-gateways/ingress-gateway-healthcheck.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.gke.io/v1 2 | kind: HealthCheckPolicy 3 | metadata: 4 | name: ingress-gateway-healthcheck 5 | namespace: asm-gateways 6 | annotations: 7 | configsync.gke.io/cluster-name-selector: us-central1_gke-ap-admin-cp-00 8 | spec: 9 | default: 10 | config: 11 | httpHealthCheck: 12 | port: 15021 13 | portSpecification: USE_FIXED_PORT 14 | requestPath: /healthz/ready 15 | type: HTTP 16 | targetRef: 17 | group: net.gke.io 18 | kind: ServiceImport 19 | name: asm-ingress-gateway-xlb 20 | namespace: asm-gateways 21 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-ingress-gateway/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: asm-gateways 5 | labels: 6 | istio-injection: enabled -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-ingress-gateway/pubic-gw-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: asm-ingress-gateway 5 | namespace: asm-gateways 6 | --- 7 | apiVersion: apps/v1 8 | kind: Deployment 9 | metadata: 10 | name: asm-ingress-gateway-xlb 11 | namespace: asm-gateways 12 | spec: 13 | selector: 14 | matchLabels: 15 | asm: asm-ingress-gateway-xlb 16 | template: 17 | metadata: 18 | annotations: 19 | inject.istio.io/templates: gateway 20 | labels: 21 | asm: asm-ingress-gateway-xlb 22 | # asm.io/rev: ${ASM_LABEL} # This is required only if the namespace is not labeled. 23 | spec: 24 | containers: 25 | - name: istio-proxy 26 | image: auto 27 | env: 28 | - name: ISTIO_META_UNPRIVILEGED_POD 29 | value: "true" 30 | ports: 31 | - containerPort: 8080 32 | protocol: TCP 33 | resources: 34 | limits: 35 | cpu: 2000m 36 | memory: 1024Mi 37 | requests: 38 | cpu: 100m 39 | memory: 128Mi 40 | securityContext: 41 | allowPrivilegeEscalation: false 42 | capabilities: 43 | drop: 44 | - all 45 | privileged: false 46 | readOnlyRootFilesystem: true 47 | volumeMounts: 48 | - mountPath: "/var/secrets" 49 | name: certs 50 | securityContext: 51 | fsGroup: 1337 52 | runAsGroup: 1337 53 | runAsNonRoot: true 54 | runAsUser: 1337 55 | serviceAccountName: asm-ingress-gateway 56 | volumes: 57 | - name: certs 58 | csi: 59 | driver: secrets-store-gke.csi.k8s.io 60 | readOnly: true 61 | volumeAttributes: 62 | secretProviderClass: asm-gateway-ss-cert 63 | --- 64 | apiVersion: secrets-store.csi.x-k8s.io/v1 65 | kind: SecretProviderClass 66 | metadata: 67 | name: asm-gateway-ss-cert 68 | namespace: asm-gateways 69 | spec: 70 | provider: gke 71 | parameters: 72 | secrets: | 73 | - resourceName: "projects/MYPROJECT/secrets/edge2mesh-credential-crt/versions/1" 74 | path: "edge2mesh-credential.crt" 75 | - resourceName: "projects/MYPROJECT/secrets/edge2mesh-credential-key/versions/1" 76 | path: "edge2mesh-credential.key" 77 | --- 78 | apiVersion: autoscaling/v2 79 | kind: HorizontalPodAutoscaler 80 | metadata: 81 | name: asm-ingress-gateway 82 | namespace: asm-gateways 83 | spec: 84 | maxReplicas: 5 85 | minReplicas: 1 86 | metrics: 87 | - type: Resource 88 | resource: 89 | name: cpu 90 | target: 91 | type: Utilization 92 | averageUtilization: 50 93 | scaleTargetRef: 94 | apiVersion: apps/v1 95 | kind: Deployment 96 | name: asm-ingress-gateway-xlb -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-ingress-gateway/public-gw.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1alpha3 2 | kind: Gateway 3 | metadata: 4 | name: asm-ingress-gateway-xlb 5 | namespace: asm-gateways 6 | # annotations: 7 | # configmanagement.gke.io/cluster-selector: selector-prod 8 | spec: 9 | selector: 10 | asm: asm-ingress-gateway-xlb # use ASM external ingress gateway 11 | servers: 12 | - port: 13 | number: 443 14 | name: https 15 | protocol: HTTPS 16 | hosts: 17 | - '*' # IMPORTANT: Must use wildcard here when using SSL, see note below 18 | tls: 19 | mode: SIMPLE 20 | serverCertificate: "/var/secrets/edge2mesh-credential.crt" 21 | privateKey: "/var/secrets/edge2mesh-credential.key" -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-ingress-gateway/role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: Role 3 | metadata: 4 | name: asm-ingress-gateway-sds 5 | namespace: asm-gateways 6 | rules: 7 | - apiGroups: [""] 8 | resources: ["secrets"] 9 | verbs: ["get", "watch", "list"] 10 | --- 11 | apiVersion: rbac.authorization.k8s.io/v1 12 | kind: RoleBinding 13 | metadata: 14 | name: asm-ingress-gateway-sds 15 | namespace: asm-gateways 16 | roleRef: 17 | apiGroup: rbac.authorization.k8s.io 18 | kind: Role 19 | name: asm-ingress-gateway-sds 20 | subjects: 21 | - kind: ServiceAccount 22 | name: asm-ingress-gateway -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-ingress-gateway/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: asm-ingress-gateway-xlb 5 | namespace: asm-gateways 6 | spec: 7 | type: ClusterIP 8 | selector: 9 | asm: asm-ingress-gateway-xlb 10 | ports: 11 | - name: status-port 12 | port: 15021 13 | protocol: TCP 14 | targetPort: 15021x 15 | - name: http 16 | port: 80 17 | targetPort: 8080 18 | appProtocol: HTTP 19 | - name: https 20 | port: 443 21 | targetPort: 8443 22 | appProtocol: HTTP2 23 | 24 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/asm-ingress-gateway/svc_export.yaml: -------------------------------------------------------------------------------- 1 | kind: ServiceExport 2 | apiVersion: net.gke.io/v1 3 | metadata: 4 | name: asm-ingress-gateway-xlb 5 | namespace: asm-gateways -------------------------------------------------------------------------------- /demos/fleets/default-configs/buffer/capacity-res-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: buffer 5 | --- 6 | apiVersion: apps/v1 7 | kind: Deployment 8 | metadata: 9 | name: capacity-res-deploy 10 | namespace: buffer 11 | spec: 12 | replicas: 10 13 | selector: 14 | matchLabels: 15 | app: reservation 16 | template: 17 | metadata: 18 | labels: 19 | app: reservation 20 | spec: 21 | priorityClassName: low-priority 22 | terminationGracePeriodSeconds: 0 23 | containers: 24 | - name: ubuntu 25 | image: ubuntu 26 | command: ["sleep"] 27 | args: ["infinity"] 28 | resources: 29 | requests: 30 | cpu: 2000m 31 | memory: 4000Mi -------------------------------------------------------------------------------- /demos/fleets/default-configs/buffer/priorityclasses.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: scheduling.k8s.io/v1 2 | kind: PriorityClass 3 | metadata: 4 | name: low-priority 5 | value: -10 6 | preemptionPolicy: Never 7 | globalDefault: false 8 | description: "Low priority workloads" 9 | --- 10 | apiVersion: scheduling.k8s.io/v1 11 | kind: PriorityClass 12 | metadata: 13 | name: default-priority 14 | value: 0 15 | preemptionPolicy: PreemptLowerPriority 16 | globalDefault: true 17 | description: "The global default priority." -------------------------------------------------------------------------------- /demos/fleets/default-configs/observability/cloudops-metrics-adapter.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: custom-metrics 5 | --- 6 | apiVersion: v1 7 | kind: ServiceAccount 8 | metadata: 9 | name: custom-metrics-stackdriver-adapter 10 | namespace: custom-metrics 11 | --- 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: ClusterRoleBinding 14 | metadata: 15 | name: custom-metrics:system:auth-delegator 16 | roleRef: 17 | apiGroup: rbac.authorization.k8s.io 18 | kind: ClusterRole 19 | name: system:auth-delegator 20 | subjects: 21 | - kind: ServiceAccount 22 | name: custom-metrics-stackdriver-adapter 23 | namespace: custom-metrics 24 | --- 25 | apiVersion: rbac.authorization.k8s.io/v1 26 | kind: RoleBinding 27 | metadata: 28 | name: custom-metrics-auth-reader 29 | namespace: kube-system 30 | roleRef: 31 | apiGroup: rbac.authorization.k8s.io 32 | kind: Role 33 | name: extension-apiserver-authentication-reader 34 | subjects: 35 | - kind: ServiceAccount 36 | name: custom-metrics-stackdriver-adapter 37 | namespace: custom-metrics 38 | --- 39 | apiVersion: rbac.authorization.k8s.io/v1 40 | kind: ClusterRole 41 | metadata: 42 | name: custom-metrics-resource-reader 43 | rules: 44 | - apiGroups: 45 | - "" 46 | resources: 47 | - pods 48 | - nodes 49 | - nodes/stats 50 | verbs: 51 | - get 52 | - list 53 | - watch 54 | --- 55 | apiVersion: rbac.authorization.k8s.io/v1 56 | kind: ClusterRoleBinding 57 | metadata: 58 | name: custom-metrics-resource-reader 59 | roleRef: 60 | apiGroup: rbac.authorization.k8s.io 61 | kind: ClusterRole 62 | name: custom-metrics-resource-reader 63 | subjects: 64 | - kind: ServiceAccount 65 | name: custom-metrics-stackdriver-adapter 66 | namespace: custom-metrics 67 | --- 68 | apiVersion: apps/v1 69 | kind: Deployment 70 | metadata: 71 | name: custom-metrics-stackdriver-adapter 72 | namespace: custom-metrics 73 | labels: 74 | run: custom-metrics-stackdriver-adapter 75 | k8s-app: custom-metrics-stackdriver-adapter 76 | spec: 77 | replicas: 1 78 | selector: 79 | matchLabels: 80 | run: custom-metrics-stackdriver-adapter 81 | k8s-app: custom-metrics-stackdriver-adapter 82 | template: 83 | metadata: 84 | labels: 85 | run: custom-metrics-stackdriver-adapter 86 | k8s-app: custom-metrics-stackdriver-adapter 87 | kubernetes.io/cluster-service: "true" 88 | spec: 89 | serviceAccountName: custom-metrics-stackdriver-adapter 90 | containers: 91 | - image: gcr.io/gke-release/custom-metrics-stackdriver-adapter:v0.15.1-gke.0 92 | imagePullPolicy: Always 93 | name: pod-custom-metrics-stackdriver-adapter 94 | command: 95 | - /adapter 96 | - --use-new-resource-model=true 97 | - --fallback-for-container-metrics=true 98 | resources: 99 | limits: 100 | cpu: 250m 101 | memory: 200Mi 102 | requests: 103 | cpu: 250m 104 | memory: 200Mi 105 | --- 106 | apiVersion: v1 107 | kind: Service 108 | metadata: 109 | labels: 110 | run: custom-metrics-stackdriver-adapter 111 | k8s-app: custom-metrics-stackdriver-adapter 112 | kubernetes.io/cluster-service: 'true' 113 | kubernetes.io/name: Adapter 114 | name: custom-metrics-stackdriver-adapter 115 | namespace: custom-metrics 116 | spec: 117 | ports: 118 | - port: 443 119 | protocol: TCP 120 | targetPort: 443 121 | selector: 122 | run: custom-metrics-stackdriver-adapter 123 | k8s-app: custom-metrics-stackdriver-adapter 124 | type: ClusterIP 125 | --- 126 | apiVersion: apiregistration.k8s.io/v1 127 | kind: APIService 128 | metadata: 129 | name: v1beta1.custom.metrics.k8s.io 130 | spec: 131 | insecureSkipTLSVerify: true 132 | group: custom.metrics.k8s.io 133 | groupPriorityMinimum: 100 134 | versionPriority: 100 135 | service: 136 | name: custom-metrics-stackdriver-adapter 137 | namespace: custom-metrics 138 | version: v1beta1 139 | --- 140 | apiVersion: apiregistration.k8s.io/v1 141 | kind: APIService 142 | metadata: 143 | name: v1beta2.custom.metrics.k8s.io 144 | spec: 145 | insecureSkipTLSVerify: true 146 | group: custom.metrics.k8s.io 147 | groupPriorityMinimum: 100 148 | versionPriority: 200 149 | service: 150 | name: custom-metrics-stackdriver-adapter 151 | namespace: custom-metrics 152 | version: v1beta2 153 | --- 154 | apiVersion: apiregistration.k8s.io/v1 155 | kind: APIService 156 | metadata: 157 | name: v1beta1.external.metrics.k8s.io 158 | spec: 159 | insecureSkipTLSVerify: true 160 | group: external.metrics.k8s.io 161 | groupPriorityMinimum: 100 162 | versionPriority: 100 163 | service: 164 | name: custom-metrics-stackdriver-adapter 165 | namespace: custom-metrics 166 | version: v1beta1 167 | --- 168 | apiVersion: rbac.authorization.k8s.io/v1 169 | kind: ClusterRole 170 | metadata: 171 | name: external-metrics-reader 172 | rules: 173 | - apiGroups: 174 | - "external.metrics.k8s.io" 175 | resources: 176 | - "*" 177 | verbs: 178 | - list 179 | - get 180 | - watch 181 | --- 182 | apiVersion: rbac.authorization.k8s.io/v1 183 | kind: ClusterRoleBinding 184 | metadata: 185 | name: external-metrics-reader 186 | roleRef: 187 | apiGroup: rbac.authorization.k8s.io 188 | kind: ClusterRole 189 | name: external-metrics-reader 190 | subjects: 191 | - kind: ServiceAccount 192 | name: horizontal-pod-autoscaler 193 | namespace: kube-system -------------------------------------------------------------------------------- /demos/fleets/default-configs/observability/prom-frontend.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: prometheus-frontend 5 | namespace: prod-tools 6 | # annotations: 7 | # configmanagement.gke.io/cluster-selector: selector-prod 8 | spec: 9 | replicas: 2 10 | selector: 11 | matchLabels: 12 | app: prometheus-frontend 13 | template: 14 | metadata: 15 | labels: 16 | app: prometheus-frontend 17 | spec: 18 | automountServiceAccountToken: true 19 | # nodeSelector: 20 | # kubernetes.io/os: linux 21 | # kubernetes.io/arch: amd64 22 | containers: 23 | - name: frontend 24 | image: "gke.gcr.io/prometheus-engine/frontend:v0.4.1-gke.0" 25 | args: 26 | - "--web.listen-address=:9090" 27 | - "--query.project-id=MYPROJECT" 28 | ports: 29 | - name: web 30 | containerPort: 9090 31 | readinessProbe: 32 | httpGet: 33 | path: /-/ready 34 | port: web 35 | livenessProbe: 36 | httpGet: 37 | path: /-/healthy 38 | port: web 39 | resources: 40 | requests: 41 | memory: 512Mi 42 | cpu: 250m 43 | --- 44 | apiVersion: v1 45 | kind: Service 46 | metadata: 47 | name: prometheus-frontend 48 | namespace: prod-tools 49 | spec: 50 | clusterIP: None 51 | selector: 52 | app: prometheus-frontend 53 | ports: 54 | - name: web 55 | port: 9090 -------------------------------------------------------------------------------- /demos/fleets/default-configs/roles/gateway-cluster-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | labels: 5 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 6 | name: custom:aggregate-to-edit:gateway 7 | rules: 8 | - apiGroups: 9 | - "net.gke.io" 10 | - "gateway.networking.k8s.io" 11 | resources: 12 | - "*" 13 | verbs: 14 | - "*" -------------------------------------------------------------------------------- /demos/fleets/default-configs/roles/istio-cluster-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | labels: 5 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 6 | name: custom:aggregate-to-edit:istio 7 | rules: 8 | - apiGroups: 9 | - "networking.istio.io" 10 | - "security.istio.io" 11 | resources: 12 | - "*" 13 | verbs: 14 | - "*" -------------------------------------------------------------------------------- /demos/fleets/default-configs/roles/monitoring-cluster-role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | labels: 5 | rbac.authorization.k8s.io/aggregate-to-edit: "true" 6 | name: custom:aggregate-to-edit:monitoring 7 | rules: 8 | - apiGroups: 9 | - "monitoring.googleapis.com" 10 | resources: 11 | - "*" 12 | verbs: 13 | - "*" -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/backendconfig.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: cloud.google.com/v1 2 | # kind: BackendConfig 3 | # metadata: 4 | # name: llm-gateway-config 5 | # namespace: inference 6 | # # annotations: 7 | # # configsync.gke.io/cluster-name-selector: gke-dev-us-central1-01 8 | # spec: 9 | # healthCheck: 10 | # requestPath: /healthz/ready 11 | # port: 15021 12 | # type: HTTP -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/export-global.yaml: -------------------------------------------------------------------------------- 1 | kind: ServiceExport 2 | apiVersion: net.gke.io/v1 3 | metadata: 4 | namespace: inference 5 | name: llm-service-global 6 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/gateway.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | apiVersion: v1 15 | kind: Namespace 16 | metadata: 17 | name: inference 18 | --- 19 | kind: Gateway 20 | apiVersion: gateway.networking.k8s.io/v1beta1 21 | metadata: 22 | name: inference-external-http 23 | namespace: inference 24 | annotations: 25 | configsync.gke.io/cluster-name-selector: gke-dev-us-central1-01 26 | spec: 27 | gatewayClassName: gke-l7-global-external-managed-mc 28 | listeners: 29 | - name: http 30 | protocol: HTTP 31 | port: 80 32 | allowedRoutes: 33 | kinds: 34 | - kind: HTTPRoute 35 | --- 36 | kind: HTTPRoute 37 | apiVersion: gateway.networking.k8s.io/v1beta1 38 | metadata: 39 | name: inference-route 40 | namespace: inference 41 | annotations: 42 | configsync.gke.io/cluster-name-selector: gke-dev-us-central1-01 43 | labels: 44 | gateway: inference-external-http 45 | spec: 46 | parentRefs: 47 | - name: inference-external-http 48 | namespace: inference 49 | sectionName: http 50 | rules: 51 | - backendRefs: 52 | - group: net.gke.io 53 | kind: ServiceImport 54 | name: llm-service-global 55 | port: 80 56 | # weight: 100 57 | # - group: net.gke.io 58 | # kind: ServiceImport 59 | # name: llm-service-us-east1 60 | # port: 80 61 | # weight: 0 62 | # - group: net.gke.io 63 | # kind: ServiceImport 64 | # name: llm-service-us-central1 65 | # port: 80 66 | # weight: 0 67 | # - group: net.gke.io 68 | # kind: ServiceImport 69 | # name: llm-service-us-east4 70 | # port: 80 71 | # weight: 0 -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/healthcheck.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.gke.io/v1 2 | kind: HealthCheckPolicy 3 | metadata: 4 | name: llm-gateway-healthcheck 5 | namespace: inference 6 | annotations: 7 | configsync.gke.io/cluster-name-selector: gke-dev-us-central1-01 8 | spec: 9 | default: 10 | checkIntervalSec: 15 11 | healthyThreshold: 10 12 | unhealthyThreshold: 1 13 | config: 14 | httpHealthCheck: 15 | port: 15021 16 | portSpecification: USE_FIXED_PORT 17 | requestPath: /health 18 | type: HTTP 19 | targetRef: 20 | group: net.gke.io 21 | kind: ServiceImport 22 | name: llm-service-global 23 | namespace: inference 24 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/hpa-custom-metrics.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: autoscaling/v2 2 | kind: HorizontalPodAutoscaler 3 | metadata: 4 | name: custom-metrics-gmp-hpa 5 | namespace: inference 6 | annotations: 7 | spec: 8 | scaleTargetRef: 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | name: tgi-gemma-deployment 12 | minReplicas: 1 13 | maxReplicas: 3 14 | metrics: 15 | - type: Pods 16 | pods: 17 | metric: 18 | name: prometheus.googleapis.com|tgi_queue_size|gauge 19 | target: 20 | type: AverageValue 21 | averageValue: 5 22 | # behavior: 23 | # scaleDown: 24 | # stabilizationWindowSeconds: 100 25 | # policies: 26 | # - type: Pods 27 | # value: 1 28 | # periodSeconds: 300 29 | # scaleUp: 30 | # stabilizationWindowSeconds: 100 31 | # policies: 32 | # - type: Pods 33 | # value: 2 34 | # periodSeconds: 300 35 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/inference/inference-ns-selector.yaml: -------------------------------------------------------------------------------- 1 | kind: NamespaceSelector 2 | apiVersion: configmanagement.gke.io/v1 3 | metadata: 4 | name: inference 5 | spec: 6 | mode: dynamic 7 | selector: 8 | matchLabels: 9 | fleet.gke.io/fleet-scope: team-llm 10 | kubernetes.io/metadata.name: inference -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/inference/inference-repo-sync.yaml: -------------------------------------------------------------------------------- 1 | kind: RepoSync 2 | apiVersion: configsync.gke.io/v1beta1 3 | metadata: 4 | name: inference 5 | annotations: 6 | configmanagement.gke.io/namespace-selector: inference 7 | # configsync.gke.io/deletion-propagation-policy: Foreground 8 | spec: 9 | sourceFormat: unstructured 10 | git: 11 | repo: "https://source.developers.google.com/p/MYPROJECT}/r/default-config-sync-repo" 12 | branch: main 13 | dir: /teams/llm 14 | auth: gcpserviceaccount 15 | gcpServiceAccountEmail: cs-service-account@MYPROJECT.iam.gserviceaccount.com 16 | --- 17 | kind: RoleBinding 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | metadata: 20 | name: inference 21 | annotations: 22 | configmanagement.gke.io/namespace-selector: inference 23 | subjects: 24 | - kind: ServiceAccount 25 | name: ns-reconciler-inference-inference-9 26 | namespace: config-management-system 27 | roleRef: 28 | kind: ClusterRole 29 | name: edit 30 | apiGroup: rbac.authorization.k8s.io 31 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/llm/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: llm-service-global 5 | namespace: inference 6 | spec: 7 | selector: 8 | app: gemma-server 9 | type: ClusterIP 10 | ports: 11 | - name: http 12 | port: 80 13 | targetPort: 8080 14 | appProtocol: HTTP 15 | - name: health 16 | protocol: TCP 17 | port: 15021 18 | targetPort: 15021 19 | appProtocol: HTTP -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/backend/backend-ns-selector.yaml: -------------------------------------------------------------------------------- 1 | kind: NamespaceSelector 2 | apiVersion: configmanagement.gke.io/v1 3 | metadata: 4 | name: whereami-backend 5 | spec: 6 | mode: dynamic 7 | selector: 8 | matchLabels: 9 | fleet.gke.io/fleet-scope: team-whereami 10 | kubernetes.io/metadata.name: whereami-backend -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/backend/backend-repo-sync.yaml: -------------------------------------------------------------------------------- 1 | kind: RepoSync 2 | apiVersion: configsync.gke.io/v1beta1 3 | metadata: 4 | name: whereami-backend 5 | annotations: 6 | configmanagement.gke.io/namespace-selector: whereami-backend 7 | # configsync.gke.io/deletion-propagation-policy: Foreground 8 | spec: 9 | sourceFormat: unstructured 10 | git: 11 | repo: "https://source.developers.google.com/p/MYPROJECT}/r/default-config-sync-repo" 12 | branch: main 13 | dir: /teams/whereami/backend 14 | auth: gcpserviceaccount 15 | gcpServiceAccountEmail: cs-service-account@MYPROJECT.iam.gserviceaccount.com 16 | --- 17 | kind: RoleBinding 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | metadata: 20 | name: whereami-backend 21 | annotations: 22 | configmanagement.gke.io/namespace-selector: whereami-backend 23 | subjects: 24 | - kind: ServiceAccount 25 | name: ns-reconciler-whereami-backend-whereami-backend-16 26 | namespace: config-management-system 27 | roleRef: 28 | kind: ClusterRole 29 | name: edit 30 | apiGroup: rbac.authorization.k8s.io 31 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/frontend/frontend-ns-selector.yaml: -------------------------------------------------------------------------------- 1 | kind: NamespaceSelector 2 | apiVersion: configmanagement.gke.io/v1 3 | metadata: 4 | name: whereami-frontend 5 | spec: 6 | mode: dynamic 7 | selector: 8 | matchLabels: 9 | fleet.gke.io/fleet-scope: team-whereami 10 | kubernetes.io/metadata.name: whereami-frontend -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/frontend/frontend-repo-sync.yaml: -------------------------------------------------------------------------------- 1 | kind: RepoSync 2 | apiVersion: configsync.gke.io/v1beta1 3 | metadata: 4 | name: whereami-frontend 5 | annotations: 6 | configmanagement.gke.io/namespace-selector: whereami-frontend 7 | # configsync.gke.io/deletion-propagation-policy: Foreground 8 | spec: 9 | sourceFormat: unstructured 10 | git: 11 | repo: "https://source.developers.google.com/p/MYPROJECT}/r/default-config-sync-repo" 12 | branch: main 13 | dir: /teams/whereami/frontend 14 | auth: gcpserviceaccount 15 | gcpServiceAccountEmail: cs-service-account@MYPROJECT.iam.gserviceaccount.com 16 | --- 17 | kind: RoleBinding 18 | apiVersion: rbac.authorization.k8s.io/v1 19 | metadata: 20 | name: whereami-frontend 21 | annotations: 22 | configmanagement.gke.io/namespace-selector: whereami-frontend 23 | subjects: 24 | - kind: ServiceAccount 25 | name: ns-reconciler-whereami-frontend-whereami-frontend-17 26 | namespace: config-management-system 27 | roleRef: 28 | kind: ClusterRole 29 | name: edit 30 | apiGroup: rbac.authorization.k8s.io 31 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/network-policy.yaml: -------------------------------------------------------------------------------- 1 | # ### from https://kubernetes.io/docs/concepts/services-networking/network-policies/#default-deny-all-ingress-traffic 2 | # apiVersion: networking.k8s.io/v1 3 | # kind: NetworkPolicy 4 | # metadata: 5 | # name: whereami-default-deny-ingress 6 | # annotations: 7 | # configmanagement.gke.io/namespace-selector: whereami-team-scope 8 | # spec: 9 | # podSelector: {} 10 | # policyTypes: 11 | # - Ingress -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/team-resource-quota.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: constraints.gatekeeper.sh/v1beta1 2 | # kind: TeamResourceQuota 3 | # metadata: 4 | # name: whereami-resource-quota 5 | # annotations: 6 | # # configmanagement.gke.io/namespace-selector: whereami-team-scope 7 | # spec: 8 | # enforcementAction: deny 9 | # match: 10 | # kinds: 11 | # - apiGroups: 12 | # - "" 13 | # kinds: 14 | # - Pod 15 | # - ResourceQuota 16 | # parameters: 17 | # fields: 18 | # limits.cpu: 1.5k 19 | # limits.memory: 128Gi 20 | # requests.cpu: 1k 21 | # requests.memory: 96Gi 22 | # requests.storage: 128Gi 23 | # services.loadbalancers: "10" 24 | # services.nodeports: "10" 25 | # selector: 26 | # label: 27 | # key: teamresourcequota 28 | # value: team-whereami 29 | -------------------------------------------------------------------------------- /demos/fleets/default-configs/tenants/whereami/team-selector.yaml: -------------------------------------------------------------------------------- 1 | kind: NamespaceSelector 2 | apiVersion: configmanagement.gke.io/v1 3 | metadata: 4 | name: whereami-team-scope 5 | spec: 6 | mode: dynamic 7 | selector: 8 | matchLabels: 9 | fleet.gke.io/fleet-scope: team-whereami -------------------------------------------------------------------------------- /demos/fleets/default-configs/tools/secret-provider-class.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: secrets-store.csi.x-k8s.io/v1 2 | kind: SecretProviderClass 3 | metadata: 4 | name: asm-gateway-ss-cert 5 | # namespace: asm-gateways 6 | spec: 7 | provider: gke 8 | parameters: 9 | secrets: | 10 | - resourceName: "projects/MYPROJECT/secrets/edge2mesh-credential-crt/versions/1" 11 | path: "edge2mesh-credential.crt" 12 | - resourceName: "projects/MYPROJECT/secrets/edge2mesh-credential-key/versions/1" 13 | path: "edge2mesh-credential.key" -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/export-us-central.yaml: -------------------------------------------------------------------------------- 1 | # # Copyright 2024 Google LLC 2 | # # 3 | # # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # # you may not use this file except in compliance with the License. 5 | # # You may obtain a copy of the License at 6 | # # 7 | # # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # # Unless required by applicable law or agreed to in writing, software 10 | # # distributed under the License is distributed on an "AS IS" BASIS, 11 | # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # # See the License for the specific language governing permissions and 13 | # # limitations under the License. 14 | 15 | # --- 16 | kind: ServiceExport 17 | apiVersion: net.gke.io/v1 18 | metadata: 19 | namespace: inference 20 | name: llm-service-us-central1 21 | annotations: 22 | configsync.gke.io/cluster-name-selector: gke-dev-us-central1-00 23 | --- 24 | apiVersion: v1 25 | kind: Service 26 | metadata: 27 | name: llm-service-us-central1 28 | annotations: 29 | configsync.gke.io/cluster-name-selector: gke-dev-us-central1-00 30 | spec: 31 | selector: 32 | app: gemma-server 33 | type: ClusterIP 34 | ports: 35 | - protocol: TCP 36 | port: 80 37 | targetPort: 8080 38 | -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/export-us-east.yaml: -------------------------------------------------------------------------------- 1 | # # Copyright 2024 Google LLC 2 | # # 3 | # # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # # you may not use this file except in compliance with the License. 5 | # # You may obtain a copy of the License at 6 | # # 7 | # # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # # Unless required by applicable law or agreed to in writing, software 10 | # # distributed under the License is distributed on an "AS IS" BASIS, 11 | # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # # See the License for the specific language governing permissions and 13 | # # limitations under the License. 14 | 15 | kind: ServiceExport 16 | apiVersion: net.gke.io/v1 17 | metadata: 18 | namespace: inference 19 | name: llm-service-us-east1 20 | annotations: 21 | configsync.gke.io/cluster-name-selector: gke-dev-us-east1-01 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: llm-service-us-east1 27 | annotations: 28 | configsync.gke.io/cluster-name-selector: gke-dev-us-east1-01 29 | spec: 30 | selector: 31 | app: gemma-server 32 | type: ClusterIP 33 | ports: 34 | - protocol: TCP 35 | port: 80 36 | targetPort: 8080 -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/export-us-east4.yaml: -------------------------------------------------------------------------------- 1 | # # Copyright 2024 Google LLC 2 | # # 3 | # # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # # you may not use this file except in compliance with the License. 5 | # # You may obtain a copy of the License at 6 | # # 7 | # # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # # Unless required by applicable law or agreed to in writing, software 10 | # # distributed under the License is distributed on an "AS IS" BASIS, 11 | # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # # See the License for the specific language governing permissions and 13 | # # limitations under the License. 14 | 15 | kind: ServiceExport 16 | apiVersion: net.gke.io/v1 17 | metadata: 18 | namespace: inference 19 | name: llm-service-us-east4 20 | annotations: 21 | configsync.gke.io/cluster-name-selector: gke-dev-us-east4-00 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | name: llm-service-us-east4 27 | annotations: 28 | configsync.gke.io/cluster-name-selector: gke-dev-us-east4-00 29 | spec: 30 | selector: 31 | app: gemma-server 32 | type: ClusterIP 33 | ports: 34 | - protocol: TCP 35 | port: 80 36 | targetPort: 8080 -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/hpa-custom-metrics.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: autoscaling/v2 2 | # kind: HorizontalPodAutoscaler 3 | # metadata: 4 | # name: custom-metrics-gmp-hpa 5 | # namespace: inference 6 | # spec: 7 | # scaleTargetRef: 8 | # apiVersion: apps/v1 9 | # kind: Deployment 10 | # name: tgi-gemma-deployment 11 | # minReplicas: 1 12 | # maxReplicas: 3 13 | # metrics: 14 | # - type: Pods 15 | # pods: 16 | # metric: 17 | # name: prometheus.googleapis.com|tgi_queue_size|gauge 18 | # target: 19 | # type: AverageValue 20 | # averageValue: 1 -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/monitoring.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: monitoring.googleapis.com/v1 16 | kind: PodMonitoring 17 | metadata: 18 | name: llm-prometheus 19 | namespace: inference 20 | labels: 21 | app.kubernetes.io/name: llm-prometheus 22 | spec: 23 | endpoints: 24 | - port: web 25 | scheme: http 26 | interval: 5s 27 | path: /metrics 28 | selector: 29 | matchLabels: 30 | app: gemma-server -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/service.yaml: -------------------------------------------------------------------------------- 1 | # # Copyright 2018 The Knative Authors 2 | # # 3 | # # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # # you may not use this file except in compliance with the License. 5 | # # You may obtain a copy of the License at 6 | # # 7 | # # https://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # # Unless required by applicable law or agreed to in writing, software 10 | # # distributed under the License is distributed on an "AS IS" BASIS, 11 | # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # # See the License for the specific language governing permissions and 13 | # # limitations under the License. 14 | # apiVersion: serving.knative.dev/v1 15 | # kind: Service 16 | # metadata: 17 | # name: autoscale-go 18 | # namespace: inference 19 | # spec: 20 | # template: 21 | # metadata: 22 | # annotations: 23 | # # Target 10 in-flight-requests per pod. 24 | # autoscaling.knative.dev/target: "10" 25 | # spec: 26 | # containers: 27 | # - image: ghcr.io/knative/autoscale-go:latest 28 | -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/tgi-2b-it-1.1-central.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | name: tgi-gemma-deployment 19 | namespace: inference 20 | annotations: 21 | configsync.gke.io/cluster-name-selector: gke-dev-us-central1-01 22 | labels: 23 | app: gemma-server 24 | spec: 25 | replicas: 1 26 | selector: 27 | matchLabels: 28 | app: gemma-server 29 | template: 30 | metadata: 31 | labels: 32 | app: gemma-server 33 | ai.gke.io/model: gemma-2b-1.1-it 34 | ai.gke.io/inference-server: text-generation-inference 35 | examples.ai.gke.io/source: user-guide 36 | spec: 37 | containers: 38 | - name: llm-healthcheck 39 | image: us-docker.pkg.dev/fleet-dev-1/llm-healthcheck/llm-healthcheck-v0.0.12 40 | resources: 41 | requests: 42 | cpu: "100m" 43 | memory: "128Mi" 44 | limits: 45 | cpu: "100m" 46 | memory: "128Mi" 47 | env: 48 | - name: METRICS_ENDPOINT 49 | value: "http://localhost:8080/metrics" 50 | - name: METRIC_THRESHOLD 51 | value: "10" 52 | - name: METRIC_TO_CHECK 53 | value: "tgi_queue_size" 54 | - name: APP_PORT 55 | value: "15021" 56 | ports: 57 | - name: healthcheck 58 | containerPort: 15021 59 | - name: inference-server 60 | image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-hf-tgi-serve:20240328_0936_RC01 61 | resources: 62 | requests: 63 | cpu: "2" 64 | memory: "7Gi" 65 | ephemeral-storage: "20Gi" 66 | nvidia.com/gpu: 1 67 | limits: 68 | cpu: "2" 69 | memory: "7Gi" 70 | ephemeral-storage: "20Gi" 71 | nvidia.com/gpu: 1 72 | args: 73 | - --model-id=$(MODEL_ID) 74 | - --num-shard=1 75 | env: 76 | - name: MODEL_ID 77 | value: google/gemma-1.1-2b-it 78 | - name: PORT 79 | value: "8080" 80 | - name: HUGGING_FACE_HUB_TOKEN 81 | valueFrom: 82 | secretKeyRef: 83 | name: hf-secret 84 | key: hf_api_token 85 | volumeMounts: 86 | - mountPath: /dev/shm 87 | name: dshm 88 | ports: 89 | - name: web 90 | containerPort: 8080 91 | volumes: 92 | - name: dshm 93 | emptyDir: 94 | medium: Memory 95 | # nodeSelector: 96 | # cloud.google.com/gke-accelerator: nvidia-l4 97 | -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/tgi-2b-it-1.1-east.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | name: tgi-gemma-deployment 19 | namespace: inference 20 | annotations: 21 | configsync.gke.io/cluster-name-selector: gke-std-dev-us-east1-00 22 | labels: 23 | app: gemma-server 24 | spec: 25 | # replicas: 1 26 | selector: 27 | matchLabels: 28 | app: gemma-server 29 | template: 30 | metadata: 31 | labels: 32 | app: gemma-server 33 | ai.gke.io/model: gemma-2b-1.1-it 34 | ai.gke.io/inference-server: text-generation-inference 35 | examples.ai.gke.io/source: user-guide 36 | spec: 37 | containers: 38 | - name: llm-healthcheck 39 | image: us-docker.pkg.dev/fleet-dev-1/llm-healthcheck/llm-healthcheck-v0.0.12 40 | resources: 41 | requests: 42 | cpu: "100m" 43 | memory: "128Mi" 44 | limits: 45 | cpu: "100m" 46 | memory: "128Mi" 47 | env: 48 | - name: METRICS_ENDPOINT 49 | value: "http://localhost:8080/metrics" 50 | - name: METRIC_THRESHOLD 51 | value: "1000" 52 | - name: METRIC_TO_CHECK 53 | value: "tgi_queue_size" 54 | - name: APP_PORT 55 | value: "15021" 56 | ports: 57 | - name: healthcheck 58 | containerPort: 15021 59 | - name: inference-server 60 | image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-hf-tgi-serve:20240328_0936_RC01 61 | resources: 62 | requests: 63 | cpu: "2" 64 | memory: "7Gi" 65 | ephemeral-storage: "20Gi" 66 | nvidia.com/gpu: 1 67 | limits: 68 | cpu: "2" 69 | memory: "7Gi" 70 | ephemeral-storage: "20Gi" 71 | nvidia.com/gpu: 1 72 | args: 73 | - --model-id=$(MODEL_ID) 74 | - --num-shard=1 75 | env: 76 | - name: MODEL_ID 77 | value: google/gemma-1.1-2b-it 78 | - name: PORT 79 | value: "8080" 80 | - name: HUGGING_FACE_HUB_TOKEN 81 | valueFrom: 82 | secretKeyRef: 83 | name: hf-secret 84 | key: hf_api_token 85 | volumeMounts: 86 | - mountPath: /dev/shm 87 | name: dshm 88 | ports: 89 | - name: web 90 | containerPort: 8080 91 | volumes: 92 | - name: dshm 93 | emptyDir: 94 | medium: Memory 95 | # nodeSelector: 96 | # cloud.google.com/gke-accelerator: nvidia-tesla-a100 97 | -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/tgi-2b-it-1.1-west.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | name: tgi-gemma-deployment 19 | namespace: inference 20 | annotations: 21 | configsync.gke.io/cluster-name-selector: gke-ap-dev-us-west1-00 22 | labels: 23 | app: gemma-server 24 | spec: 25 | selector: 26 | matchLabels: 27 | app: gemma-server 28 | template: 29 | metadata: 30 | labels: 31 | app: gemma-server 32 | ai.gke.io/model: gemma-2b-1.1-it 33 | ai.gke.io/inference-server: text-generation-inference 34 | examples.ai.gke.io/source: user-guide 35 | spec: 36 | containers: 37 | - name: llm-healthcheck 38 | image: us-docker.pkg.dev/fleet-dev-1/llm-healthcheck/llm-healthcheck-v0.0.12 39 | resources: 40 | requests: 41 | cpu: "100m" 42 | memory: "128Mi" 43 | limits: 44 | cpu: "100m" 45 | memory: "128Mi" 46 | env: 47 | - name: METRICS_ENDPOINT 48 | value: "http://localhost:8080/metrics" 49 | - name: METRIC_THRESHOLD 50 | value: "10" 51 | - name: METRIC_TO_CHECK 52 | value: "tgi_queue_size" 53 | - name: APP_PORT 54 | value: "15021" 55 | ports: 56 | - name: healthcheck 57 | containerPort: 15021 58 | - name: inference-server 59 | image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-hf-tgi-serve:20240328_0936_RC01 60 | resources: 61 | requests: 62 | cpu: "2" 63 | memory: "7Gi" 64 | ephemeral-storage: "20Gi" 65 | nvidia.com/gpu: 1 66 | limits: 67 | cpu: "2" 68 | memory: "7Gi" 69 | ephemeral-storage: "20Gi" 70 | nvidia.com/gpu: 1 71 | args: 72 | - --model-id=$(MODEL_ID) 73 | - --num-shard=1 74 | env: 75 | - name: MODEL_ID 76 | value: google/gemma-1.1-2b-it 77 | - name: PORT 78 | value: "8080" 79 | - name: HUGGING_FACE_HUB_TOKEN 80 | valueFrom: 81 | secretKeyRef: 82 | name: hf-secret 83 | key: hf_api_token 84 | volumeMounts: 85 | - mountPath: /dev/shm 86 | name: dshm 87 | ports: 88 | - name: web 89 | containerPort: 8080 90 | volumes: 91 | - name: dshm 92 | emptyDir: 93 | medium: Memory 94 | nodeSelector: 95 | cloud.google.com/gke-accelerator: nvidia-l4 96 | -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/tgi-2b-it-1.1.yaml: -------------------------------------------------------------------------------- 1 | # # Copyright 2024 Google LLC 2 | # # 3 | # # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # # you may not use this file except in compliance with the License. 5 | # # You may obtain a copy of the License at 6 | # # 7 | # # http://www.apache.org/licenses/LICENSE-2.0 8 | # # 9 | # # Unless required by applicable law or agreed to in writing, software 10 | # # distributed under the License is distributed on an "AS IS" BASIS, 11 | # # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # # See the License for the specific language governing permissions and 13 | # # limitations under the License. 14 | 15 | # apiVersion: apps/v1 16 | # kind: Deployment 17 | # metadata: 18 | # name: tgi-gemma-deployment 19 | # namespace: inference 20 | # labels: 21 | # app: gemma-server 22 | # spec: 23 | # replicas: 1 24 | # selector: 25 | # matchLabels: 26 | # app: gemma-server 27 | # template: 28 | # metadata: 29 | # labels: 30 | # app: gemma-server 31 | # ai.gke.io/model: gemma-2b-1.1-it 32 | # ai.gke.io/inference-server: text-generation-inference 33 | # examples.ai.gke.io/source: user-guide 34 | # spec: 35 | # containers: 36 | # - name: busybox 37 | # image: radial/busyboxplus:curl 38 | # command: ['sh', '-c', 'while true; do date; sleep 3; done'] 39 | # resources: 40 | # requests: 41 | # cpu: "100m" 42 | # memory: "128Mi" 43 | # - name: llm-healthcheck 44 | # image: us-docker.pkg.dev/fleet-dev-1/llm-healthcheck/llm-healthcheck-v0.0.12 45 | # resources: 46 | # requests: 47 | # cpu: "100m" 48 | # memory: "128Mi" 49 | # env: 50 | # - name: METRICS_ENDPOINT 51 | # value: "http://localhost:8080/metrics" 52 | # - name: METRICS_THRESHOLD 53 | # value: "15" 54 | # - name: METRIC_TO_CHECK 55 | # value: "tgi_queue_size" 56 | # - name: APP_PORT 57 | # value: "15021" 58 | # ports: 59 | # - name: healthcheck 60 | # containerPort: 15021 61 | # - name: inference-server 62 | # image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-hf-tgi-serve:20240328_0936_RC01 63 | # # readinessProbe: 64 | # # httpGet: 65 | # # path: /health 66 | # # port: 15021 # Port the llm-healthchecker is listening on 67 | # # failureThreshold: 2 68 | # # successThreshold: 2 69 | # # initialDelaySeconds: 20 # Delay before first probe 70 | # # periodSeconds: 5 71 | # resources: 72 | # requests: 73 | # cpu: "2" 74 | # memory: "7Gi" 75 | # ephemeral-storage: "20Gi" 76 | # nvidia.com/gpu: 1 77 | # limits: 78 | # cpu: "2" 79 | # memory: "7Gi" 80 | # ephemeral-storage: "20Gi" 81 | # nvidia.com/gpu: 1 82 | # args: 83 | # - --model-id=$(MODEL_ID) 84 | # - --num-shard=1 85 | # env: 86 | # - name: MODEL_ID 87 | # value: google/gemma-1.1-2b-it 88 | # - name: PORT 89 | # value: "8080" 90 | # - name: HUGGING_FACE_HUB_TOKEN 91 | # valueFrom: 92 | # secretKeyRef: 93 | # name: hf-secret 94 | # key: hf_api_token 95 | # volumeMounts: 96 | # - mountPath: /dev/shm 97 | # name: dshm 98 | # ports: 99 | # - name: web 100 | # containerPort: 8080 101 | # volumes: 102 | # - name: dshm 103 | # emptyDir: 104 | # medium: Memory 105 | # nodeSelector: 106 | # cloud.google.com/gke-accelerator: nvidia-tesla-a100 107 | -------------------------------------------------------------------------------- /demos/fleets/teams/llm/inference/tgi-kn.yaml: -------------------------------------------------------------------------------- 1 | # apiVersion: serving.knative.dev/v1 2 | # kind: Service 3 | # metadata: 4 | # name: tgi-gemma-deployment 5 | # namespace: inference 6 | # spec: 7 | # template: 8 | # metadata: 9 | # labels: 10 | # app: gemma-server 11 | # ai.gke.io/model: gemma-2b-1.1-it 12 | # ai.gke.io/inference-server: text-generation-inference 13 | # examples.ai.gke.io/source: user-guide 14 | # annotations: 15 | # autoscaling.knative.dev/min-scale: "1" 16 | # autoscaling.knative.dev/initial-scale: "1" 17 | # spec: 18 | # nodeSelector: 19 | # cloud.google.com/gke-accelerator: nvidia-tesla-a100 20 | # containers: 21 | # - name: inference-server 22 | # image: us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-hf-tgi-serve:20240328_0936_RC01 23 | # resources: 24 | # requests: 25 | # cpu: "2" 26 | # memory: "7Gi" 27 | # ephemeral-storage: "20Gi" 28 | # nvidia.com/gpu: 1 29 | # limits: 30 | # cpu: "2" 31 | # memory: "7Gi" 32 | # ephemeral-storage: "20Gi" 33 | # nvidia.com/gpu: 1 34 | # args: 35 | # - --model-id=$(MODEL_ID) 36 | # - --num-shard=1 37 | # env: 38 | # - name: MODEL_ID 39 | # value: google/gemma-1.1-2b-it 40 | # # - name: PORT 41 | # # value: "8000" 42 | # - name: HUGGING_FACE_HUB_TOKEN 43 | # valueFrom: 44 | # secretKeyRef: 45 | # name: hf-secret 46 | # key: hf_api_token 47 | # volumeMounts: 48 | # - mountPath: /dev/shm 49 | # name: dshm 50 | # ports: 51 | # - containerPort: 8080 52 | # volumes: 53 | # - name: dshm 54 | # emptyDir: 55 | # medium: Memory 56 | -------------------------------------------------------------------------------- /demos/fleets/teams/whereami/backend/app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | # annotations: 5 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 6 | labels: 7 | app: whereami-backend 8 | name: whereami-backend 9 | namespace: whereami-backend 10 | --- 11 | apiVersion: v1 12 | kind: ConfigMap 13 | metadata: 14 | # annotations: 15 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 16 | labels: 17 | app: whereami-backend 18 | name: whereami-backend 19 | namespace: whereami-backend 20 | data: 21 | BACKEND_ENABLED: "False" 22 | BACKEND_SERVICE: http://whereami-backend 23 | ECHO_HEADERS: "False" 24 | GRPC_ENABLED: "False" 25 | HOST: 0.0.0.0 26 | METADATA: backend 27 | TRACE_SAMPLING_RATIO: "0.00" 28 | --- 29 | apiVersion: v1 30 | kind: Service 31 | metadata: 32 | # annotations: 33 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 34 | labels: 35 | app: whereami-backend 36 | name: whereami-backend 37 | namespace: whereami-backend 38 | spec: 39 | ports: 40 | - name: http 41 | port: 80 42 | protocol: TCP 43 | targetPort: 8080 44 | selector: 45 | app: whereami-backend 46 | type: ClusterIP 47 | --- 48 | apiVersion: apps/v1 49 | kind: Deployment 50 | metadata: 51 | # annotations: 52 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 53 | labels: 54 | app: whereami-backend 55 | name: whereami-backend 56 | namespace: whereami-backend 57 | spec: 58 | replicas: 3 59 | selector: 60 | matchLabels: 61 | app: whereami-backend 62 | template: 63 | metadata: 64 | labels: 65 | app: whereami-backend 66 | version: v1 67 | spec: 68 | containers: 69 | - env: 70 | - name: NODE_NAME 71 | valueFrom: 72 | fieldRef: 73 | fieldPath: spec.nodeName 74 | - name: POD_NAMESPACE 75 | valueFrom: 76 | fieldRef: 77 | fieldPath: metadata.namespace 78 | - name: POD_IP 79 | valueFrom: 80 | fieldRef: 81 | fieldPath: status.podIP 82 | - name: POD_SERVICE_ACCOUNT 83 | valueFrom: 84 | fieldRef: 85 | fieldPath: spec.serviceAccountName 86 | - name: BACKEND_ENABLED 87 | valueFrom: 88 | configMapKeyRef: 89 | key: BACKEND_ENABLED 90 | name: whereami-backend 91 | - name: BACKEND_SERVICE 92 | valueFrom: 93 | configMapKeyRef: 94 | key: BACKEND_SERVICE 95 | name: whereami-backend 96 | - name: METADATA 97 | valueFrom: 98 | configMapKeyRef: 99 | key: METADATA 100 | name: whereami-backend 101 | - name: ECHO_HEADERS 102 | valueFrom: 103 | configMapKeyRef: 104 | key: ECHO_HEADERS 105 | name: whereami-backend 106 | - name: GRPC_ENABLED 107 | valueFrom: 108 | configMapKeyRef: 109 | key: GRPC_ENABLED 110 | name: whereami-backend 111 | - name: TRACE_SAMPLING_RATIO 112 | valueFrom: 113 | configMapKeyRef: 114 | key: TRACE_SAMPLING_RATIO 115 | name: whereami-backend 116 | - name: HOST 117 | valueFrom: 118 | configMapKeyRef: 119 | key: HOST 120 | name: whereami-backend 121 | image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.22 122 | livenessProbe: 123 | httpGet: 124 | path: /healthz 125 | port: 8080 126 | scheme: HTTP 127 | initialDelaySeconds: 10 128 | periodSeconds: 15 129 | timeoutSeconds: 5 130 | name: whereami 131 | ports: 132 | - containerPort: 8080 133 | name: http 134 | readinessProbe: 135 | httpGet: 136 | path: /healthz 137 | port: 8080 138 | scheme: HTTP 139 | initialDelaySeconds: 10 140 | timeoutSeconds: 1 141 | resources: 142 | limits: 143 | cpu: 250m 144 | memory: 512Mi 145 | requests: 146 | cpu: 250m 147 | memory: 512Mi 148 | securityContext: 149 | allowPrivilegeEscalation: false 150 | capabilities: 151 | drop: 152 | - all 153 | privileged: false 154 | readOnlyRootFilesystem: true 155 | securityContext: 156 | fsGroup: 1000 157 | runAsGroup: 1000 158 | runAsNonRoot: true 159 | runAsUser: 1000 160 | serviceAccountName: whereami-backend 161 | -------------------------------------------------------------------------------- /demos/fleets/teams/whereami/backend/whereami-backend-dr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: DestinationRule 3 | metadata: 4 | name: backend 5 | namespace: whereami-backend 6 | spec: 7 | host: whereami-backend.whereami-backend.svc.cluster.local 8 | trafficPolicy: 9 | connectionPool: 10 | http: 11 | maxRequestsPerConnection: 0 12 | loadBalancer: 13 | simple: LEAST_REQUEST 14 | localityLbSetting: 15 | enabled: true 16 | failover: 17 | - from: us-east1 18 | to: us-central1 19 | - from: us-central1 20 | to: us-east1 21 | outlierDetection: 22 | consecutive5xxErrors: 1 23 | interval: 1s 24 | baseEjectionTime: 1m 25 | -------------------------------------------------------------------------------- /demos/fleets/teams/whereami/backend/whereami-backend-vs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: VirtualService 3 | metadata: 4 | name: whereami-vs 5 | namespace: whereami-backend 6 | spec: 7 | hosts: 8 | - 'whereami-backend.whereami-backend.svc.cluster.local' 9 | http: 10 | - route: 11 | - destination: 12 | host: whereami-backend 13 | port: 14 | number: 80 15 | -------------------------------------------------------------------------------- /demos/fleets/teams/whereami/frontend/app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | # annotations: 5 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 6 | labels: 7 | app: whereami-frontend 8 | name: whereami-frontend 9 | namespace: whereami-frontend 10 | --- 11 | kind: ConfigMap 12 | metadata: 13 | # annotations: 14 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 15 | labels: 16 | app: whereami-frontend 17 | name: whereami-frontend 18 | namespace: whereami-frontend 19 | apiVersion: v1 20 | data: 21 | BACKEND_ENABLED: "True" 22 | BACKEND_SERVICE: http://whereami-backend.whereami-backend.svc.cluster.local 23 | ECHO_HEADERS: "False" 24 | GRPC_ENABLED: "False" 25 | HOST: 0.0.0.0 26 | METADATA: frontend 27 | TRACE_SAMPLING_RATIO: "0.00" 28 | --- 29 | apiVersion: v1 30 | kind: Service 31 | metadata: 32 | # annotations: 33 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 34 | labels: 35 | app: whereami-frontend 36 | name: whereami-frontend 37 | namespace: whereami-frontend 38 | spec: 39 | ports: 40 | - name: http 41 | port: 80 42 | protocol: TCP 43 | targetPort: 8080 44 | selector: 45 | app: whereami-frontend 46 | type: ClusterIP 47 | --- 48 | apiVersion: apps/v1 49 | kind: Deployment 50 | metadata: 51 | # annotations: 52 | # configmanagement.gke.io/namespace-selector: wherami-fleet-scope-selector 53 | labels: 54 | app: whereami-frontend 55 | name: whereami-frontend 56 | namespace: whereami-frontend 57 | spec: 58 | replicas: 3 59 | selector: 60 | matchLabels: 61 | app: whereami-frontend 62 | template: 63 | metadata: 64 | labels: 65 | app: whereami-frontend 66 | version: v1 67 | spec: 68 | containers: 69 | - env: 70 | - name: NODE_NAME 71 | valueFrom: 72 | fieldRef: 73 | fieldPath: spec.nodeName 74 | - name: POD_NAMESPACE 75 | valueFrom: 76 | fieldRef: 77 | fieldPath: metadata.namespace 78 | - name: POD_IP 79 | valueFrom: 80 | fieldRef: 81 | fieldPath: status.podIP 82 | - name: POD_SERVICE_ACCOUNT 83 | valueFrom: 84 | fieldRef: 85 | fieldPath: spec.serviceAccountName 86 | - name: BACKEND_ENABLED 87 | valueFrom: 88 | configMapKeyRef: 89 | key: BACKEND_ENABLED 90 | name: whereami-frontend 91 | - name: BACKEND_SERVICE 92 | valueFrom: 93 | configMapKeyRef: 94 | key: BACKEND_SERVICE 95 | name: whereami-frontend 96 | - name: METADATA 97 | valueFrom: 98 | configMapKeyRef: 99 | key: METADATA 100 | name: whereami-frontend 101 | - name: ECHO_HEADERS 102 | valueFrom: 103 | configMapKeyRef: 104 | key: ECHO_HEADERS 105 | name: whereami-frontend 106 | - name: GRPC_ENABLED 107 | valueFrom: 108 | configMapKeyRef: 109 | key: GRPC_ENABLED 110 | name: whereami-frontend 111 | - name: TRACE_SAMPLING_RATIO 112 | valueFrom: 113 | configMapKeyRef: 114 | key: TRACE_SAMPLING_RATIO 115 | name: whereami-frontend 116 | - name: HOST 117 | valueFrom: 118 | configMapKeyRef: 119 | key: HOST 120 | name: whereami-frontend 121 | image: us-docker.pkg.dev/google-samples/containers/gke/whereami:v1.2.22 122 | livenessProbe: 123 | httpGet: 124 | path: /healthz 125 | port: 8080 126 | scheme: HTTP 127 | initialDelaySeconds: 10 128 | periodSeconds: 15 129 | timeoutSeconds: 5 130 | name: whereami 131 | ports: 132 | - containerPort: 8080 133 | name: http 134 | readinessProbe: 135 | httpGet: 136 | path: /healthz 137 | port: 8080 138 | scheme: HTTP 139 | initialDelaySeconds: 10 140 | timeoutSeconds: 1 141 | resources: 142 | limits: 143 | cpu: 250m 144 | memory: 512Mi 145 | requests: 146 | cpu: 250m 147 | memory: 512Mi 148 | securityContext: 149 | allowPrivilegeEscalation: false 150 | capabilities: 151 | drop: 152 | - all 153 | privileged: false 154 | readOnlyRootFilesystem: true 155 | securityContext: 156 | fsGroup: 1000 157 | runAsGroup: 1000 158 | runAsNonRoot: true 159 | runAsUser: 1000 160 | serviceAccountName: whereami-frontend -------------------------------------------------------------------------------- /demos/fleets/teams/whereami/frontend/whereami-frontend-dr.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: DestinationRule 3 | metadata: 4 | name: frontend 5 | namespace: whereami-frontend 6 | spec: 7 | host: whereami-frontend.whereami-frontend.svc.cluster.local 8 | trafficPolicy: 9 | connectionPool: 10 | http: 11 | maxRequestsPerConnection: 0 12 | loadBalancer: 13 | simple: LEAST_REQUEST 14 | localityLbSetting: 15 | enabled: true 16 | failover: 17 | - from: us-east1 18 | to: us-central1 19 | - from: us-central1 20 | to: us-east1 21 | outlierDetection: 22 | consecutive5xxErrors: 1 23 | interval: 1s 24 | baseEjectionTime: 1m 25 | -------------------------------------------------------------------------------- /demos/fleets/teams/whereami/frontend/whereami-frontend-vs.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: networking.istio.io/v1beta1 2 | kind: VirtualService 3 | metadata: 4 | name: whereami-vs 5 | namespace: whereami-frontend 6 | spec: 7 | gateways: 8 | - asm-gateways/asm-ingress-gateway-xlb 9 | hosts: 10 | - 'frontend.endpoints.MYPROJECT.cloud.goog' 11 | http: 12 | - route: 13 | - destination: 14 | host: whereami-frontend 15 | port: 16 | number: 80 17 | -------------------------------------------------------------------------------- /docs/analytics.md: -------------------------------------------------------------------------------- 1 | # Analytics 2 | 3 | The GKE PoC Toolkit collects usage data on an **opt-in** basis. You can opt-in to data collection when running `gkekitctl init` during setup: 4 | 5 | ```bash 6 | ➜ gke-poc-toolkit ./gkekitctl init 7 | INFO[0000] ☸️ ----- GKE POC TOOLKIT ----- 🛠 8 | INFO[0000] 🔄 Initializing flat files for gkekitctl... 9 | INFO[0000] 📊 Send anonymous analytics to GKE PoC Toolkit maintainers? 10 | Use the arrow keys to navigate: ↓ ↑ → ← 11 | ? Select[Yes/No]: 12 | ▸ Yes 13 | No 14 | ``` 15 | 16 | If you opt in, the following data is collected every time you run `gkekitctl create`: (Included here with example values) 17 | 18 | ```JSON 19 | { 20 | "create_id": "12345", 21 | "cluster_id": "12345", 22 | "version": "v1", 23 | "gitCommit": "xyz", 24 | "timestamp": "2021-11-12T11:45:26.371Z", 25 | "os": "darwin_x64", 26 | "terraformState": "local", 27 | "region": "us-central1", 28 | "enableWorkloadIdentity": false, 29 | "enablePreemptibleNodepool": false, 30 | "defaultNodepoolOS": "cos", 31 | "privateEndpoint": false, 32 | "enableConfigSync": true, 33 | "enablePolicyController": true, 34 | "anthosServiceMesh": true, 35 | "multiClusterGateway": false, 36 | "vpcType": "standalone", 37 | "clusterIndex": 0, 38 | "clusterNumNodes": 3, 39 | "clusterType": "public", 40 | "clusterMachineType": "e2-standard-4", 41 | "clusterRegion": "us-central1", 42 | "clusterZone": "us-central1-b" 43 | } 44 | ``` 45 | 46 | One of these objects is sent for every cluster you create with the Toolkit. Note that no PII is collected. We do not collect your GCP project ID, your GCP account info, or your organization info. **`create_id`** and **`cluster_id`** are both randomly-generated UUIDs that allows us to group together clusters from a single `gkekitctl create` run, thus tracking how many clusters, on average, users want to create per environment. 47 | 48 | ### links to code 49 | - [Analytics server](/analytics/server.go) 50 | - [Analytics client](/cli/pkg/analytics/client.go) 51 | -------------------------------------------------------------------------------- /docs/building-demos.md: -------------------------------------------------------------------------------- 1 | # Building Demos with the Toolkit 2 | 3 | The GKE PoC Toolkit is designed to provide a reusable base-layer for different GKE demos. A toolkit environment gets you as far as as project with GKE clusters and add-ons installed, including Anthos Service Mesh and Anthos Config Management. 4 | 5 | Beyond this base layer, you can deploy sample applications and install additional tools. 6 | 7 | 8 | ## Getting `kubectl` access to your clusters 9 | 10 | To authenticate to all the clusters created with a single run of `gkekictl create`, you'll need a [kubeconfig](https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/). `gkekitctl create` generates this file on every run and saves it as `[YOUR-TOOLKIT_DIR]/kubeconfig`. 11 | 12 | So once `gkekitctl create` completes, set your KUBECONFIG env variable like this: 13 | 14 | ```bash 15 | export KUBECONFIG=[PATH/TO/gke-poc-toolkit]/kubeconfig 16 | ``` 17 | 18 | From here, you can use the [`kubectx`](https://github.com/ahmetb/kubectx) tool to easily switch back and forth between your Toolkit clusters and run commands like `kubectl get pods`. 19 | 20 | ## Deploying workloads 21 | 22 | Let's say you want to deploy a sample application like [Online Boutique](https://github.com/GoogleCloudPlatform/microservices-demo) to your Toolkit environment. Or you want to install an addional platform layer like [Kubeflow](https://www.kubeflow.org/docs/distributions/gke/). 23 | 24 | First, get the YAML you want to deploy (eg. [release manifests](https://github.com/GoogleCloudPlatform/microservices-demo/blob/main/release/kubernetes-manifests.yaml)), and save it locally. 25 | 26 | From here, the most straightforward way to deploy Kubernetes workloads and configuration to your toolkit clusters is via Config Sync (Anthos Config Management). To do this, clone your toolkit Config Sync repo, add some YAML, and push to the `main` branch. 27 | 28 | By default, these configs will be deployed to all your toolkit clusters; you can also [use cluster selectors](https://cloud.google.com/anthos-config-management/docs/how-to/clusterselectors) to choose which configs land where. 29 | 30 | You can also set up CI/CD using Cloud Build or Cloud Deploy to GKE - this is ideal for demos targeting app developers. 31 | -------------------------------------------------------------------------------- /docs/frequently-asked-questions.md: -------------------------------------------------------------------------------- 1 | # FAQ 2 | 3 | ### What is the purpose of the GKE PoC Toolkit? 4 | 5 | The GKE PoC Toolkit sets out to provide a set of infrastructure as code (IaC) which deploys GKE clusters with a strong security posture that can be used to step through demos, stand up a POC, and deliver a codified example that can be re-used / provide inspiration for production GKE environments. 6 | 7 | The maintainers of this repo need to set up GKE demos often, but since every environment was a little bit different, they needed a bunch of forked scripts, gcloud commands, and Terraform in order to do that, which is hard to maintain and re-use over time. The maintainers wanted an easier, one-click way of building GKE demos, so they compiled a bunch of Terraform modules together and wrapped it in a friendly Go CLI. This tool is open source so that anyone can explore the awesomeness of GKE! 8 | 9 | ### What user analytics data do you collect? 10 | 11 | The maintainers of the Toolkit collect anonymous usage statistics to understand its usage and make improvements. Data collection is **opt-in only** when you run `gkekitctl init`. See the [Analytics](/docs/analytics.md) doc for the list of what we collect, should you opt in to anonymous analytics. 12 | ### What kinds of things can I build with the Toolkit? 13 | 14 | You can build pretty much any kind of GKE / Anthos on GCP demo you want, from minimal single-cluster demos, to best-practices GKE security architectures, to large multi-cluster Anthos Service Mesh reference demos. 15 | 16 | Check out the [Building Demos](/docs/building-demos.md) doc to learn more. 17 | 18 | ### How can I contribute? 19 | 20 | We love contributions! Check out the [list of open issues](https://github.com/gke-poc-toolkit/issues) and the [contributing guide](/CONTRIBUTING.md) to get started. 21 | -------------------------------------------------------------------------------- /terraform/modules/clusters/clusters.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | // Presets for project and network settings 19 | project_id = var.shared_vpc ? var.vpc_project_id : var.project_id 20 | network = "projects/${local.project_id}/global/networks/${var.vpc_name}" 21 | vpc_selflink = format("projects/%s/global/networks/%s", local.project_id, var.vpc_name) 22 | } 23 | 24 | # Create clusters listing in the cluster_config variable 25 | resource "google_container_cluster" "gke_ap" { 26 | for_each = var.cluster_config 27 | provider = google-beta 28 | name = each.key 29 | project = var.project_id 30 | location = each.value.region 31 | enable_autopilot = true 32 | initial_node_count = var.initial_node_count 33 | network = "projects/${var.vpc_project_id}/global/networks/${var.vpc_name}" 34 | subnetwork = "projects/${var.vpc_project_id}/regions/${each.value.region}/subnetworks/${each.value.subnet_name}" 35 | # networking_mode = "VPC_NATIVE" 36 | # datapath_provider = "ADVANCED_DATAPATH" 37 | 38 | addons_config { 39 | # HTTP Load Balancing is required to be enabled in Autopilot clusters 40 | http_load_balancing { 41 | disabled = false 42 | } 43 | # Horizontal Pod Autoscaling is required to be enabled in Autopilot clusters 44 | horizontal_pod_autoscaling { 45 | disabled = false 46 | } 47 | cloudrun_config { 48 | disabled = true 49 | } 50 | 51 | kalm_config { 52 | enabled = false 53 | } 54 | config_connector_config { 55 | enabled = false 56 | } 57 | gke_backup_agent_config { 58 | enabled = true 59 | } 60 | } 61 | 62 | authenticator_groups_config { security_group = var.authenticator_security_group } 63 | cluster_autoscaling { autoscaling_profile = "OPTIMIZE_UTILIZATION" } 64 | cost_management_config { enabled = true } 65 | deletion_protection = false 66 | fleet { project = var.fleet_project } 67 | gateway_api_config { channel = "CHANNEL_STANDARD" } 68 | ip_allocation_policy { 69 | cluster_secondary_range_name = var.vpc_ip_range_pods_name 70 | services_secondary_range_name = var.vpc_ip_range_services_name 71 | } 72 | logging_config { 73 | enable_components = ["SYSTEM_COMPONENTS", "WORKLOADS", "APISERVER", "CONTROLLER_MANAGER", "SCHEDULER"] 74 | } 75 | master_authorized_networks_config { 76 | cidr_blocks { 77 | cidr_block = "10.0.0.0/8" 78 | display_name = "Internal VMs" 79 | } 80 | } 81 | monitoring_config { 82 | managed_prometheus { enabled = true } 83 | enable_components = ["SYSTEM_COMPONENTS", "APISERVER", "CONTROLLER_MANAGER", "SCHEDULER", "STORAGE", "HPA", "POD", "DAEMONSET", "DEPLOYMENT", "STATEFULSET", "KUBELET", "CADVISOR", "DCGM"] 84 | } 85 | private_cluster_config { 86 | enable_private_nodes = true 87 | enable_private_endpoint = false 88 | master_ipv4_cidr_block = "172.16.${index(keys(var.cluster_config), each.key)}.16/28" 89 | master_global_access_config { 90 | enabled = true 91 | } 92 | } 93 | release_channel { channel = var.release_channel } 94 | secret_manager_config { enabled = true } 95 | 96 | security_posture_config { 97 | mode = "ENTERPRISE" 98 | vulnerability_mode = "VULNERABILITY_ENTERPRISE" 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /terraform/modules/clusters/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "project_id" { 18 | description = "Deployment project ID" 19 | value = var.project_id 20 | } 21 | 22 | output "get_credential_commands" { 23 | description = "gcloud get-credentials command to generate kubeconfig for the private cluster" 24 | value = flatten([for s in google_container_cluster.gke_ap : (format("gcloud container fleet memberships get-credentials %s --project %s --location=%s", s.name, var.project_id, s.location))]) 25 | } 26 | 27 | output "cluster_names" { 28 | description = "List of GKE cluster names" 29 | value = flatten([for s in google_container_cluster.gke_ap : s.name]) 30 | } 31 | 32 | output "endpoints" { 33 | sensitive = true 34 | description = "List of GKE cluster endpoints" 35 | value = flatten([for s in google_container_cluster.gke_ap : s.endpoint]) 36 | } 37 | -------------------------------------------------------------------------------- /terraform/modules/clusters/provider.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">=1.3" 19 | 20 | required_providers { 21 | google = { 22 | source = "hashicorp/google" 23 | version = ">= 5.40.0, < 7" 24 | } 25 | kubernetes = { 26 | source = "hashicorp/kubernetes" 27 | version = "~> 2.10" 28 | } 29 | random = { 30 | source = "hashicorp/random" 31 | version = ">= 2.1" 32 | } 33 | } 34 | } 35 | 36 | provider "google" { 37 | project = var.project_id 38 | region = var.region 39 | } 40 | -------------------------------------------------------------------------------- /terraform/modules/clusters/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "The project ID to host the cluster in" 20 | } 21 | 22 | variable "fleet_project" { 23 | description = "(Optional) Register the cluster with the fleet in this project." 24 | type = string 25 | default = null 26 | } 27 | 28 | variable "regional_clusters" { 29 | type = bool 30 | description = "Enable regional control plane." 31 | default = true 32 | } 33 | 34 | variable "region" { 35 | type = string 36 | description = "The region to host the cluster in" 37 | default = "us-central1" 38 | } 39 | 40 | variable "shared_vpc" { 41 | type = bool 42 | description = "boolean value for determining whether to create Standalone VPC or use a preexisting Shared VPC" 43 | default = false 44 | } 45 | 46 | variable "vpc_name" { 47 | type = string 48 | description = "The name of the network being created to host the cluster in" 49 | default = "gke-toolkit-network" 50 | } 51 | 52 | 53 | variable "vpc_project_id" { 54 | type = string 55 | description = "The Share VPC Project ID - This is optional and only valid if a Shared VPC is used" 56 | default = "" 57 | } 58 | 59 | variable "vpc_ip_range_pods_name" { 60 | type = string 61 | description = "The secondary ip range to use for pods in the shared vpc - This is optional and only valid if a Shared VPC is used" 62 | default = "" 63 | } 64 | 65 | variable "vpc_ip_range_services_name" { 66 | type = string 67 | description = "The secondary ip range to use for services in the shared vpc - This is optional and only valid if a Shared VPC is used" 68 | default = "" 69 | } 70 | 71 | variable "release_channel" { 72 | type = string 73 | default = "regular" 74 | } 75 | 76 | variable "node_pool" { 77 | type = string 78 | default = "gke-toolkit-pool" 79 | } 80 | 81 | variable "initial_node_count" { 82 | type = number 83 | default = 4 84 | } 85 | 86 | variable "min_node_count" { 87 | type = number 88 | default = 4 89 | } 90 | 91 | variable "max_node_count" { 92 | type = number 93 | default = 10 94 | } 95 | 96 | variable "linux_machine_type" { 97 | type = string 98 | default = "n1-standard-4" 99 | } 100 | 101 | variable "private_endpoint" { 102 | type = bool 103 | default = false 104 | } 105 | 106 | variable "authenticator_security_group" { 107 | type = string 108 | description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" 109 | default = null 110 | } 111 | 112 | variable "cluster_config" { 113 | description = "For each cluster, create an object that contain the required fields" 114 | default = {} 115 | } 116 | 117 | variable "auth_cidr" { 118 | type = string 119 | default = "172.16.100.16/28" 120 | } -------------------------------------------------------------------------------- /terraform/modules/fleet/admin-cluster.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_container_cluster" "admin" { 18 | provider = google-beta 19 | name = "gke-ap-admin-cp-00" 20 | project = var.fleet_project 21 | location = "us-central1" 22 | enable_autopilot = true 23 | initial_node_count = 1 24 | network = "projects/${var.vpc_project_id}/global/networks/${var.vpc_name}" 25 | subnetwork = "projects/${var.vpc_project_id}/regions/us-central1/subnetworks/admin-control-plane" 26 | # networking_mode = "VPC_NATIVE" 27 | # datapath_provider = "ADVANCED_DATAPATH" 28 | 29 | addons_config { 30 | # HTTP Load Balancing is required to be enabled in Autopilot clusters 31 | http_load_balancing { 32 | disabled = false 33 | } 34 | # Horizontal Pod Autoscaling is required to be enabled in Autopilot clusters 35 | horizontal_pod_autoscaling { 36 | disabled = false 37 | } 38 | cloudrun_config { 39 | disabled = true 40 | } 41 | 42 | kalm_config { 43 | enabled = false 44 | } 45 | config_connector_config { 46 | enabled = false 47 | } 48 | gke_backup_agent_config { 49 | enabled = true 50 | } 51 | } 52 | 53 | authenticator_groups_config { security_group = var.authenticator_security_group } 54 | cluster_autoscaling { autoscaling_profile = "OPTIMIZE_UTILIZATION" } 55 | cost_management_config { enabled = true } 56 | deletion_protection = false 57 | fleet { project = var.fleet_project } 58 | gateway_api_config { channel = "CHANNEL_STANDARD" } 59 | ip_allocation_policy { 60 | cluster_secondary_range_name = "admin-pods" 61 | services_secondary_range_name = "admin-svcs" 62 | } 63 | logging_config { 64 | enable_components = ["SYSTEM_COMPONENTS", "WORKLOADS", "APISERVER", "CONTROLLER_MANAGER", "SCHEDULER"] 65 | } 66 | master_authorized_networks_config { 67 | cidr_blocks { 68 | cidr_block = "10.0.0.0/8" 69 | display_name = "Internal VMs" 70 | } 71 | } 72 | monitoring_config { 73 | managed_prometheus { enabled = true } 74 | enable_components = ["SYSTEM_COMPONENTS", "APISERVER", "CONTROLLER_MANAGER", "SCHEDULER", "STORAGE", "HPA", "POD", "DAEMONSET", "DEPLOYMENT", "STATEFULSET", "KUBELET", "CADVISOR", "DCGM"] 75 | } 76 | private_cluster_config { 77 | enable_private_nodes = true 78 | enable_private_endpoint = false 79 | master_ipv4_cidr_block = "172.16.100.16/28" 80 | master_global_access_config { 81 | enabled = true 82 | } 83 | } 84 | release_channel { channel = var.release_channel } 85 | secret_manager_config { enabled = true } 86 | 87 | security_posture_config { 88 | mode = "ENTERPRISE" 89 | vulnerability_mode = "VULNERABILITY_ENTERPRISE" 90 | } 91 | depends_on = [ 92 | module.enabled_service_project_apis, 93 | google_gke_hub_feature.mesh_config_defaults, 94 | google_gke_hub_fleet.default, 95 | google_project_iam_member.hubsa, 96 | 97 | ] 98 | } 99 | -------------------------------------------------------------------------------- /terraform/modules/fleet/feature-defaults.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sourcerepo_repository 18 | // Create 1 centralized Cloud Source Repo, that all GKE clusters will sync to 19 | resource "google_sourcerepo_repository" "default-config-sync-repo" { 20 | name = var.config_sync_repo 21 | project = var.fleet_project 22 | } 23 | 24 | # Fleet Policy Defaults 25 | resource "google_gke_hub_feature" "fleet_policy_defaults" { 26 | project = var.fleet_project 27 | location = "global" 28 | name = "policycontroller" 29 | 30 | fleet_default_member_config { 31 | policycontroller { 32 | policy_controller_hub_config { 33 | install_spec = "INSTALL_SPEC_ENABLED" 34 | policy_content { 35 | bundles { 36 | bundle = "cis-k8s-v1.5.1" 37 | } 38 | } 39 | audit_interval_seconds = 30 40 | referential_rules_enabled = true 41 | } 42 | } 43 | } 44 | 45 | depends_on = [module.enabled_service_project_apis] 46 | } 47 | 48 | # Config Sync Defaults 49 | resource "google_gke_hub_feature" "config_management" { 50 | name = "configmanagement" 51 | project = var.fleet_project 52 | location = "global" 53 | provider = google 54 | 55 | fleet_default_member_config { 56 | configmanagement { 57 | management = "MANAGEMENT_AUTOMATIC" 58 | config_sync { 59 | source_format = "unstructured" 60 | git { 61 | sync_repo = "https://source.developers.google.com/p/${var.fleet_project}/r/${var.config_sync_repo}" 62 | sync_branch = var.config_sync_repo_branch 63 | policy_dir = var.config_sync_repo_dir 64 | secret_type = "gcpserviceaccount" 65 | gcp_service_account_email = local.cs_service_account_email 66 | } 67 | } 68 | } 69 | } 70 | 71 | depends_on = [ 72 | resource.google_endpoints_service.whereami_service, 73 | resource.google_endpoints_service.inference_service, 74 | ] 75 | } 76 | 77 | # Mesh Config Defaults 78 | resource "google_gke_hub_feature" "mesh_config_defaults" { 79 | project = var.fleet_project 80 | location = "global" 81 | name = "servicemesh" 82 | 83 | fleet_default_member_config { 84 | mesh { 85 | management = "MANAGEMENT_AUTOMATIC" 86 | } 87 | } 88 | 89 | # depends_on = [google_project_iam_member.hubsa] 90 | } 91 | 92 | # Fleet Observability 93 | resource "google_gke_hub_feature" "fleet_observability" { 94 | name = "fleetobservability" 95 | project = var.fleet_project 96 | location = "global" 97 | 98 | spec { 99 | fleetobservability { 100 | logging_config { 101 | default_config { 102 | mode = "COPY" 103 | } 104 | fleet_scope_logs_config { 105 | mode = "COPY" 106 | } 107 | } 108 | } 109 | } 110 | 111 | depends_on = [module.enabled_service_project_apis] 112 | } 113 | 114 | # Fleet Resource 115 | resource "google_gke_hub_fleet" "default" { 116 | project = var.fleet_project 117 | 118 | default_cluster_config { 119 | security_posture_config { 120 | mode = "ENTERPRISE" 121 | vulnerability_mode = "VULNERABILITY_BASIC" 122 | } 123 | } 124 | 125 | depends_on = [module.enabled_service_project_apis] 126 | } 127 | -------------------------------------------------------------------------------- /terraform/modules/fleet/iam.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | # Config Sync Service Account 3 | cs_service_account = "cs-service-account" 4 | cs_service_account_email = "${local.cs_service_account}@${var.fleet_project}.iam.gserviceaccount.com" 5 | # Hub service account 6 | hub_service_account_email = format("service-%s@gcp-sa-gkehub.iam.gserviceaccount.com", data.google_project.fleet_project.number) 7 | hub_service_account = "serviceAccount:${local.hub_service_account_email}" 8 | } 9 | 10 | # # Create Hub Service Account 11 | # resource "google_project_iam_member" "hubsa" { 12 | # project = var.fleet_project 13 | # role = "roles/gkehub.serviceAgent" 14 | # member = local.hub_service_account 15 | # depends_on = [ 16 | # module.enabled_service_project_apis, 17 | # ] 18 | # } 19 | 20 | // create ACM service account 21 | // Todo - use KSA directly 22 | module "service_accounts" { 23 | source = "terraform-google-modules/service-accounts/google" 24 | # version = "~> 4.2.0" 25 | project_id = var.fleet_project 26 | display_name = "CS service account" 27 | names = [local.cs_service_account] 28 | project_roles = ["${var.fleet_project}=>roles/source.reader"] 29 | } 30 | 31 | module "cs_service_account-iam-bindings" { 32 | depends_on = [ 33 | resource.google_gke_hub_feature.config_management, 34 | ] 35 | source = "terraform-google-modules/iam/google//modules/service_accounts_iam" 36 | 37 | service_accounts = [local.cs_service_account_email] 38 | project = var.fleet_project 39 | bindings = { 40 | "roles/iam.workloadIdentityUser" = [ 41 | "serviceAccount:${var.fleet_project}.svc.id.goog[config-management-system/root-reconciler]", 42 | ] 43 | } 44 | } 45 | 46 | module "asm-service_account-iam-bindings" { 47 | depends_on = [ 48 | resource.google_gke_hub_feature.config_management, 49 | ] 50 | source = "terraform-google-modules/iam/google//modules/service_accounts_iam" 51 | 52 | project = var.fleet_project 53 | bindings = { 54 | "roles/secretmanager.secretAccessor" = [ 55 | "serviceAccount:${var.fleet_project}.svc.id.goog[asm-gateways/asm-ingress-gateway]", 56 | ] 57 | } 58 | } 59 | 60 | module "prom-service_account-iam-bindings" { 61 | depends_on = [ 62 | resource.google_gke_hub_feature.config_management, 63 | ] 64 | source = "terraform-google-modules/iam/google//modules/service_accounts_iam" 65 | 66 | project = var.fleet_project 67 | bindings = { 68 | "roles/monitoring.viewer" = [ 69 | "serviceAccount:${var.fleet_project}.svc.id.goog[custom-metrics/custom-metrics-stackdriver-adapter]", 70 | ] 71 | } 72 | } 73 | 74 | // Create IAM binding granting the ASM Gateway KSA access to the self signed certs stored in secret manager 75 | resource "google_project_iam_binding" "asm-gw-secret-accessor" { 76 | role = "roles/secretmanager.secretAccessor" 77 | project = var.fleet_project 78 | members = [ 79 | "serviceAccount:${var.fleet_project}.svc.id.goog[asm-gateways/asm-ingress-gateway]", 80 | ] 81 | } -------------------------------------------------------------------------------- /terraform/modules/fleet/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "fleet_project" { 18 | value = var.fleet_project 19 | } -------------------------------------------------------------------------------- /terraform/modules/fleet/provider.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">=1.3" 19 | 20 | required_providers { 21 | google = { 22 | source = "hashicorp/google" 23 | version = ">= 5.40.0, < 7" 24 | } 25 | kubernetes = { 26 | source = "hashicorp/kubernetes" 27 | version = "~> 2.10" 28 | } 29 | random = { 30 | source = "hashicorp/random" 31 | version = ">= 2.1" 32 | } 33 | } 34 | } 35 | 36 | provider "google" { 37 | project = var.project_id 38 | } 39 | 40 | provider "google-beta" { 41 | project = var.project_id 42 | } 43 | -------------------------------------------------------------------------------- /terraform/modules/fleet/services.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # Enable Fleet Services 18 | module "enabled_service_project_apis" { 19 | source = "terraform-google-modules/project-factory/google//modules/project_services" 20 | version = "~> 17.0" 21 | 22 | project_id = var.fleet_project 23 | disable_services_on_destroy = false 24 | 25 | activate_apis = [ 26 | "container.googleapis.com", 27 | "anthos.googleapis.com", 28 | "dns.googleapis.com", 29 | "gkehub.googleapis.com", 30 | "gkeconnect.googleapis.com", 31 | "anthosconfigmanagement.googleapis.com", 32 | "anthospolicycontroller.googleapis.com", 33 | "meshconfig.googleapis.com", 34 | "meshca.googleapis.com", 35 | "containersecurity.googleapis.com", 36 | "logging.googleapis.com", 37 | "cloudresourcemanager.googleapis.com", 38 | "multiclusterservicediscovery.googleapis.com", 39 | "multiclusteringress.googleapis.com", 40 | "compute.googleapis.com", 41 | "iam.googleapis.com", 42 | "sourcerepo.googleapis.com", 43 | "endpoints.googleapis.com", 44 | "certificatemanager.googleapis.com", 45 | ] 46 | } -------------------------------------------------------------------------------- /terraform/modules/fleet/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "The project ID to host the cluster in" 20 | } 21 | 22 | variable "fleet_project" { 23 | type = string 24 | description = "(Optional) Register the cluster with the fleet in this project." 25 | } 26 | 27 | variable "vpc_project_id" { 28 | type = string 29 | description = "Shared VPC project needed for setting MCI and MCS RBAC." 30 | } 31 | 32 | variable "config_sync_repo" { 33 | description = "Git repo used as the default config sync repo for your fleet." 34 | type = string 35 | default = null 36 | } 37 | 38 | variable "config_sync_repo_branch" { 39 | description = "Git repo branch used as the default config sync repo for your fleet." 40 | type = string 41 | default = null 42 | } 43 | 44 | variable "config_sync_repo_dir" { 45 | description = "Git repo directory used as the default config sync repo for your fleet." 46 | type = string 47 | default = null 48 | } 49 | 50 | variable "shared_vpc" { 51 | type = bool 52 | description = "Determines whether to create a standalone VPC or use an existing Shared VPC" 53 | default = false 54 | } 55 | 56 | variable "vpc_ip_range_pods_name" { 57 | type = string 58 | description = "The secondary IP range to use for pods in the shared VPC" 59 | default = "" 60 | } 61 | 62 | variable "vpc_ip_range_services_name" { 63 | type = string 64 | description = "The secondary IP range to use for services in the shared VPC" 65 | default = "" 66 | } 67 | 68 | variable "release_channel" { 69 | type = string 70 | description = "The release channel of the cluster" 71 | default = "regular" 72 | } 73 | 74 | variable "authenticator_security_group" { 75 | type = string 76 | description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC." 77 | default = null 78 | } 79 | 80 | variable "vpc_name" { 81 | type = string 82 | description = "The name of the VPC - used for shared or local VPC" 83 | default = "" 84 | } 85 | 86 | variable "cluster_config" { 87 | type = map(object({ 88 | subnet_name = string 89 | region = string 90 | })) 91 | description = "For each cluster, create an object that contains the required fields" 92 | default = {} 93 | } 94 | -------------------------------------------------------------------------------- /terraform/modules/network/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Data Resources 18 | data "google_project" "project" { 19 | project_id = var.project_id 20 | } 21 | 22 | // Locals used to construct names of stuffs. 23 | locals { 24 | # VPC Self-link 25 | vpc_selflink = format("projects/%s/global/networks/%s", var.project_id, var.vpc_name) 26 | 27 | # Distinct cluster regions 28 | distinct_cluster_regions = toset([for cluster in var.cluster_config : cluster.region]) 29 | 30 | // Presets for Service Accounts 31 | clu_service_account = format("service-%s@container-engine-robot.iam.gserviceaccount.com", data.google_project.project.number) 32 | prj_service_account = format("%s@cloudservices.gserviceaccount.com", data.google_project.project.number) 33 | 34 | // Dynamically create subnet and secondary subnet inputs for multi-cluster creation 35 | admin_subnet = flatten([ 36 | { 37 | subnet_name = "admin-control-plane" 38 | subnet_ip = "10.0.100.0/24" 39 | subnet_region = "us-central1" 40 | subnet_private_access = true 41 | description = "This subnet is for the admin control plane and is managed by Terraform" 42 | } 43 | ]) 44 | nested_subnets_raw = flatten([ 45 | for name, config in var.cluster_config : [ 46 | { 47 | subnet_name = config.subnet_name 48 | subnet_ip = "10.0.${index(keys(var.cluster_config), name)}.0/24" 49 | subnet_region = config.region 50 | subnet_private_access = true 51 | description = "This subnet is managed by Terraform" 52 | } 53 | ] 54 | ]) 55 | 56 | nested_subnets = concat(local.admin_subnet, local.nested_subnets_raw) 57 | 58 | admin_secondary_subnets = { 59 | "admin-control-plane" = [ 60 | { 61 | range_name = "admin-pods" 62 | ip_cidr_range = "10.101.0.0/17" 63 | }, 64 | { 65 | range_name = "admin-svcs" 66 | ip_cidr_range = "10.103.0.0/17" 67 | } 68 | ] 69 | } 70 | 71 | nested_secondary_subnets = merge(local.admin_secondary_subnets, { 72 | for name, config in var.cluster_config : config.subnet_name => [ 73 | { 74 | range_name = var.vpc_ip_range_pods_name 75 | ip_cidr_range = "10.${index(keys(var.cluster_config), name) + 1}.0.0/17" 76 | }, 77 | { 78 | range_name = var.vpc_ip_range_services_name 79 | ip_cidr_range = "10.${index(keys(var.cluster_config), name) + 1}.128.0/17" 80 | }, 81 | ] 82 | }) 83 | } 84 | 85 | module "enabled_shared_vpc_apis" { 86 | source = "terraform-google-modules/project-factory/google//modules/project_services" 87 | version = "~> 17.0" 88 | 89 | project_id = var.vpc_project_id 90 | disable_services_on_destroy = true 91 | 92 | activate_apis = [ 93 | "compute.googleapis.com", 94 | "container.googleapis.com", 95 | "dns.googleapis.com", 96 | "iam.googleapis.com", 97 | ] 98 | } 99 | 100 | module "enabled_service_project_apis" { 101 | source = "terraform-google-modules/project-factory/google//modules/project_services" 102 | version = "~> 17.0" 103 | 104 | project_id = var.project_id 105 | disable_services_on_destroy = false 106 | 107 | activate_apis = [ 108 | "compute.googleapis.com", 109 | "container.googleapis.com", 110 | "iam.googleapis.com", 111 | "compute.googleapis.com", 112 | "container.googleapis.com", 113 | "cloudresourcemanager.googleapis.com", 114 | ] 115 | } -------------------------------------------------------------------------------- /terraform/modules/network/network.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "vpc" { 18 | depends_on = [ 19 | module.enabled_shared_vpc_apis, 20 | module.enabled_service_project_apis 21 | ] 22 | source = "terraform-google-modules/network/google" 23 | version = "~> 9.2.0" 24 | 25 | project_id = var.vpc_project_id 26 | network_name = var.vpc_name 27 | routing_mode = "GLOBAL" 28 | 29 | subnets = local.nested_subnets 30 | 31 | secondary_ranges = local.nested_secondary_subnets 32 | } 33 | 34 | # Cloud NAT Module 35 | module "cluster_nat" { 36 | depends_on = [module.vpc] 37 | for_each = local.distinct_cluster_regions 38 | 39 | source = "terraform-google-modules/cloud-nat/google" 40 | version = "~> 5.0" 41 | create_router = true 42 | project_id = var.project_id 43 | region = each.key 44 | router = "gke-toolkit-rtr-${each.key}" 45 | network = local.vpc_selflink 46 | source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_PRIMARY_IP_RANGES" 47 | } -------------------------------------------------------------------------------- /terraform/modules/network/provider.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">=0.13.0" 19 | required_providers { 20 | google = { 21 | source = "hashicorp/google" 22 | version = ">= 5.41, < 7" 23 | } 24 | google-beta = { 25 | source = "hashicorp/google-beta" 26 | version = ">= 5.41, < 7" 27 | } 28 | } 29 | } 30 | 31 | provider "google" { 32 | project = var.vpc_project_id 33 | } 34 | -------------------------------------------------------------------------------- /terraform/modules/network/shared_vpc.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Performs necessary steps to attach service project to Shared VPC host project 18 | // Modules and resources below do not get executed if SHARED_VPC=false 19 | 20 | resource "google_compute_shared_vpc_host_project" "host_project" { 21 | count = var.shared_vpc ? 1 : 0 22 | depends_on = [ 23 | module.vpc, 24 | ] 25 | provider = google-beta 26 | project = var.vpc_project_id 27 | } 28 | 29 | resource "google_compute_subnetwork_iam_binding" "subnet_networkuser" { 30 | depends_on = [ 31 | module.vpc 32 | ] 33 | for_each = { for key, value in var.cluster_config : key => value if var.shared_vpc != false } 34 | project = var.vpc_project_id 35 | region = each.value.region 36 | subnetwork = each.value.subnet_name 37 | role = "roles/compute.networkUser" 38 | members = [ 39 | "serviceAccount:${local.clu_service_account}", 40 | "serviceAccount:${local.prj_service_account}", 41 | ] 42 | } 43 | 44 | resource "google_compute_subnetwork_iam_binding" "subnet_networkuser_cp" { 45 | count = var.shared_vpc ? 1 : 0 46 | depends_on = [ 47 | google_compute_subnetwork_iam_binding.subnet_networkuser 48 | ] 49 | project = var.vpc_project_id 50 | region = "us-central1" 51 | subnetwork = "admin-control-plane" 52 | role = "roles/compute.networkUser" 53 | members = [ 54 | "serviceAccount:${local.clu_service_account}", 55 | "serviceAccount:${local.prj_service_account}", 56 | ] 57 | } 58 | 59 | resource "google_project_iam_binding" "shared_vpc_serviceagent" { 60 | count = var.shared_vpc ? 1 : 0 61 | depends_on = [ 62 | google_compute_subnetwork_iam_binding.subnet_networkuser 63 | ] 64 | role = "roles/container.hostServiceAgentUser" 65 | project = var.vpc_project_id 66 | members = [ 67 | "serviceAccount:${local.clu_service_account}", 68 | ] 69 | } 70 | 71 | resource "google_compute_shared_vpc_service_project" "attach_toolkit" { 72 | count = var.shared_vpc ? 1 : 0 73 | depends_on = [ 74 | google_compute_subnetwork_iam_binding.subnet_networkuser, 75 | google_project_iam_binding.shared_vpc_serviceagent, 76 | ] 77 | provider = google-beta 78 | host_project = var.vpc_project_id 79 | service_project = var.project_id 80 | } 81 | -------------------------------------------------------------------------------- /terraform/modules/network/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "The project ID to host the cluster in" 20 | } 21 | 22 | variable "vpc_project_id" { 23 | type = string 24 | description = "The Share VPC Project ID - This is optional and only valid if a Shared VPC is used" 25 | default = "" 26 | } 27 | 28 | variable "shared_vpc" { 29 | type = bool 30 | description = "Determines whether to create a standalone VPC or use an existing Shared VPC" 31 | default = false 32 | } 33 | 34 | variable "region" { 35 | type = string 36 | description = "The region to host the cluster in" 37 | default = "us-central1" 38 | } 39 | 40 | variable "vpc_name" { 41 | type = string 42 | description = "The name of the Shared VPC - This is optional and only valid if a Shared VPC is used" 43 | default = "" 44 | } 45 | 46 | variable "vpc_ip_range_pods_name" { 47 | type = string 48 | description = "The secondary ip range to use for pods in the shared vpc - This is optional and only valid if a Shared VPC is used" 49 | default = "" 50 | } 51 | 52 | variable "vpc_ip_range_services_name" { 53 | type = string 54 | description = "The secondary ip range to use for services in the shared vpc - This is optional and only valid if a Shared VPC is used" 55 | default = "" 56 | } 57 | 58 | variable "cluster_config" { 59 | description = "For each cluster, create an object that contain the required fields" 60 | default = {} 61 | } -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # End-to-end Tests 2 | 3 | This directory contains Cloud Build pipelines and test scripts for the GKE PoC Toolkit. 4 | -------------------------------------------------------------------------------- /test/e2e/cleanup-cloud-run/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:latest 2 | 3 | # install basic dependencies 4 | RUN apt-get update && apt-get install -y --no-install-recommends \ 5 | ca-certificates \ 6 | curl \ 7 | gnupg2 \ 8 | software-properties-common \ 9 | wget \ 10 | python3 \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-sdk -y 14 | 15 | COPY delete_old_projects.sh . 16 | EXPOSE 8080 17 | RUN chmod +x delete_old_projects.sh 18 | CMD ["/delete_old_projects.sh"] 19 | -------------------------------------------------------------------------------- /test/e2e/cleanup-cloud-run/README.md: -------------------------------------------------------------------------------- 1 | # CI Project Cleanup job 2 | 3 | This directory contains the Dockerfile and bash script for a Cloud Run service that periodically deletes CI projects older than 6 hours. 4 | 5 | The reason we have a job for this, rather than a Cloud Build cleanup step in the pipeline, is because if a job fails, the Cloud Build pipeline will stop abruptly and not clean up the project. 6 | Cloud Build doesn't have a good way to "jump to" or "cleanup" in the case of a failed build. So this Cloud Run service does a periodic sweep through the test project folder and cleans up old projects. 7 | 8 | The image can be found here: `gcr.io/gkepoctoolkit/ci-project-cleanup:latest` 9 | 10 | Note that this image must be configured with a service account key and GCP project folder ID to watch. We use the Cloud Run + Secret Manager integration to pass this sensitive data as env vars. 11 | -------------------------------------------------------------------------------- /test/e2e/cleanup-cloud-run/delete_old_projects.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export GPT_TEST_FOLDER_ID="673643246072" 3 | echo "Folder ID is: $GPT_TEST_FOLDER_ID" 4 | 5 | while true 6 | do 7 | echo "Deleting all projects more than 6hr old in folder: $GPT_TEST_FOLDER_ID" 8 | gcloud projects list --filter="parent.id=$GPT_TEST_FOLDER_ID AND parent.type=folder" --format='value(project_id)' | while read project_id; do 9 | CREATE_TIME=`gcloud projects describe $project_id --format='value(createTime)'` 10 | echo "Project: $project_id created at: $CREATE_TIME" 11 | SIX_HOURS_AGO=`date -d '6 hour ago' "+%Y-%m-%dT%H:%M:%SZ"` 12 | if [[ $CREATE_TIME < $SIX_HOURS_AGO ]]; then 13 | gcloud alpha resource-manager liens list --project $project_id --format='value(name)' | while read lien; do 14 | echo "🗑 Deleting lien: $lien" 15 | gcloud alpha resource-manager liens delete $lien --project $project_id 16 | done 17 | 18 | echo "🌎 Removing shared vpc permissions on test project folder for serviceAccount:e2e-test@$project_id.iam.gserviceaccount.com..." 19 | gcloud beta resource-manager folders remove-iam-policy-binding $GPT_TEST_FOLDER_ID --member="serviceAccount:e2e-test@$project_id.iam.gserviceaccount.com" --role="roles/compute.xpnAdmin" 20 | 21 | echo "❌ Deleting project: $project_id" 22 | gcloud projects delete --quiet $project_id 23 | else 24 | echo "✅ Project $project_id is newer than 6 hours (it was created at $CREATE_TIME), keeping..." 25 | fi 26 | done 27 | sleep 600 28 | done 29 | -------------------------------------------------------------------------------- /test/e2e/cloudbuild-default-config.yaml: -------------------------------------------------------------------------------- 1 | steps: 2 | - name: 'gcr.io/cloud-builders/gcloud:latest' 3 | id: create-test-project 4 | entrypoint: 'bash' 5 | args: 6 | - '-eEuo' 7 | - 'pipefail' 8 | - '-c' 9 | - |- 10 | if [ -n "$_HEAD_REPO_URL" ] 11 | then 12 | echo "⬆️ THIS IS A PR- _HEAD_REPO_URL is set ($_HEAD_REPO_URL)" 13 | RAW_REPO_NAME="$_HEAD_REPO_URL//terraform/modules/" 14 | export CLEAN_REPO_NAME=${RAW_REPO_NAME#*//} 15 | export CLEAN_BRANCH_NAME=$_HEAD_BRANCH 16 | else 17 | echo "😺 THIS IS A DIRECT MAIN BRANCH COMMIT, NOT A PR, _HEAD_REPO_URL is unset" 18 | export CLEAN_REPO_NAME="github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 19 | export CLEAN_BRANCH_NAME="main" 20 | fi 21 | echo "🌟 REPO INFO: NAME: $$CLEAN_REPO_NAME, BRANCH: $$CLEAN_BRANCH_NAME" 22 | TIMESTAMP=`date "+%m%d%y-%H%M%S"` 23 | TEST_PROJECT_ID="gpt-$$TIMESTAMP" 24 | echo $$TEST_PROJECT_ID > /workspace/test-project-id.txt 25 | gcloud projects create $$TEST_PROJECT_ID --folder="$$FOLDER_ID" 26 | gcloud projects add-iam-policy-binding $$TEST_PROJECT_ID --member=serviceAccount:152393131587@cloudbuild.gserviceaccount.com --role=roles/owner 27 | gcloud services enable cloudbilling.googleapis.com --project=$$TEST_PROJECT_ID 28 | gcloud alpha billing projects link $$TEST_PROJECT_ID --billing-account $$BILLING_ID 29 | secretEnv: ['FOLDER_ID', 'BILLING_ID'] 30 | - name: 'gcr.io/cloud-builders/gcloud:latest' 31 | id: setup-gcloud-auth 32 | entrypoint: 'bash' 33 | args: 34 | - '-eEuo' 35 | - 'pipefail' 36 | - '-c' 37 | - |- 38 | SA_NAME="e2e-test" 39 | KEY_PATH="/workspace/gcloud/e2e-test.json" 40 | TEST_PROJECT_ID=$(cat "/workspace/test-project-id.txt") 41 | gcloud config set project $$TEST_PROJECT_ID 42 | gcloud iam service-accounts create $$SA_NAME --project=$$TEST_PROJECT_ID 43 | gcloud projects add-iam-policy-binding $$TEST_PROJECT_ID --member="serviceAccount:$$SA_NAME@$$TEST_PROJECT_ID.iam.gserviceaccount.com" --role="roles/owner" 44 | gcloud iam service-accounts keys create $$KEY_PATH \ 45 | --iam-account=$$SA_NAME@$$TEST_PROJECT_ID.iam.gserviceaccount.com 46 | gcloud auth activate-service-account $$SA_NAME@$$TEST_PROJECT_ID.iam.gserviceaccount.com --key-file=$$KEY_PATH --project=$$TEST_PROJECT_ID 47 | export GOOGLE_APPLICATION_CREDENTIALS=$$KEY_PATH 48 | gcloud auth application-default print-access-token 49 | gcloud auth list 50 | gcloud config set project $$TEST_PROJECT_ID 51 | sleep 5 52 | gcloud services enable iam.googleapis.com compute.googleapis.com cloudresourcemanager.googleapis.com 53 | gcloud iam service-accounts list 54 | - name: 'gcr.io/gkepoctoolkit/e2etest:latest' 55 | id: test-cli-compile 56 | entrypoint: 'bash' 57 | args: 58 | - '-eEuo' 59 | - 'pipefail' 60 | - '-c' 61 | - |- 62 | TEST_PROJECT_ID=$(cat "/workspace/test-project-id.txt") 63 | pwd 64 | ls 65 | cd cli 66 | go build 67 | mkdir -p /workspace/test 68 | cp ./gkekitctl /workspace/test 69 | cd /workspace/test 70 | chmod +x ./gkekitctl 71 | - name: 'gcr.io/gkepoctoolkit/e2etest:latest' 72 | id: test-gkekitctl-init 73 | entrypoint: 'bash' 74 | args: 75 | - '-eEuo' 76 | - 'pipefail' 77 | - '-c' 78 | - |- 79 | printenv 80 | cd /workspace/test 81 | ./gkekitctl init 82 | if [ -n "$_HEAD_REPO_URL" ] 83 | then 84 | echo "⬆️ THIS IS A PR- _HEAD_REPO_URL is set ($_HEAD_REPO_URL)" 85 | RAW_REPO_NAME="$_HEAD_REPO_URL//terraform/modules/" 86 | export CLEAN_REPO_NAME=${RAW_REPO_NAME#*//} 87 | export CLEAN_BRANCH_NAME=$_HEAD_BRANCH 88 | else 89 | echo "😺 THIS IS A DIRECT MAIN BRANCH COMMIT, NOT A PR, _HEAD_REPO_URL is unset" 90 | export CLEAN_REPO_NAME="github.com/GoogleCloudPlatform/gke-poc-toolkit//terraform/modules/" 91 | export CLEAN_BRANCH_NAME="main" 92 | fi 93 | echo "🌟 REPO INFO: NAME: $$CLEAN_REPO_NAME, BRANCH: $$CLEAN_BRANCH_NAME" 94 | TEST_PROJECT_ID=$(cat "/workspace/test-project-id.txt") 95 | sed -i 's|.*tfModuleRepo.*|tfModuleRepo: '"$$CLEAN_REPO_NAME"'|' samples/default-config.yaml 96 | sed -i 's|.*tfModuleBranch.*|tfModuleBranch: '"$$CLEAN_BRANCH_NAME"'|' samples/default-config.yaml 97 | sed -i 's|.*clustersProjectId.*|clustersProjectId: '"$$TEST_PROJECT_ID"'|' samples/default-config.yaml 98 | sed -i 's|.*governanceProjectId.*|governanceProjectId: '"$$TEST_PROJECT_ID"'|' samples/default-config.yaml 99 | sed -i 's|.*vpcProjectId.*| vpcProjectId: '"$$TEST_PROJECT_ID"'|' samples/default-config.yaml 100 | echo "Final config for testing..." 101 | cat samples/default-config.yaml 102 | - name: 'gcr.io/gkepoctoolkit/e2etest:latest' 103 | id: test-gkekitctl-create 104 | entrypoint: 'bash' 105 | args: 106 | - '-eEuo' 107 | - 'pipefail' 108 | - '-c' 109 | - |- 110 | cd /workspace/test 111 | SA_NAME="e2e-test" 112 | TEST_PROJECT_ID=$(cat "/workspace/test-project-id.txt") 113 | KEY_PATH="/workspace/gcloud/e2e-test.json" 114 | gcloud auth activate-service-account $$SA_NAME@$$TEST_PROJECT_ID.iam.gserviceaccount.com --key-file=$$KEY_PATH --project=$$TEST_PROJECT_ID 115 | export GOOGLE_APPLICATION_CREDENTIALS=$$KEY_PATH 116 | gcloud auth application-default print-access-token 117 | gcloud config set project $$TEST_PROJECT_ID 118 | gcloud auth list 119 | ./gkekitctl apply --config samples/default-config.yaml 120 | timeout: "60m" 121 | availableSecrets: 122 | secretManager: 123 | - versionName: projects/${PROJECT_ID}/secrets/folder-id/versions/latest 124 | env: 'FOLDER_ID' 125 | - versionName: projects/${PROJECT_ID}/secrets/billing-id/versions/latest 126 | env: 'BILLING_ID' -------------------------------------------------------------------------------- /test/e2e/custom-builder/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22.0 2 | 3 | # install basic dependencies 4 | RUN apt-get update && apt-get install -y --no-install-recommends \ 5 | ca-certificates \ 6 | curl \ 7 | gnupg2 \ 8 | software-properties-common \ 9 | wget \ 10 | && rm -rf /var/lib/apt/lists/* 11 | 12 | 13 | # install Google Cloud SDK 14 | # https://cloud.google.com/sdk/docs/install#deb 15 | RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-sdk -y 16 | 17 | 18 | # install Terraform 19 | # https://www.terraform.io/cli/install/apt 20 | RUN curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - 21 | RUN apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" 22 | RUN apt-get update && apt-get install terraform 23 | 24 | # install kubectl 25 | # https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/ 26 | RUN apt-get update && apt-get install -y apt-transport-https ca-certificates curl && curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list && apt-get update && apt-get install -y kubectl 27 | -------------------------------------------------------------------------------- /test/e2e/custom-builder/README.md: -------------------------------------------------------------------------------- 1 | # E2E Test Custom Builder 2 | 3 | This directory contains a Dockerfile for a custom Cloud Builder used during testing. It's an image with Go, kubectl, gcloud, and Terrarform pre-installed. (Covering the user prerequisites to run the toolkit.) 4 | 5 | This container image can be found at: `gcr.io/gkepoctoolkit/e2etest:latest` 6 | --------------------------------------------------------------------------------