├── .gcloudignore ├── .github ├── actions │ └── build_and_publish_image │ │ └── action.yaml └── workflows │ ├── publish_all_images.yaml │ ├── publish_changed_images.yaml │ └── publish_release.yaml ├── .gitignore ├── LICENSE ├── README.md ├── cloudbuild.yaml ├── examples └── jupyter-notebook │ ├── README.md │ ├── bundle-manifests │ ├── patch-user-pod.yaml.tmpl │ ├── resource-jupyter-service.yaml │ ├── resource-jupyter-statefulset.yaml │ └── resource-jupyter-virtual-service.yaml.tmpl │ ├── cloudbuild.yaml │ ├── jupyter-notebook-bokerappconfig.yaml │ ├── jupyter-notebook-gitops-init.sh │ └── kustomization.yaml ├── images ├── cloudbuild.yaml ├── controller │ ├── .gitignore │ ├── Dockerfile │ ├── Makefile │ ├── cmd │ │ ├── app_finder │ │ │ └── app_finder.go │ │ ├── app_publisher │ │ │ └── app_publisher.go │ │ ├── image_finder │ │ │ └── image_finder.go │ │ ├── image_puller │ │ │ └── image_puller.go │ │ ├── image_puller_worker │ │ │ ├── docker_pull_parser.go │ │ │ └── image_puller_worker.go │ │ ├── pod_broker │ │ │ └── pod_broker.go │ │ └── reservation_broker │ │ │ └── reservation_broker.go │ ├── config │ │ ├── deployment │ │ │ ├── app │ │ │ │ ├── kustomization.yaml.tmpl │ │ │ │ ├── resource-namespace.yaml.tmpl │ │ │ │ ├── resource-network-policy.yaml │ │ │ │ └── resource-service-account.yaml.tmpl │ │ │ └── user │ │ │ │ └── kustomization.yaml.tmpl │ │ └── statefulset │ │ │ ├── app │ │ │ ├── jsonpatch-deploy-last-applied-resources.yaml │ │ │ └── kustomization.yaml.tmpl │ │ │ └── user │ │ │ ├── kustomization.yaml.tmpl │ │ │ ├── resource-namespace.yaml.tmpl │ │ │ ├── resource-network-policy.yaml │ │ │ └── resource-service-account.yaml.tmpl │ ├── go.mod │ ├── go.sum │ └── pkg │ │ ├── appconfig.go │ │ ├── appconfig_informer.go │ │ ├── appconfigbundle.go │ │ ├── appmanifest.go │ │ ├── appuserconfig.go │ │ ├── appuserconfig_informer.go │ │ ├── broker.go │ │ ├── builder.go │ │ ├── dockerconfig.go │ │ ├── imagequeuesync.go │ │ ├── informer.go │ │ ├── types.go │ │ └── util.go ├── gce-proxy │ ├── .dockerignore │ ├── .gitignore │ ├── Dockerfile │ └── src │ │ ├── index.js │ │ ├── package-lock.json │ │ └── package.json └── installer │ └── Dockerfile └── setup ├── .gitignore ├── README.md ├── example_tfvars ├── README.md └── selkies-cluster-min.auto.tfvars ├── infra ├── .gitignore ├── cloudbuild.yaml ├── cluster │ ├── .gitignore │ ├── cloudbuild.yaml │ ├── deploy.sh │ ├── locals.tf │ ├── main.tf │ ├── network.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf ├── create_cloudep.sh ├── deploy.sh ├── gcr.tf ├── lb.tf ├── networks.tf ├── node-pool-apps │ ├── cloudbuild.yaml │ ├── deploy.sh │ ├── main.tf │ ├── tier1-ubuntu.tf │ ├── tier1.tf │ ├── tier2.tf │ ├── variables.tf │ └── versions.tf ├── node-pool-gpu │ ├── cloudbuild.yaml │ ├── deploy.sh │ ├── gpu-cos.tf │ ├── locals.tf │ ├── main.tf │ ├── variables.tf │ └── versions.tf ├── outputs.tf ├── private-cluster │ ├── .gitignore │ ├── cloudbuild.yaml │ ├── deploy.sh │ ├── locals.tf │ ├── main.tf │ ├── network.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf ├── regional-lb │ ├── cloudbuild.yaml │ ├── create_cloudep.sh │ ├── deploy.sh │ ├── lb.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf ├── sa.tf ├── services.tf ├── variables.tf ├── versions.tf └── wi-sa │ ├── cloudbuild.yaml │ ├── deploy.sh │ ├── versions.tf │ └── wi.tf ├── manifests ├── base │ ├── autoneg-system │ │ ├── autoneg-serviceaccount.yaml │ │ ├── autoneg-system-namespace.yaml │ │ ├── autoneg.yaml │ │ └── kustomization.yaml │ ├── custom-metrics │ │ ├── custom-metrics-adapter.yaml │ │ ├── jsonpatch-add-wi.yaml │ │ ├── kustomization.yaml │ │ └── wi-sa-iampolicymember.yaml │ ├── ingress │ │ ├── default-gateway.yaml │ │ ├── kustomization.yaml │ │ └── virtualservice.yaml │ ├── istio │ │ ├── istiocontrolplane.yaml │ │ └── istiooperator-1.7.yaml │ ├── node │ │ ├── kustomization.yaml │ │ ├── node-init.yaml │ │ ├── node-termination-handler-ds.yaml │ │ ├── node-termination-handler-rbac.yaml │ │ ├── pd-balanced-storage-class.yaml │ │ ├── pd-ssd-storage-class.yaml │ │ └── pd-standard-storage-class.yaml │ └── pod-broker │ │ ├── app-publisher │ │ ├── config │ │ │ └── app-publish-job.yaml.tmpl │ │ └── kustomization.yaml │ │ ├── broker-deployment.yaml │ │ ├── crd.yaml │ │ ├── image-puller │ │ ├── config │ │ │ └── image-pull-job.yaml.tmpl │ │ ├── image-puller-daemonset.yaml │ │ ├── kustomization.yaml │ │ ├── patch-wait-for-image-cache.yaml │ │ └── subscription-reaper-cronjob.yaml │ │ ├── kustomization.yaml │ │ ├── namespace.yaml │ │ ├── network-policy.yaml │ │ └── redirect │ │ ├── kustomization.yaml │ │ ├── redirect-gateway.yaml │ │ ├── redirect-serviceentry.yaml │ │ └── redirect-virtualservice.yaml ├── cloudbuild.yaml ├── deploy.sh ├── fix_pod_cidr_masq.sh ├── install_istio_1.14.sh ├── install_istio_1.4.sh ├── install_istio_1.7.sh ├── make_generated_manifests.sh └── wait_for_clusters.sh └── scripts ├── add_iap_user.sh ├── create_autoscaling_cronjob.sh ├── create_oauth_client.sh ├── find_gpu.sh ├── fix_autoneg.sh ├── gcping.sh ├── gitops-init.sh ├── pod-broker-gitops-init.sh ├── selkies_cli.sh ├── stream_logs.sh ├── util.bash ├── wait_for_iap.sh └── workload_identity_test.sh /.gcloudignore: -------------------------------------------------------------------------------- 1 | .git 2 | .gitignore 3 | **/**/.terraform -------------------------------------------------------------------------------- /.github/actions/build_and_publish_image/action.yaml: -------------------------------------------------------------------------------- 1 | name: Build & publish an image 2 | 3 | description: Build an image from source, then publish it to GHCR 4 | 5 | inputs: 6 | build_args: 7 | description: "`ARG=value`s to pass to `docker build`" 8 | required: false 9 | dockerfile: 10 | default: "Dockerfile" 11 | description: Image source file name 12 | required: false 13 | github_personal_access_token: 14 | description: GitHub personal access token (PAT), for authentication 15 | required: true 16 | github_username: 17 | description: GitHub username, for authentication 18 | required: true 19 | image_name: 20 | description: The name for the image (i.e. "image" in "ghcr.io/user/repo/image:foo") 21 | required: true 22 | image_source_directory: 23 | description: Image source directory path 24 | required: true 25 | image_version_1: 26 | description: A tag for the image (i.e. "foo" in "ghcr.io/user/repo/image:foo") 27 | required: true 28 | image_version_2: 29 | description: Another tag for the image (i.e. "bar" in "ghcr.io/user/repo/image:bar") 30 | required: false 31 | use_cache: 32 | default: "true" 33 | description: Use cached layers from pre-published image with version 1 (true/false) 34 | required: false 35 | 36 | runs: 37 | using: composite 38 | steps: 39 | - shell: bash 40 | run: | 41 | set -x 42 | echo ${{ inputs.github_personal_access_token }} | docker login ghcr.io -u ${{ inputs.github_username }} --password-stdin 43 | export BUILD_ARGS="" 44 | if [[ -n "${{ inputs.build_args }}" ]]; then 45 | IFS=";" read -ra args <<< "${{ inputs.build_args }}" 46 | for arg in ${args[@]}; do BUILD_ARGS+="--build-arg $arg "; done 47 | fi 48 | 49 | export IMAGE_TAG_URL_PREFIX=ghcr.io/selkies-project/selkies/ 50 | export IMAGE_TAG_1="${IMAGE_TAG_URL_PREFIX}${{ inputs.image_name }}:${{ inputs.image_version_1 }}" 51 | if [[ "${{ inputs.use_cache }}" == "true" ]]; then 52 | docker pull $IMAGE_TAG_1 || true 53 | (cd ${{ inputs.image_source_directory }} && docker build $BUILD_ARGS -f ${{ inputs.dockerfile }} --cache-from $IMAGE_TAG_1 -t $IMAGE_TAG_1 .) 54 | else 55 | (cd ${{ inputs.image_source_directory }} && docker build $BUILD_ARGS -f ${{ inputs.dockerfile }} -t $IMAGE_TAG_1 .) 56 | fi 57 | docker push $IMAGE_TAG_1 58 | 59 | if [ '${{ inputs.image_version_2 }}' != '' ]; then 60 | export IMAGE_TAG_2="${IMAGE_TAG_URL_PREFIX}${{ inputs.image_name }}:${{ inputs.image_version_2 }}${{ inputs.image_version_suffix }}" 61 | docker tag $IMAGE_TAG_1 $IMAGE_TAG_2 62 | docker push $IMAGE_TAG_2 63 | fi 64 | -------------------------------------------------------------------------------- /.github/workflows/publish_all_images.yaml: -------------------------------------------------------------------------------- 1 | name: Publish all images 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | - master 8 | paths: 9 | - '.github/**' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | # Note: When modifying this job, copy modifications to all other workflows' image jobs. 14 | all_images: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | include: 19 | - name: broker-installer 20 | source_directory: images/installer 21 | 22 | - name: controller 23 | source_directory: images/controller 24 | 25 | - name: gce-proxy 26 | source_directory: images/gce-proxy 27 | 28 | name: ${{ matrix.name }}${{ matrix.version_suffix }} image build & publish 29 | steps: 30 | - uses: actions/checkout@v2 31 | 32 | - name: Build & publish ${{ matrix.name }} image 33 | uses: ./.github/actions/build_and_publish_image 34 | with: 35 | build_args: ${{ matrix.build_args }} 36 | github_personal_access_token: ${{ secrets.GITHUB_TOKEN }} 37 | github_username: $GITHUB_ACTOR 38 | image_name: ${{ matrix.name }} 39 | image_source_directory: ${{ matrix.source_directory }} 40 | image_version_1: $GITHUB_REF_NAME${{ matrix.version_suffix }} 41 | -------------------------------------------------------------------------------- /.github/workflows/publish_changed_images.yaml: -------------------------------------------------------------------------------- 1 | name: Publish changed images 2 | 3 | on: 4 | push: 5 | branches: 6 | - dev 7 | - master 8 | paths: 9 | - 'images/**' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | # Note: When modifying this job, copy modifications to all other workflows' image jobs. 14 | changed_images: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | include: 19 | - name: broker-installer 20 | source_directory: images/installer 21 | source_files_for_diff: images/installer 22 | 23 | - name: controller 24 | source_directory: images/controller 25 | source_files_for_diff: images/controller 26 | 27 | - name: gce-proxy 28 | source_directory: images/gce-proxy 29 | source_files_for_diff: images/gce-proxy 30 | 31 | name: ${{ matrix.name }}${{ matrix.version_suffix }} image build & publish 32 | steps: 33 | - uses: actions/checkout@v2 34 | with: 35 | fetch-depth: 2 # This is for changed-files. 36 | 37 | - id: check 38 | name: Check for changes to ${{ matrix.name }} image source 39 | uses: tj-actions/changed-files@v1.1.2 40 | with: 41 | files: ${{ matrix.source_files_for_diff }} 42 | 43 | - name: Build & publish ${{ matrix.name }} image 44 | if: steps.check.outputs.any_changed == 'true' || steps.check.outputs.any_deleted == 'true' 45 | uses: ./.github/actions/build_and_publish_image 46 | with: 47 | build_args: ${{ matrix.build_args }} 48 | github_personal_access_token: ${{ secrets.GITHUB_TOKEN }} 49 | github_username: $GITHUB_ACTOR 50 | image_name: ${{ matrix.name }} 51 | image_source_directory: ${{ matrix.source_directory }} 52 | image_version_1: $GITHUB_REF_NAME${{ matrix.version_suffix }} 53 | -------------------------------------------------------------------------------- /.github/workflows/publish_release.yaml: -------------------------------------------------------------------------------- 1 | name: Publish release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | workflow_dispatch: 8 | 9 | jobs: 10 | # Note: When modifying this job, copy modifications to all other workflows' image jobs. 11 | all_images: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | include: 16 | - name: broker-installer 17 | source_directory: images/installer 18 | 19 | - name: controller 20 | source_directory: images/controller 21 | 22 | - name: gce-proxy 23 | source_directory: images/gce-proxy 24 | 25 | name: ${{ matrix.name }}${{ matrix.version_suffix }} image build & publish 26 | steps: 27 | - uses: actions/checkout@v2 28 | 29 | - name: Build & publish ${{ matrix.name }} image 30 | uses: ./.github/actions/build_and_publish_image 31 | with: 32 | build_args: ${{ matrix.build_args }} 33 | github_personal_access_token: ${{ secrets.GITHUB_TOKEN }} 34 | github_username: $GITHUB_ACTOR 35 | image_name: ${{ matrix.name }} 36 | image_source_directory: ${{ matrix.source_directory }} 37 | image_version_1: $GITHUB_REF_NAME${{ matrix.version_suffix }} 38 | image_version_2: latest${{ matrix.version_suffix }} 39 | 40 | create_release: 41 | runs-on: ubuntu-latest 42 | needs: 43 | - all_images 44 | steps: 45 | - id: create_release 46 | uses: actions/create-release@v1 47 | env: 48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | with: 50 | draft: true 51 | prerelease: false 52 | release_name: Release ${{ github.ref_name }} 53 | tag_name: ${{ github.ref_name }} 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vars*.sh 2 | NOTES.md 3 | setup/infra/**/*.auto.tfvars -------------------------------------------------------------------------------- /examples/jupyter-notebook/README.md: -------------------------------------------------------------------------------- 1 | ## Jupyter Notebook Broker App Example 2 | 3 | This example shows how to deploy a per-user Jupyter Notebook. 4 | 5 | ## Setup 6 | 7 | 1. Clone the source repo and change to the examples directory: 8 | 9 | ```bash 10 | gcloud source repos clone kube-pod-broker --project=cloud-ce-pso-shared-code 11 | ``` 12 | 13 | ```bash 14 | cd kube-pod-broker/examples 15 | ``` 16 | 17 | ## Option 1 - Deployment with GitOps 18 | 19 | 1. Run script to create and configure gitops Cloud Source Repository for the app: 20 | 21 | ```bash 22 | cd jupyter-notebook 23 | ``` 24 | 25 | ```bash 26 | ./jupyter-notebook-gitops-init.sh 27 | ``` 28 | 29 | > Follow the prompts to create the repository and trigger the cloud build. 30 | 31 | 2. Wait for Cloud Build to complete. 32 | 33 | 3. Navigate to the broker web interface and launch the app. 34 | 35 | 4. Make any changes to the app and push them to the `gitops` remote to deploy them to the cluster. 36 | 37 | 5. If you want to deploy to clusters in other regions, edit the `cloudbuild.yaml` file. 38 | 39 | ## Option 2 - Deployment with kubectl 40 | 41 | 1. Deploy the manifests and BrokerAppConfig custom resource: 42 | 43 | ```bash 44 | kubectl apply -k jupyter-notebook/ 45 | ``` 46 | 47 | 2. Navigate to the broker web interface and launch the app. -------------------------------------------------------------------------------- /examples/jupyter-notebook/bundle-manifests/patch-user-pod.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: jupyter 19 | annotations: 20 | app.broker/user: "{{.User}}" 21 | spec: 22 | {{- if .UserParams.enablePersistence}} 23 | {{- if eq .UserParams.enablePersistence "true"}} 24 | volumeClaimTemplates: 25 | - metadata: 26 | name: persist 27 | annotations: 28 | app.broker/user: "{{.User}}" 29 | spec: 30 | accessModes: [ "ReadWriteOnce" ] 31 | resources: 32 | requests: 33 | storage: 50Gi 34 | {{- end}} 35 | {{- end}} 36 | template: 37 | metadata: 38 | annotations: 39 | app.broker/user: "{{.User}}" 40 | spec: 41 | nodeSelector: 42 | app.broker/tier: {{ .NodeTier.NodeLabel }} 43 | cloud.google.com/gke-nodepool: "{{ .NodeTier.NodeLabel }}" 44 | affinity: 45 | nodeAffinity: 46 | requiredDuringSchedulingIgnoredDuringExecution: 47 | nodeSelectorTerms: 48 | - matchExpressions: 49 | - key: app.broker/initialized 50 | operator: Exists 51 | - key: app.broker/tier 52 | operator: In 53 | values: ["{{ .NodeTier.NodeLabel }}"] 54 | tolerations: 55 | - key: "app.broker/tier" 56 | effect: "NoSchedule" 57 | operator: "Exists" 58 | - key: "app.broker/node-init" 59 | effect: "NoSchedule" 60 | operator: "Exists" 61 | containers: 62 | - name: jupyter 63 | # Replace image with user provided image from broker. 64 | image: {{.ImageRepo}}:{{.ImageTag}} 65 | {{- if .UserParams.enablePersistence}} 66 | {{- if eq .UserParams.enablePersistence "true"}} 67 | volumeMounts: 68 | - name: persist 69 | mountPath: /home 70 | {{- end}} 71 | {{- end}} 72 | env: 73 | - name: BROKER_USER 74 | value: "{{.User}}" 75 | resources: 76 | {{- if .NodeTier.Resources.Requests }} 77 | requests: 78 | {{- if .NodeTier.Resources.Requests.CPU }} 79 | cpu: {{.NodeTier.Resources.Requests.CPU}} 80 | {{- end}} 81 | {{- if .NodeTier.Resources.Requests.Memory }} 82 | memory: {{.NodeTier.Resources.Requests.Memory}} 83 | {{- end}} 84 | {{- end}} 85 | {{- if .NodeTier.Resources.Limits }} 86 | limits: 87 | {{- if .NodeTier.Resources.Limits.CPU }} 88 | cpu: {{.NodeTier.Resources.Limits.CPU}} 89 | {{- end}} 90 | {{- if .NodeTier.Resources.Limits.Memory }} 91 | memory: {{.NodeTier.Resources.Limits.Memory}} 92 | {{- end}} 93 | {{- end}} -------------------------------------------------------------------------------- /examples/jupyter-notebook/bundle-manifests/resource-jupyter-service.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: jupyter 19 | spec: 20 | selector: 21 | app: jupyter 22 | ports: 23 | # Port 80 is the port routed by the virtual service. 24 | - port: 80 25 | name: web 26 | targetPort: 8888 27 | -------------------------------------------------------------------------------- /examples/jupyter-notebook/bundle-manifests/resource-jupyter-statefulset.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: StatefulSet 17 | metadata: 18 | name: jupyter 19 | spec: 20 | selector: 21 | matchLabels: 22 | app: jupyter 23 | replicas: 1 24 | serviceName: jupyter 25 | updateStrategy: 26 | type: RollingUpdate 27 | volumeClaimTemplates: [] 28 | template: 29 | metadata: 30 | labels: 31 | app: jupyter 32 | spec: 33 | serviceAccountName: default 34 | containers: 35 | - name: jupyter 36 | image: gcr.io/kubeflow/jupyterhub:1.0 37 | ports: 38 | - containerPort: 8888 39 | command: ["/bin/bash"] 40 | args: 41 | - "-exc" 42 | - | 43 | groupadd jupyter --gid 1000 44 | useradd jupyter --uid 1000 --gid 1000 --shell /bin/bash 45 | [[ ! -d /home/jupyter ]] && mkdir -p /home/jupyter && chown jupyter:jupyter /home/jupyter 46 | cd /home/jupyter 47 | su -c "/usr/local/bin/jupyter-notebook --ip=0.0.0.0 --port 8888 --NotebookApp.token='' --NotebookApp.password='' --NotebookApp.base_url='/jupyter-notebook/' --no-browser" jupyter 48 | -------------------------------------------------------------------------------- /examples/jupyter-notebook/bundle-manifests/resource-jupyter-virtual-service.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.istio.io/v1beta1 16 | kind: VirtualService 17 | metadata: 18 | name: {{.ServiceName}} 19 | spec: 20 | hosts: 21 | - "*" 22 | gateways: 23 | - pod-broker-system/pod-broker-gateway 24 | http: 25 | ### 26 | # rule for webrtc 27 | ### 28 | - match: 29 | - uri: 30 | prefix: /{{.App}}/ 31 | headers: 32 | cookie: 33 | regex: ".*broker_{{.App}}={{.CookieValue}}.*" 34 | route: 35 | - destination: 36 | host: {{.FullName}}-{{.ServiceName}} 37 | port: 38 | number: 80 39 | - match: 40 | - uri: 41 | prefix: / 42 | route: 43 | - destination: 44 | host: pod-broker 45 | port: 46 | number: 80 47 | -------------------------------------------------------------------------------- /examples/jupyter-notebook/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 300s 16 | 17 | substitutions: 18 | _REGION: us-central1 19 | 20 | steps: 21 | ### 22 | # Deploy manifests to cluster. 23 | ### 24 | - name: "gcr.io/cloud-builders/kubectl" 25 | id: deploy-manifests 26 | args: 27 | - apply 28 | - -k 29 | - . 30 | env: 31 | - "PROJECT_ID=${PROJECT_ID}" 32 | - "CLOUDSDK_CORE_PROJECT=${PROJECT_ID}" 33 | - "CLOUDSDK_COMPUTE_REGION=${_REGION}" 34 | - "CLOUDSDK_CONTAINER_CLUSTER=broker-${_REGION}" 35 | -------------------------------------------------------------------------------- /examples/jupyter-notebook/jupyter-notebook-bokerappconfig.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: gcp.solutions/v1 16 | kind: BrokerAppConfig 17 | metadata: 18 | name: jupyter-notebook 19 | spec: 20 | version: 1.0.0 21 | name: jupyter-notebook 22 | displayName: Jupyter Notebook 23 | description: Jupyter Notebook 24 | icon: https://upload.wikimedia.org/wikipedia/commons/3/38/Jupyter_logo.svg 25 | bundle: 26 | configMapRef: 27 | name: jupyter-notebook-manifests-1.2.0 28 | serviceName: jupyter 29 | defaultRepo: gcr.io/cloud-solutions-images/jupyterhub 30 | defaultTag: "1.0" 31 | userParams: 32 | - name: enablePersistence 33 | displayName: "Persistent Home" 34 | type: bool 35 | default: "false" 36 | defaultTier: chill 37 | nodeTiers: 38 | - name: chill 39 | nodeLabel: tier1 40 | resources: 41 | requests: 42 | cpu: 1 43 | memory: 1Gi 44 | limits: 45 | cpu: 4 46 | memory: 4Gi 47 | - name: ludicrous 48 | nodeLabel: tier2 49 | resources: 50 | requests: 51 | cpu: 1 52 | memory: 1Gi 53 | -------------------------------------------------------------------------------- /examples/jupyter-notebook/jupyter-notebook-gitops-init.sh: -------------------------------------------------------------------------------- 1 | ../../setup/scripts/gitops-init.sh -------------------------------------------------------------------------------- /examples/jupyter-notebook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | 18 | namespace: pod-broker-system 19 | 20 | generatorOptions: 21 | # Generate stable ConfigMap names for compatability with the BrokerAppConfig configMapRef. 22 | disableNameSuffixHash: true 23 | 24 | configMapGenerator: 25 | - name: jupyter-notebook-manifests-1.2.0 26 | files: 27 | - bundle-manifests/patch-user-pod.yaml.tmpl 28 | - bundle-manifests/resource-jupyter-service.yaml 29 | - bundle-manifests/resource-jupyter-statefulset.yaml 30 | - bundle-manifests/resource-jupyter-virtual-service.yaml.tmpl 31 | 32 | resources: 33 | - jupyter-notebook-bokerappconfig.yaml 34 | -------------------------------------------------------------------------------- /images/controller/.gitignore: -------------------------------------------------------------------------------- 1 | test/ -------------------------------------------------------------------------------- /images/controller/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.15-alpine as build 16 | RUN apk add --no-cache -u git 17 | ENV GO111MODULE=on 18 | WORKDIR /go/src/selkies.io/controller 19 | COPY go.mod . 20 | COPY go.sum . 21 | RUN go mod download 22 | COPY . . 23 | RUN go build cmd/app_finder/app_finder.go 24 | RUN go build cmd/app_publisher/app_publisher.go 25 | RUN go build cmd/image_finder/image_finder.go 26 | RUN go build cmd/image_puller/image_puller.go 27 | RUN go build -o image_puller_worker cmd/image_puller_worker/*.go 28 | RUN go build cmd/pod_broker/pod_broker.go 29 | RUN go build cmd/reservation_broker/reservation_broker.go 30 | 31 | FROM gcr.io/k8s-skaffold/skaffold:v1.10.1 as skaffold 32 | FROM gcr.io/google.com/cloudsdktool/cloud-sdk:alpine 33 | 34 | # Install kustomize 35 | COPY --from=skaffold /usr/local/bin/kustomize /usr/local/bin/kustomize 36 | 37 | # Install kubectl 38 | COPY --from=skaffold /usr/local/bin/kubectl /usr/local/bin/kubectl 39 | 40 | # Copy build from previous layer 41 | COPY --from=build /go/src/selkies.io/controller/app_finder /usr/local/bin/app-finder 42 | COPY --from=build /go/src/selkies.io/controller/app_publisher /usr/local/bin/app-publisher 43 | COPY --from=build /go/src/selkies.io/controller/image_finder /usr/local/bin/image-finder 44 | COPY --from=build /go/src/selkies.io/controller/image_puller /usr/local/bin/image-puller 45 | COPY --from=build /go/src/selkies.io/controller/image_puller_worker /usr/local/bin/image-puller-worker 46 | COPY --from=build /go/src/selkies.io/controller/pod_broker /usr/local/bin/pod-broker 47 | COPY --from=build /go/src/selkies.io/controller/reservation_broker /usr/local/bin/reservation-broker 48 | 49 | # Copy build assets 50 | WORKDIR /opt/broker/buildsrc/ 51 | COPY config . 52 | 53 | WORKDIR /var/run/build 54 | 55 | ENTRYPOINT ["/usr/local/bin/pod-broker"] 56 | -------------------------------------------------------------------------------- /images/controller/Makefile: -------------------------------------------------------------------------------- 1 | APPS_JSON := test/var/run/buildsrc/apps.json 2 | BROKER_ENV := test/env 3 | BUILD_SRC_DIR := $(dir $(APPS_JSON)) 4 | BUILD_DIR := test/var/run/build 5 | BUILD_USR_DIR := test/var/run/buildusr 6 | BROKER_ENV := test/env 7 | USER_ID := $(shell id -u) 8 | GROUP_ID := $(shell id -g) 9 | 10 | build: 11 | docker build -t selkies-controller:latest . 12 | 13 | $(BUILD_DIR): 14 | mkdir -p $@ 15 | 16 | $(BUILD_SRC_DIR): 17 | mkdir -p $@ 18 | 19 | $(BUILD_USR_DIR): 20 | mkdir -p $@ 21 | 22 | $(BROKER_ENV): 23 | mkdir -p $(dir $@) 24 | kubectl -n pod-broker-system get cm pod-broker-config -o json | jq -r '.data | to_entries[] | [.key, .value] | join("=")' > $@ 25 | kubectl -n pod-broker-system get secret oauth-client-id -o json | jq -r '.data.CLIENT_ID' | base64 -d | xargs printf "OAUTH_CLIENT_ID=%s\n" >> $@ 26 | kubectl -n pod-broker-system get secret pod-broker -o json | jq -r '.data.COOKIE_SECRET' | base64 -d | xargs printf "COOKIE_SECRET=%s\n" >> $@ 27 | gcloud config get-value project 2>/dev/null | xargs printf "PROJECT_ID=%s\n" >> $@ 28 | 29 | .PRECIOUS: $(APPS_JSON) 30 | $(APPS_JSON): $(BROKER_ENV) $(BUILD_SRC_DIR) 31 | kubectl version >/dev/null && \ 32 | docker run -it --rm --entrypoint=app-finder \ 33 | -u $(USER_ID):$(GROUP_ID) \ 34 | --env-file=test/env \ 35 | -e NAMESPACE=pod-broker-system \ 36 | -e SINGLE_ITERATION=true \ 37 | -e KUBECONFIG=/etc/kubeconfig \ 38 | -v $(abspath $(BUILD_SRC_DIR)):/var/run/buildsrc \ 39 | -v $$HOME/.kube/config:/etc/kubeconfig \ 40 | selkies-controller:latest 41 | 42 | .PHONY: fetch 43 | fetch: clean-json $(APPS_JSON) 44 | 45 | test-%: $(APPS_JSON) $(BUILD_DIR) $(BUILD_USR_DIR) 46 | kubectl version >/dev/null && \ 47 | docker run -it --rm --entrypoint=$* \ 48 | -u $(USER_ID):$(GROUP_ID) \ 49 | --env-file=test/env \ 50 | -e NAMESPACE=pod-broker-system \ 51 | -e KUBECONFIG=/etc/kubeconfig \ 52 | -v $$HOME/.kube/config:/etc/kubeconfig \ 53 | -v $(abspath $(BUILD_DIR)):/var/run/build \ 54 | -v $(abspath $(BUILD_SRC_DIR)):/var/run/buildsrc \ 55 | -v $(abspath $(BUILD_USR_DIR)):/var/run/buildusr \ 56 | -p 8082:8082 \ 57 | -p 8081:8081 \ 58 | selkies-controller:latest 59 | 60 | .PHONY: clean 61 | clean-json: 62 | rm -f $(APPS_JSON) 63 | clean: 64 | rm -rf $(BUILD_SRC_DIR) $(BUILD_DIR) $(BUILD_USR_DIR) 65 | -------------------------------------------------------------------------------- /images/controller/cmd/image_puller_worker/image_puller_worker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "os" 12 | "os/exec" 13 | "time" 14 | ) 15 | 16 | type statMetadataSpec struct { 17 | Name string `json:"name"` 18 | Node string `json:"node"` 19 | Tier string `json:"tier"` 20 | } 21 | 22 | type statProgressSpec struct { 23 | Image string `json:"image"` 24 | PullProgress int `json:"pull_progress"` 25 | } 26 | 27 | type statSpec struct { 28 | Metadta statMetadataSpec `json:"metadata"` 29 | Stats statProgressSpec `json:"stats"` 30 | } 31 | 32 | const ( 33 | timeout = 30 * time.Minute 34 | statsInterval = 1 * time.Second 35 | ) 36 | 37 | var ( 38 | image = flag.String("image", "", "image name and tag to pull") 39 | statsEndpoint = flag.String("stats", "", "endpoint to POST progress stats to") 40 | jobName = flag.String("jobName", "", "job name to use when POSTing stats") 41 | nodeName = flag.String("nodeName", "", "name of node the image is being pulled on, for POSTing stats") 42 | nodeTier = flag.String("nodeTier", "", "node tier the image is being pulled on, for POSTing stats") 43 | ) 44 | 45 | func main() { 46 | flag.Parse() 47 | if len(*image) == 0 { 48 | log.Printf("missing image") 49 | flag.PrintDefaults() 50 | os.Exit(1) 51 | } 52 | 53 | if !checkDocker() { 54 | log.Fatalf("docker not found in path.") 55 | } 56 | 57 | if !checkSkopeo() { 58 | log.Fatalf("skopeo not found in path.") 59 | } 60 | 61 | log.Printf("Pulling image: %s", *image) 62 | 63 | progressCh := make(chan int, 0) 64 | go func() { 65 | lastStat := 0 66 | start := time.Now() 67 | 68 | for p := range progressCh { 69 | lastStat = p 70 | elapsed := time.Now().Sub(start) 71 | if elapsed > statsInterval { 72 | log.Printf("Pull progress for %s: %d%%", *image, p) 73 | if len(*statsEndpoint) > 0 { 74 | if err := publishStats(*statsEndpoint, *image, lastStat, *jobName, *nodeName, *nodeTier); err != nil { 75 | log.Printf("ERROR: could not publish stats: %v", err) 76 | } 77 | } 78 | start = time.Now() 79 | } 80 | if p == 100 { 81 | break 82 | } 83 | } 84 | }() 85 | 86 | pullStart := time.Now() 87 | if err := dockerPullWithProgress(*image, progressCh, timeout); err != nil { 88 | log.Fatal(err) 89 | } 90 | 91 | totalPullTime := time.Now().Sub(pullStart) 92 | 93 | log.Printf("Done, image pulled in %.3f minutes", totalPullTime.Minutes()) 94 | } 95 | 96 | func checkDocker() bool { 97 | cmd := exec.Command("docker", "version") 98 | _, err := cmd.CombinedOutput() 99 | return err == nil 100 | } 101 | 102 | func checkSkopeo() bool { 103 | cmd := exec.Command("skopeo", "-v") 104 | _, err := cmd.CombinedOutput() 105 | return err == nil 106 | } 107 | 108 | func publishStats(statsEndpoint, image string, progress int, jobName, nodeName, nodeTier string) error { 109 | stat := statSpec{ 110 | Metadta: statMetadataSpec{ 111 | Name: jobName, 112 | Node: nodeName, 113 | Tier: nodeTier, 114 | }, 115 | Stats: statProgressSpec{ 116 | Image: image, 117 | PullProgress: progress, 118 | }, 119 | } 120 | jsonStr, _ := json.MarshalIndent(stat, "", " ") 121 | req, err := http.NewRequest("POST", statsEndpoint, bytes.NewBuffer(jsonStr)) 122 | if err != nil { 123 | return err 124 | } 125 | req.Header.Set("Content-Type", "application/json") 126 | client := &http.Client{} 127 | resp, err := client.Do(req) 128 | if err != nil { 129 | return err 130 | } 131 | defer resp.Body.Close() 132 | if resp.StatusCode != http.StatusOK { 133 | body, _ := ioutil.ReadAll(resp.Body) 134 | return fmt.Errorf("failed to post stats: %s", string(body)) 135 | } 136 | return nil 137 | } 138 | -------------------------------------------------------------------------------- /images/controller/config/deployment/app/kustomization.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | namespace: {{.App}} 18 | commonLabels: 19 | app.kubernetes.io/name: {{.App}} 20 | app.kubernetes.io/managed-by: pod-broker 21 | commonAnnotations: 22 | app.broker/app: {{.App}} 23 | resources: 24 | {{- range .Resources }} 25 | - {{ . }} 26 | {{- end }} 27 | {{- if .Patches }} 28 | patches: 29 | {{- range .Patches }} 30 | - {{ . }} 31 | {{- end }} 32 | {{- end }} 33 | {{- if or .JSONPatchesNamespace .JSONPatchesServiceAccount .JSONPatchesService .JSONPatchesDeploy }} 34 | patchesJson6902: 35 | {{- range .JSONPatchesNamespace }} 36 | - target: 37 | version: v1 38 | kind: Namespace 39 | name: {{$.Namespace}} 40 | path: {{ . }} 41 | {{- end }} 42 | {{- range .JSONPatchesServiceAccount }} 43 | - target: 44 | version: v1 45 | kind: ServiceAccount 46 | name: {{$.Namespace}} 47 | path: {{ . }} 48 | {{- end }} 49 | {{- range .JSONPatchesNetworkPolicy }} 50 | - target: 51 | version: networking.k8s.io/v1 52 | kind: NetworkPolicy 53 | name: {{$.App}} 54 | path: {{ . }} 55 | {{- end }} 56 | {{- range .JSONPatchesService }} 57 | - target: 58 | version: v1 59 | kind: Service 60 | name: {{$.App}} 61 | path: {{ . }} 62 | {{- end }} 63 | {{- range .JSONPatchesDeploy }} 64 | - target: 65 | group: apps 66 | version: v1 67 | kind: Deployment 68 | name: {{$.App}} 69 | path: {{ . }} 70 | {{- end }} 71 | {{- end }} 72 | {{- if .AppSpec.Images }} 73 | images: 74 | {{- range .AppSpec.Images }} 75 | - name: {{ .OldRepo }} 76 | newName: {{ .NewRepo }} 77 | newTag: {{ .NewTag | quote }} 78 | {{- if .Digest }} 79 | digest: {{ .Digest | quote }} 80 | {{- end }} 81 | {{- end }} 82 | {{- end}} -------------------------------------------------------------------------------- /images/controller/config/deployment/app/resource-namespace.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Namespace 17 | metadata: 18 | name: {{.Namespace}} 19 | annotations: 20 | cnrm.cloud.google.com/project-id: {{.ProjectID}} 21 | labels: {} 22 | -------------------------------------------------------------------------------- /images/controller/config/deployment/app/resource-network-policy.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.k8s.io/v1 16 | kind: NetworkPolicy 17 | metadata: 18 | name: deny-all 19 | spec: 20 | podSelector: {} 21 | policyTypes: 22 | - Ingress 23 | ingress: 24 | - from: 25 | - namespaceSelector: 26 | matchLabels: 27 | install.operator.istio.io/owner-kind: IstioControlPlane 28 | -------------------------------------------------------------------------------- /images/controller/config/deployment/app/resource-service-account.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ServiceAccount 17 | metadata: 18 | name: {{.Namespace}} 19 | annotations: 20 | iam.gke.io/gcp-service-account: broker-user@{{.ProjectID}}.iam.gserviceaccount.com 21 | --- 22 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 23 | kind: IAMPolicyMember 24 | metadata: 25 | name: {{.Namespace}}-wi 26 | annotations: 27 | cnrm.cloud.google.com/deletion-policy: "abandon" 28 | spec: 29 | resourceRef: 30 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 31 | kind: IAMServiceAccount 32 | external: projects/{{.ProjectID}}/serviceAccounts/broker-user@{{.ProjectID}}.iam.gserviceaccount.com 33 | member: serviceAccount:{{.ProjectID}}.svc.id.goog[{{.Namespace}}/{{.Namespace}}] 34 | role: roles/iam.workloadIdentityUser -------------------------------------------------------------------------------- /images/controller/config/deployment/user/kustomization.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | namespace: {{.Namespace}} 18 | namePrefix: {{.FullName}}- 19 | commonLabels: 20 | app.kubernetes.io/name: {{.App}} 21 | app.kubernetes.io/instance: {{.FullName}} 22 | app.kubernetes.io/managed-by: pod-broker 23 | commonAnnotations: 24 | app.broker/user: {{.User}} 25 | resources: 26 | {{- range .Resources }} 27 | - {{ . }} 28 | {{- end }} 29 | {{- if .Patches }} 30 | patches: 31 | {{- range .Patches }} 32 | - {{ . }} 33 | {{- end }} 34 | {{- end }} 35 | 36 | {{- if or .JSONPatchesService .JSONPatchesVirtualService }} 37 | patchesJson6902: 38 | {{- range .JSONPatchesService }} 39 | - target: 40 | version: v1 41 | kind: Service 42 | name: {{$.ServiceName}} 43 | path: {{ . }} 44 | {{- end }} 45 | {{- range .JSONPatchesVirtualService }} 46 | - target: 47 | group: networking.istio.io 48 | version: v1beta1 49 | kind: VirtualService 50 | name: {{$.ServiceName}} 51 | path: {{ . }} 52 | {{- end }} 53 | {{- end }} 54 | -------------------------------------------------------------------------------- /images/controller/config/statefulset/app/jsonpatch-deploy-last-applied-resources.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ### 16 | # This is a placeholder file that will be overwritten at runtime. 17 | # This patch is used to track the last applied object types so they can be properly deleted. 18 | ### 19 | - op: test 20 | path: /kind 21 | value: StatefulSet -------------------------------------------------------------------------------- /images/controller/config/statefulset/app/kustomization.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | namespace: {{.Namespace}} 18 | namePrefix: {{.FullName}}- 19 | commonLabels: 20 | app.kubernetes.io/name: {{.App}} 21 | app.kubernetes.io/instance: {{.FullName}} 22 | app.kubernetes.io/managed-by: pod-broker 23 | commonAnnotations: 24 | app.broker/user: {{.User}} 25 | resources: 26 | {{- range .Resources }} 27 | - {{ . }} 28 | {{- end }} 29 | {{- if .Patches }} 30 | patches: 31 | {{- range .Patches }} 32 | - {{ . }} 33 | {{- end }} 34 | {{- end }} 35 | 36 | {{- if or .JSONPatchesService .JSONPatchesVirtualService .JSONPatchesDeploy }} 37 | patchesJson6902: 38 | {{- range .JSONPatchesService }} 39 | - target: 40 | version: v1 41 | kind: Service 42 | name: {{$.ServiceName}} 43 | path: {{ . }} 44 | {{- end }} 45 | {{- range .JSONPatchesVirtualService }} 46 | - target: 47 | group: networking.istio.io 48 | version: v1beta1 49 | kind: VirtualService 50 | name: {{$.ServiceName}} 51 | path: {{ . }} 52 | {{- end }} 53 | {{- range .JSONPatchesDeploy }} 54 | - target: 55 | group: apps 56 | version: v1 57 | kind: StatefulSet 58 | name: {{$.ServiceName}} 59 | path: {{ . }} 60 | {{- end }} 61 | {{- end }} 62 | 63 | {{- if .AppSpec.Images }} 64 | images: 65 | {{- range .AppSpec.Images }} 66 | - name: {{ .OldRepo }} 67 | newName: {{ .NewRepo }} 68 | newTag: {{ .NewTag | quote }} 69 | {{- if .Digest }} 70 | digest: {{ .Digest | quote }} 71 | {{- end }} 72 | {{- end }} 73 | {{- end}} 74 | -------------------------------------------------------------------------------- /images/controller/config/statefulset/user/kustomization.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | namespace: {{.Namespace}} 18 | commonLabels: 19 | app.kubernetes.io/managed-by: pod-broker 20 | commonAnnotations: 21 | app.broker/user: {{.User}} 22 | resources: 23 | {{- range .Resources }} 24 | - {{ . }} 25 | {{- end }} 26 | {{- if .Patches }} 27 | patches: 28 | {{- range .Patches }} 29 | - {{ . }} 30 | {{- end }} 31 | {{- end }} 32 | {{- if or .JSONPatchesNamespace .JSONPatchesServiceAccount }} 33 | patchesJson6902: 34 | {{- range .JSONPatchesNamespace }} 35 | - target: 36 | version: v1 37 | kind: Namespace 38 | name: {{$.Namespace}} 39 | path: {{ . }} 40 | {{- end }} 41 | {{- range .JSONPatchesServiceAccount }} 42 | - target: 43 | version: v1 44 | kind: ServiceAccount 45 | name: {{$.Namespace}} 46 | path: {{ . }} 47 | {{- end }} 48 | {{- end }} 49 | -------------------------------------------------------------------------------- /images/controller/config/statefulset/user/resource-namespace.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Namespace 17 | metadata: 18 | name: {{.Namespace}} 19 | annotations: 20 | cnrm.cloud.google.com/project-id: {{.ProjectID}} 21 | labels: {} 22 | -------------------------------------------------------------------------------- /images/controller/config/statefulset/user/resource-network-policy.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.k8s.io/v1 16 | kind: NetworkPolicy 17 | metadata: 18 | name: deny-all 19 | spec: 20 | podSelector: {} 21 | policyTypes: 22 | - Ingress 23 | ingress: 24 | - from: 25 | - namespaceSelector: 26 | matchLabels: 27 | install.operator.istio.io/owner-kind: IstioControlPlane 28 | -------------------------------------------------------------------------------- /images/controller/config/statefulset/user/resource-service-account.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ServiceAccount 17 | metadata: 18 | name: {{.Namespace}} 19 | annotations: 20 | iam.gke.io/gcp-service-account: broker-user@{{.ProjectID}}.iam.gserviceaccount.com 21 | --- 22 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 23 | kind: IAMPolicyMember 24 | metadata: 25 | name: {{.Namespace}}-wi 26 | annotations: 27 | cnrm.cloud.google.com/deletion-policy: "abandon" 28 | spec: 29 | resourceRef: 30 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 31 | kind: IAMServiceAccount 32 | external: projects/{{.ProjectID}}/serviceAccounts/broker-user@{{.ProjectID}}.iam.gserviceaccount.com 33 | member: serviceAccount:{{.ProjectID}}.svc.id.goog[{{.Namespace}}/{{.Namespace}}] 34 | role: roles/iam.workloadIdentityUser -------------------------------------------------------------------------------- /images/controller/go.mod: -------------------------------------------------------------------------------- 1 | module selkies.io/controller 2 | 3 | go 1.15 4 | 5 | require ( 6 | cloud.google.com/go v0.83.0 7 | cloud.google.com/go/pubsub v1.3.1 8 | github.com/Masterminds/goutils v1.1.0 // indirect 9 | github.com/Masterminds/semver v1.5.0 // indirect 10 | github.com/Masterminds/sprig v2.22.0+incompatible 11 | github.com/google/go-containerregistry v0.6.0 12 | github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f 13 | github.com/gorilla/mux v1.8.0 14 | github.com/huandu/xstrings v1.3.2 // indirect 15 | github.com/mitchellh/copystructure v1.0.0 // indirect 16 | github.com/mitchellh/reflectwalk v1.0.1 // indirect 17 | golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c 18 | google.golang.org/api v0.47.0 19 | gopkg.in/yaml.v2 v2.4.0 20 | k8s.io/api v0.20.6 21 | k8s.io/apimachinery v0.20.6 22 | k8s.io/client-go v0.20.6 23 | ) 24 | -------------------------------------------------------------------------------- /images/controller/pkg/appconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "os/exec" 23 | ) 24 | 25 | func (spec *AppConfigSpec) NodeTierNames() []string { 26 | tierNames := []string{} 27 | for _, tier := range spec.NodeTiers { 28 | tierNames = append(tierNames, tier.Name) 29 | } 30 | return tierNames 31 | } 32 | 33 | func FetchBrokerAppConfigs(namespace string) ([]AppConfigObject, error) { 34 | appConfigs := make([]AppConfigObject, 0) 35 | 36 | type appConfigItems struct { 37 | Items []AppConfigObject `yaml:"items"` 38 | } 39 | 40 | // Fetch all broker app configs 41 | cmd := exec.Command("sh", "-c", fmt.Sprintf("kubectl get brokerappconfigs -n %s -o json", namespace)) 42 | output, err := cmd.Output() 43 | if err != nil { 44 | return appConfigs, err 45 | } 46 | 47 | var items appConfigItems 48 | err = json.Unmarshal(output, &items) 49 | 50 | // Set default values 51 | for i := range items.Items { 52 | // Default app type to StatefulSet 53 | if items.Items[i].Spec.Type == "" { 54 | items.Items[i].Spec.Type = AppTypeStatefulSet 55 | } 56 | 57 | // Default userBundles to empty list if not provided. 58 | if items.Items[i].Spec.UserBundles == nil { 59 | items.Items[i].Spec.UserBundles = make([]UserBundleSpec, 0) 60 | } 61 | 62 | if items.Items[i].Spec.Type == AppTypeDeployment { 63 | // Default deployment selector to match app name. 64 | if len(items.Items[i].Spec.Deployment.Selector) == 0 { 65 | items.Items[i].Spec.Deployment.Selector = fmt.Sprintf("app=%s", items.Items[i].Spec.Name) 66 | } 67 | 68 | if items.Items[i].Spec.Deployment.Replicas == nil { 69 | // Default number of deployment replicas. 70 | // This value is a pointer so that it can accept 0 as a valid value. 71 | defaultReplicas := DefaultDeploymentReplicas 72 | items.Items[i].Spec.Deployment.Replicas = &defaultReplicas 73 | } 74 | } 75 | } 76 | 77 | return items.Items, err 78 | } 79 | -------------------------------------------------------------------------------- /images/controller/pkg/appconfig_informer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Selkies Authors. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | ) 23 | 24 | type AppConfigInformer struct { 25 | AddFunc func(appConfig AppConfigObject) 26 | DeleteFunc func(appConfig AppConfigObject) 27 | UpdateFunc func(oldObj, newObj AppConfigObject) 28 | ApiVersion string 29 | Kind string 30 | ApiGroup string 31 | PodBrokerInformer 32 | } 33 | 34 | func NewAppConfigInformer(addFunc func(appConfig AppConfigObject), deleteFunc func(appConfig AppConfigObject), updateFunc func(oldObj, newObj AppConfigObject)) AppConfigInformer { 35 | resp := AppConfigInformer{ 36 | ApiVersion: ApiVersion, 37 | Kind: BrokerAppConfigKind, 38 | ApiGroup: BrokerAppConfigApiGroup, 39 | AddFunc: addFunc, 40 | DeleteFunc: deleteFunc, 41 | UpdateFunc: updateFunc, 42 | } 43 | return resp 44 | } 45 | 46 | // Interface function 47 | func (pbi AppConfigInformer) makeObjectFromUnstructured(obj interface{}) (interface{}, error) { 48 | d := AppConfigObject{} 49 | err := runtime.DefaultUnstructuredConverter. 50 | FromUnstructured(obj.(*unstructured.Unstructured).UnstructuredContent(), &d) 51 | if err != nil { 52 | return d, err 53 | } 54 | d.ApiVersion = pbi.ApiVersion 55 | d.Kind = pbi.Kind 56 | return d, nil 57 | } 58 | 59 | // Interface function 60 | func (pbi AppConfigInformer) apiGroup() string { 61 | return pbi.ApiGroup 62 | } 63 | 64 | // Interface function 65 | func (pbi AppConfigInformer) kind() string { 66 | return pbi.Kind 67 | } 68 | 69 | // Interface function 70 | func (pbi AppConfigInformer) addFunc(obj interface{}) { 71 | pbi.AddFunc(obj.(AppConfigObject)) 72 | } 73 | 74 | // Interface function 75 | func (pbi AppConfigInformer) deleteFunc(obj interface{}) { 76 | pbi.DeleteFunc(obj.(AppConfigObject)) 77 | } 78 | 79 | // Interface function 80 | func (pbi AppConfigInformer) updateFunc(oldObj, newObj interface{}) { 81 | pbi.UpdateFunc(oldObj.(AppConfigObject), newObj.(AppConfigObject)) 82 | } 83 | -------------------------------------------------------------------------------- /images/controller/pkg/appconfigbundle.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import ( 20 | "fmt" 21 | "io/ioutil" 22 | "os" 23 | "os/exec" 24 | "path" 25 | 26 | "gopkg.in/yaml.v2" 27 | ) 28 | 29 | func (cm *ConfigMapObject) SaveDataToDirectory(destDir string) error { 30 | if err := os.MkdirAll(destDir, os.ModePerm); err != nil { 31 | return err 32 | } 33 | 34 | for fileName, data := range cm.Data { 35 | destFile := path.Join(destDir, fileName) 36 | if err := ioutil.WriteFile(destFile, []byte(data), 0644); err != nil { 37 | return err 38 | } 39 | } 40 | return nil 41 | } 42 | 43 | func GetConfigMaps(namespace string) ([]ConfigMapObject, error) { 44 | objs := make([]ConfigMapObject, 0) 45 | 46 | type configMapItems struct { 47 | Items []ConfigMapObject `yaml:"items,omitempty"` 48 | } 49 | 50 | cmd := exec.Command("sh", "-c", fmt.Sprintf("kubectl get configmaps -n %s -o yaml", namespace)) 51 | output, err := cmd.Output() 52 | if err != nil { 53 | return objs, err 54 | } 55 | 56 | var items configMapItems 57 | err = yaml.Unmarshal(output, &items) 58 | if err != nil { 59 | return objs, err 60 | } 61 | if items.Items == nil { 62 | // Possible single-value query. 63 | var obj ConfigMapObject 64 | err = yaml.Unmarshal(output, &obj) 65 | if err != nil { 66 | return objs, err 67 | } 68 | objs = append(objs, obj) 69 | } else { 70 | for _, item := range items.Items { 71 | objs = append(objs, item) 72 | } 73 | } 74 | return objs, err 75 | } 76 | -------------------------------------------------------------------------------- /images/controller/pkg/appmanifest.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import ( 20 | "encoding/json" 21 | "io/ioutil" 22 | ) 23 | 24 | func (manifest *RegisteredAppsManifest) Add(app AppConfigSpec) { 25 | manifest.Apps[app.Name] = app 26 | } 27 | 28 | func (manifest *RegisteredAppsManifest) WriteJSON(destFile string) error { 29 | data, err := json.MarshalIndent(manifest, "", " ") 30 | if err != nil { 31 | return err 32 | } 33 | 34 | return ioutil.WriteFile(destFile, data, 0644) 35 | } 36 | 37 | func NewRegisteredAppManifest() RegisteredAppsManifest { 38 | return RegisteredAppsManifest{ 39 | Apps: make(map[string]AppConfigSpec, 0), 40 | } 41 | } 42 | 43 | func NewRegisteredAppManifestFromJSON(srcFile string, appType AppType) (RegisteredAppsManifest, error) { 44 | var manifest RegisteredAppsManifest 45 | 46 | data, err := ioutil.ReadFile(srcFile) 47 | if err != nil { 48 | return manifest, err 49 | } 50 | 51 | err = json.Unmarshal(data, &manifest) 52 | 53 | apps := make(map[string]AppConfigSpec, 0) 54 | for k, v := range manifest.Apps { 55 | if v.Type == appType || appType == AppTypeAll { 56 | apps[k] = v 57 | } 58 | } 59 | manifest.Apps = apps 60 | 61 | return manifest, err 62 | } 63 | -------------------------------------------------------------------------------- /images/controller/pkg/appuserconfig.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Google Inc. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io/ioutil" 23 | "os" 24 | "os/exec" 25 | "path/filepath" 26 | 27 | "gopkg.in/yaml.v2" 28 | ) 29 | 30 | func (userConfig *AppUserConfigObject) WriteJSON(destFile string) error { 31 | if err := os.MkdirAll(filepath.Dir(destFile), os.ModePerm); err != nil { 32 | return err 33 | } 34 | 35 | data, err := json.MarshalIndent(userConfig, "", " ") 36 | if err != nil { 37 | return err 38 | } 39 | 40 | return ioutil.WriteFile(destFile, data, 0644) 41 | } 42 | 43 | func NewAppUserConfig(name, namespace string, spec AppUserConfigSpec) AppUserConfigObject { 44 | return AppUserConfigObject{ 45 | KubeObjectBase: KubeObjectBase{ 46 | ApiVersion: ApiVersion, 47 | Kind: BrokerAppUserConfigKind, 48 | }, 49 | Metadata: KubeObjectMeta{ 50 | Name: name, 51 | Namespace: namespace, 52 | Labels: map[string]string{ 53 | "app.kubernetes.io/name": spec.AppName, 54 | "app.kubernetes.io/instance": name, 55 | "app.kubernetes.io/managed-by": "pod-broker", 56 | }, 57 | Annotations: map[string]string{}, 58 | }, 59 | Spec: spec, 60 | } 61 | } 62 | 63 | func NewAppUserConfigFromJSON(srcFile string) (AppUserConfigObject, error) { 64 | userConfig := AppUserConfigObject{} 65 | 66 | data, err := ioutil.ReadFile(srcFile) 67 | if err != nil { 68 | return userConfig, err 69 | } 70 | 71 | err = yaml.Unmarshal([]byte(data), &userConfig) 72 | return userConfig, err 73 | } 74 | 75 | func FetchAppUserConfigs() ([]AppUserConfigObject, error) { 76 | userConfigs := make([]AppUserConfigObject, 0) 77 | 78 | type appUserConfigItems struct { 79 | Items []AppUserConfigObject `json:"items"` 80 | } 81 | 82 | // Fetch all app user config objects 83 | // kubectl get brokerappuserconfigs -l app.kubernetes.io/managed-by=pod-broker 84 | cmd := exec.Command("sh", "-c", "kubectl get brokerappuserconfigs --all-namespaces -l app.kubernetes.io/managed-by=pod-broker -o json") 85 | output, err := cmd.Output() 86 | if err != nil { 87 | return userConfigs, err 88 | } 89 | 90 | var items appUserConfigItems 91 | if err := json.Unmarshal(output, &items); err != nil { 92 | return userConfigs, err 93 | } 94 | userConfigs = items.Items 95 | 96 | return userConfigs, nil 97 | } 98 | 99 | func GetAppUserConfig(srcFile string) (AppUserConfigObject, error) { 100 | userConfig := AppUserConfigObject{} 101 | 102 | if _, err := os.Stat(srcFile); os.IsNotExist(err) { 103 | return userConfig, fmt.Errorf("app user config not found: %s", srcFile) 104 | } 105 | 106 | data, err := ioutil.ReadFile(srcFile) 107 | if err != nil { 108 | return userConfig, err 109 | } 110 | 111 | err = json.Unmarshal([]byte(data), &userConfig) 112 | if err != nil { 113 | return userConfig, nil 114 | } 115 | 116 | return userConfig, nil 117 | } 118 | -------------------------------------------------------------------------------- /images/controller/pkg/appuserconfig_informer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Selkies Authors. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | ) 23 | 24 | type AppUserConfigInformer struct { 25 | AddFunc func(appConfig AppUserConfigObject) 26 | DeleteFunc func(appConfig AppUserConfigObject) 27 | UpdateFunc func(oldObj, newObj AppUserConfigObject) 28 | ApiVersion string 29 | Kind string 30 | ApiGroup string 31 | } 32 | 33 | func NewAppUserConfigInformer(addFunc func(appConfig AppUserConfigObject), deleteFunc func(appConfig AppUserConfigObject), updateFunc func(oldObj, newObj AppUserConfigObject)) AppUserConfigInformer { 34 | resp := AppUserConfigInformer{ 35 | ApiVersion: ApiVersion, 36 | Kind: BrokerAppUserConfigKind, 37 | ApiGroup: BrokerAppUserConfigApiGroup, 38 | AddFunc: addFunc, 39 | DeleteFunc: deleteFunc, 40 | UpdateFunc: updateFunc, 41 | } 42 | return resp 43 | } 44 | 45 | // Interface function 46 | func (pbi AppUserConfigInformer) makeObjectFromUnstructured(obj interface{}) (interface{}, error) { 47 | d := AppUserConfigObject{} 48 | err := runtime.DefaultUnstructuredConverter. 49 | FromUnstructured(obj.(*unstructured.Unstructured).UnstructuredContent(), &d) 50 | if err != nil { 51 | return d, err 52 | } 53 | d.ApiVersion = pbi.ApiVersion 54 | d.Kind = pbi.Kind 55 | return d, nil 56 | } 57 | 58 | // Interface function 59 | func (pbi AppUserConfigInformer) apiGroup() string { 60 | return pbi.ApiGroup 61 | } 62 | 63 | // Interface function 64 | func (pbi AppUserConfigInformer) kind() string { 65 | return pbi.Kind 66 | } 67 | 68 | // Interface function 69 | func (pbi AppUserConfigInformer) addFunc(obj interface{}) { 70 | pbi.AddFunc(obj.(AppUserConfigObject)) 71 | } 72 | 73 | // Interface function 74 | func (pbi AppUserConfigInformer) deleteFunc(obj interface{}) { 75 | pbi.DeleteFunc(obj.(AppUserConfigObject)) 76 | } 77 | 78 | // Interface function 79 | func (pbi AppUserConfigInformer) updateFunc(oldObj, newObj interface{}) { 80 | pbi.UpdateFunc(oldObj.(AppUserConfigObject), newObj.(AppUserConfigObject)) 81 | } 82 | -------------------------------------------------------------------------------- /images/controller/pkg/imagequeuesync.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 The Selkies Authors. All rights reserved. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package pod_broker 18 | 19 | import "sync" 20 | 21 | // Synchronous slice of de-duplicated images to process 22 | type ImageQueueSync struct { 23 | sync.Mutex 24 | ImageQueue []string 25 | } 26 | 27 | func NewImageQueueSync() *ImageQueueSync { 28 | iq := &ImageQueueSync{} 29 | iq.ImageQueue = make([]string, 0) 30 | return iq 31 | } 32 | 33 | func (iq *ImageQueueSync) Len() int { 34 | iq.Lock() 35 | defer iq.Unlock() 36 | return len(iq.ImageQueue) 37 | } 38 | 39 | func (iq *ImageQueueSync) Push(newImage string) { 40 | iq.Lock() 41 | defer iq.Unlock() 42 | for _, currImage := range iq.ImageQueue { 43 | if currImage == newImage { 44 | return 45 | } 46 | } 47 | iq.ImageQueue = append(iq.ImageQueue, newImage) 48 | } 49 | 50 | func (iq *ImageQueueSync) Pop() string { 51 | iq.Lock() 52 | defer iq.Unlock() 53 | if len(iq.ImageQueue) == 0 { 54 | return "" 55 | } 56 | v := iq.ImageQueue[0] 57 | iq.ImageQueue = iq.ImageQueue[1:] 58 | return v 59 | } 60 | 61 | func (iq *ImageQueueSync) Remove(image string) bool { 62 | iq.Lock() 63 | defer iq.Unlock() 64 | newQueue := make([]string, 0) 65 | found := false 66 | for _, currImage := range iq.ImageQueue { 67 | if currImage != image { 68 | newQueue = append(newQueue, currImage) 69 | } else { 70 | found = true 71 | } 72 | } 73 | return found 74 | } 75 | 76 | func (iq *ImageQueueSync) Contains(image string) bool { 77 | iq.Lock() 78 | defer iq.Unlock() 79 | for _, currImage := range iq.ImageQueue { 80 | if currImage == image { 81 | return true 82 | } 83 | } 84 | return false 85 | } 86 | -------------------------------------------------------------------------------- /images/gce-proxy/.dockerignore: -------------------------------------------------------------------------------- 1 | src/node_modules/ -------------------------------------------------------------------------------- /images/gce-proxy/.gitignore: -------------------------------------------------------------------------------- 1 | src/node_modules/ -------------------------------------------------------------------------------- /images/gce-proxy/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM node:10-slim 16 | 17 | WORKDIR /opt/app 18 | 19 | COPY src/* ./ 20 | 21 | RUN npm install 22 | 23 | ENTRYPOINT npm start -------------------------------------------------------------------------------- /images/gce-proxy/src/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gst-webrtc-app-proxy", 3 | "version": "1.0.0", 4 | "description": "Lightweight proxy for gst-webrtc-app", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js" 8 | }, 9 | "author": "disla@google.com", 10 | "license": "Apache-2.0", 11 | "dependencies": { 12 | "http-proxy": "^1.1.0" 13 | }, 14 | "devDependencies": { 15 | "http-proxy-cli": "^1.1.0" 16 | } 17 | } -------------------------------------------------------------------------------- /images/installer/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM gcr.io/cloud-builders/gke-deploy as gke-deploy 16 | FROM gcr.io/cloud-builders/kubectl as kubectl 17 | 18 | FROM gcr.io/google.com/cloudsdktool/cloud-sdk:alpine 19 | RUN apk add -u \ 20 | jq \ 21 | coreutils 22 | 23 | ARG TERRAFORM_VERSION=1.2.3 24 | ARG KUSTOMIZE_VERSION=3.5.3 25 | # ARG ISTIO_OPERATOR_VERSION=1.4.7 26 | 27 | # istioctl and istio bundle to be installed if 1.14 not found. 28 | ENV LATEST_ISTIO=1.14.3 29 | ENV LATEST_ISTIO_MAJOR=1.14 30 | # istoctl capable of detecting version 1.4+ 31 | ENV ISTIO_COMPAT=1.14.3 32 | 33 | 34 | # Install terraform 35 | RUN wget --timeout 2 --tries 5 https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ 36 | unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ 37 | chmod +x terraform && \ 38 | mv terraform /usr/local/bin && \ 39 | rm -rf terraform_${TERRAFORM_VERSION}_linux_amd64.zip 40 | 41 | # Install kustomize 42 | RUN wget --timeout 2 --tries 5 -O- https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz | \ 43 | tar zxvf - kustomize && \ 44 | chmod +x kustomize && \ 45 | mv kustomize /usr/local/bin 46 | 47 | # Install gke-deploy 48 | COPY --from=gke-deploy /gke-deploy /usr/local/bin/ 49 | 50 | # Install kubectl 51 | COPY --from=kubectl /builder/google-cloud-sdk/bin/kubectl /usr/local/bin/kubectl 52 | # Install gke-gcloud-auth-plugin 53 | RUN gcloud components install gke-gcloud-auth-plugin --quiet 54 | 55 | # # Download the istio operator release 56 | # RUN mkdir -p /opt/istio-operator && cd /opt/istio-operator && \ 57 | # wget --timeout 2 --tries 5 -qO- https://github.com/istio/operator/archive/${ISTIO_OPERATOR_VERSION}.tar.gz | tar --strip-components=1 -zxf - 58 | 59 | # Download compat istio release 60 | RUN mkdir -p /opt/istio-${ISTIO_COMPAT}/bin && cd /opt/istio-${ISTIO_COMPAT}/bin && \ 61 | curl -L https://github.com/istio/istio/releases/download/${ISTIO_COMPAT}/istioctl-${ISTIO_COMPAT}-linux-amd64.tar.gz | tar zxvf - istioctl > istioctl && chmod +x istioctl 62 | 63 | # Download latest istio releases 64 | RUN cd /opt && curl -L https://istio.io/downloadIstio | ISTIO_VERSION=${LATEST_ISTIO} TARGET_ARCH=x86_64 sh - -------------------------------------------------------------------------------- /setup/.gitignore: -------------------------------------------------------------------------------- 1 | terraform.tfvars* 2 | .terraform 3 | *.tfstate* 4 | backend.tf 5 | **/generated 6 | .env 7 | .git -------------------------------------------------------------------------------- /setup/example_tfvars/README.md: -------------------------------------------------------------------------------- 1 | # Selkies infrastructure example tfvars 2 | 3 | Usage: 4 | 5 | ``` 6 | gcloud secrets create broker-tfvars-selkies-cluster-min \ 7 | --replication-policy=automatic \ 8 | --data-file selkies-cluster-min.auto.tfvars 9 | ``` -------------------------------------------------------------------------------- /setup/example_tfvars/selkies-cluster-min.auto.tfvars: -------------------------------------------------------------------------------- 1 | default_pool_machine_type = "e2-highcpu-2" 2 | default_pool_initial_node_count = 3 3 | default_pool_preemptive_nodes = true 4 | default_pool_disk_size_gb = 50 5 | default_pool_disk_type = "pd-standard" 6 | 7 | tier1_pool_machine_type = "e2-standard-4" 8 | tier1_pool_preemptive_nodes = true 9 | tier1_pool_disk_size_gb = 50 10 | tier1_pool_disk_type = "pd-standard" 11 | 12 | tier1_ubuntu_pool_enabled = false 13 | tier2_pool_enabled = false -------------------------------------------------------------------------------- /setup/infra/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform -------------------------------------------------------------------------------- /setup/infra/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | tags: 20 | - selkies-infra 21 | - selkies-setup 22 | steps: 23 | ### 24 | # Create GCS bucket to store terraform state 25 | ### 26 | - name: "gcr.io/cloud-builders/gsutil" 27 | id: "create-tf-state-bucket" 28 | entrypoint: "/bin/bash" 29 | args: 30 | - "-xe" 31 | - "-c" 32 | - | 33 | gsutil mb gs://${PROJECT_ID}-${_NAME}-tf-state || true 34 | waitFor: ["-"] 35 | 36 | ### 37 | # Apply terraform to create base infrastructure 38 | ### 39 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 40 | id: "terraform-apply" 41 | entrypoint: "/workspace/deploy.sh" 42 | env: 43 | - TF_VAR_project_id=${PROJECT_ID} 44 | - TF_VAR_name=${_NAME} 45 | - TERRAFORM_WORKSPACE_NAME=default 46 | - ACTION=${_ACTION} 47 | -------------------------------------------------------------------------------- /setup/infra/cluster/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | *.tfstate -------------------------------------------------------------------------------- /setup/infra/cluster/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | _REGION: us-central1 20 | tags: 21 | - selkies-cluster 22 | - selkies-setup 23 | steps: 24 | ### 25 | # Apply terraform to create cluster 26 | ### 27 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 28 | id: "terraform-apply" 29 | entrypoint: "/workspace/deploy.sh" 30 | env: 31 | - TF_VAR_project_id=${PROJECT_ID} 32 | - TF_VAR_name=${_NAME} 33 | - TF_VAR_region=${_REGION} 34 | - TERRAFORM_WORKSPACE_NAME=${_REGION} 35 | - ACTION=${_ACTION} 36 | -------------------------------------------------------------------------------- /setup/infra/cluster/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | export RED='\033[1;31m' 20 | export CYAN='\033[1;36m' 21 | export GREEN='\033[1;32m' 22 | export NC='\033[0m' # No Color 23 | function log_red() { echo -e "${RED}$@${NC}"; } 24 | function log_cyan() { echo -e "${CYAN}$@${NC}"; } 25 | function log_green() { echo -e "${GREEN}$@${NC}"; } 26 | 27 | SCRIPT_DIR=$(dirname $(readlink -f $0 2>/dev/null) 2>/dev/null || echo "${PWD}/$(dirname $0)") 28 | 29 | cd "${SCRIPT_DIR}" 30 | 31 | # Fetch any Secret Manager secrets named broker-tfvars* and same them to .auto.tfvars files. 32 | for secret in $(gcloud -q secrets list --filter=name~broker-tfvars- --format="value(name)"); do 33 | latest=$(gcloud secrets versions list ${secret} --sort-by=created --format='value(name)' --filter='STATE=enabled' --limit=1) 34 | [[ -z "${latest}" ]] && log_red "WARN: no enabled versions found for secret ${secret}" && continue 35 | dest="${secret/broker-tfvars-/}.auto.tfvars" 36 | log_cyan "Creating ${dest} from secret: ${secret}" 37 | gcloud -q secrets versions access ${latest} --secret ${secret} > ${dest} 38 | done 39 | 40 | export TF_IN_AUTOMATION=1 41 | 42 | # Set default project for google provider. 43 | export GOOGLE_PROJECT=${TF_VAR_project_id?} 44 | 45 | # Initialize backend and select workspace 46 | terraform init -upgrade=true -input=false \ 47 | -backend-config="bucket=${TF_VAR_project_id?}-${TF_VAR_name?}-tf-state" \ 48 | -backend-config="prefix=${TF_VAR_name?}" || true 49 | terraform workspace select ${TERRAFORM_WORKSPACE_NAME?} || terraform workspace new ${TERRAFORM_WORKSPACE_NAME?} 50 | terraform init -input=false \ 51 | -backend-config="bucket=${TF_VAR_project_id?}-${TF_VAR_name?}-tf-state" \ 52 | -backend-config="prefix=${TF_VAR_name?}" || true 53 | 54 | if [[ "${ACTION?}" == "destroy" ]]; then 55 | log_cyan "Running terraform destroy..." 56 | terraform destroy -auto-approve -input=false 57 | elif [[ "${ACTION?}" == "plan" ]]; then 58 | log_cyan "Running terraform plan..." 59 | terraform plan -out terraform.tfplan -input=false 60 | elif [[ "${ACTION?}" == "apply" ]]; then 61 | log_cyan "Running terraform plan..." 62 | terraform plan -out terraform.tfplan -input=false 63 | 64 | log_cyan "Running terraform apply..." 65 | terraform apply -input=false terraform.tfplan 66 | fi 67 | 68 | log_green "Done" -------------------------------------------------------------------------------- /setup/infra/cluster/locals.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Built from: 18 | // gcloud compute accelerator-types list |grep nvidia-tesla- | grep -v vws | sort -k2 -r 19 | locals { 20 | // NOTE: if you plan on using accelerators, choose regions that contain the desired accelerator type 21 | 22 | // Map used to create subnets, integer values are used for CIDR range offsets. 23 | cluster_regions = { 24 | "us-west1" = 0, # The Dalles, Oregon, USA 25 | "us-west2" = 1, # Los Angeles, California, USA 26 | "us-central1" = 2, # Council Bluffs, Iowa, USA 27 | "us-east1" = 3, # Moncks Corner, South Carolina, USA 28 | "us-east4" = 4, # Ashburn, Northern Virginia, USA 29 | "northamerica-northeast1" = 5, # Montréal, Québec, Canada 30 | "southamerica-east1" = 6, # Osasco (São Paulo), Brazil 31 | "europe-west1" = 7, # St. Ghislain, Belgium 32 | "europe-west2" = 8, # London, England, UK 33 | "europe-west3" = 16, # Frankfurt, Germany 34 | "europe-west4" = 9, # Eemshaven, Netherlands 35 | "asia-east1" = 10, # Changhua County, Taiwan 36 | "asia-northeast1" = 11, # Tokyo, Japan 37 | "asia-northeast3" = 12, # Seoul, South Korea 38 | "asia-south1" = 13, # Mumbai, India 39 | "asia-southeast1" = 14, # Jurong West, Singapore 40 | "australia-southeast1" = 15, # Sydney, Australia 41 | } 42 | 43 | // Map of regions to zones that have accelerators available. 44 | cluster_node_zones = { 45 | "us-west1" = ["us-west1-a"], 46 | "us-west2" = ["us-west2-b"], 47 | "us-central1" = ["us-central1-a"], 48 | "us-east1" = ["us-east1-c"], 49 | "us-east4" = ["us-east4-a"], 50 | "northamerica-northeast1" = ["northamerica-northeast1-a"], 51 | "southamerica-east1" = ["southamerica-east1-c"], 52 | "europe-west1" = ["europe-west1-b"], 53 | "europe-west2" = ["europe-west2-a"], 54 | "europe-west3" = ["europe-west3-b"], 55 | "europe-west4" = ["europe-west4-b"], 56 | "asia-east1" = ["asia-east1-a"], 57 | "asia-northeast1" = ["asia-northeast1-a"], 58 | "asia-northeast3" = ["asia-northeast3-b"], 59 | "asia-south1" = ["asia-south1-a"], 60 | "asia-southeast1" = ["asia-southeast1-b"], 61 | "australia-southeast1" = ["australia-southeast1-a"], 62 | } 63 | } -------------------------------------------------------------------------------- /setup/infra/cluster/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "broker" { 18 | source = "github.com/terraform-google-modules/terraform-google-kubernetes-engine//modules/beta-public-cluster?ref=v9.0.0" 19 | project_id = var.project_id 20 | release_channel = var.release_channel 21 | name = "${var.name}-${var.region}" 22 | regional = true 23 | region = var.region 24 | network = data.google_compute_network.broker.name 25 | subnetwork = google_compute_subnetwork.broker.name 26 | ip_range_pods = google_compute_subnetwork.broker.secondary_ip_range[0].range_name 27 | ip_range_services = google_compute_subnetwork.broker.secondary_ip_range[1].range_name 28 | node_metadata = "GKE_METADATA_SERVER" 29 | create_service_account = false 30 | service_account = length(var.service_account) == 0 ? "broker@${var.project_id}.iam.gserviceaccount.com" : var.service_account 31 | remove_default_node_pool = true 32 | network_policy = var.network_policy 33 | gce_pd_csi_driver = true 34 | default_max_pods_per_node = var.max_pods_per_node 35 | config_connector = true 36 | 37 | # Zones must be compatible with the chosen accelerator_type in the gpu-* node pools. 38 | zones = length(var.zones) == 0 ? lookup(local.cluster_node_zones, var.region) : var.zones 39 | 40 | node_pools = [ 41 | { 42 | # Default node pool 43 | name = "default-node-pool" 44 | machine_type = var.default_pool_machine_type 45 | initial_node_count = var.default_pool_initial_node_count 46 | min_count = var.default_pool_min_node_count 47 | max_count = var.default_pool_max_node_count 48 | local_ssd_count = 0 49 | disk_size_gb = var.default_pool_disk_size_gb 50 | disk_type = var.default_pool_disk_type 51 | image_type = "COS" 52 | auto_repair = true 53 | auto_upgrade = true 54 | service_account = length(var.service_account) == 0 ? "broker@${var.project_id}.iam.gserviceaccount.com" : var.service_account 55 | preemptible = var.default_pool_preemptive_nodes 56 | } 57 | ] 58 | 59 | node_pools_oauth_scopes = { 60 | all = [] 61 | default-node-pool = [ 62 | "https://www.googleapis.com/auth/cloud-platform", 63 | ] 64 | } 65 | 66 | node_pools_labels = { 67 | all = {} 68 | default-node-pool = {} 69 | } 70 | 71 | node_pools_metadata = { 72 | all = {} 73 | default-node-pool = {} 74 | } 75 | 76 | node_pools_taints = { 77 | all = [] 78 | default-node-pool = [] 79 | } 80 | 81 | node_pools_tags = { 82 | all = [] 83 | default-node-pool = [] 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /setup/infra/cluster/network.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | data "google_compute_network" "broker" { 18 | name = var.name 19 | } 20 | 21 | resource "google_compute_subnetwork" "broker" { 22 | name = "${var.name}-${var.region}" 23 | ip_cidr_range = var.ip_cidr_range.nodes != "" ? var.ip_cidr_range.nodes : "10.${2 + lookup(local.cluster_regions, var.region)}.0.0/16" 24 | region = var.region 25 | network = data.google_compute_network.broker.self_link 26 | private_ip_google_access = true 27 | 28 | secondary_ip_range = [ 29 | { 30 | range_name = "${var.region}-pods" 31 | ip_cidr_range = var.ip_cidr_range.pods != "" ? var.ip_cidr_range.pods : "172.${16 + lookup(local.cluster_regions, var.region)}.0.0/16" 32 | }, 33 | { 34 | range_name = "${var.region}-services" 35 | ip_cidr_range = var.ip_cidr_range.services != "" ? var.ip_cidr_range.services : "192.168.${lookup(local.cluster_regions, var.region)}.0/24" 36 | }, 37 | ] 38 | } -------------------------------------------------------------------------------- /setup/infra/cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "project_id" { 18 | description = "Project ID" 19 | value = var.project_id 20 | } 21 | 22 | output "region" { 23 | description = "Region" 24 | value = var.region 25 | } -------------------------------------------------------------------------------- /setup/infra/cluster/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable project_id {} 18 | variable region {} 19 | 20 | variable release_channel { 21 | description = "Configuration options for the Release channels https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels" 22 | default = "REGULAR" 23 | } 24 | 25 | variable service_account { 26 | # If not specified, will use: broker@${var.project_id}.iam.gserviceaccount.com 27 | default = "" 28 | } 29 | 30 | variable zones { 31 | type = list 32 | default = [] 33 | } 34 | 35 | variable name { 36 | default = "broker" 37 | } 38 | 39 | variable network_policy { 40 | default = true 41 | } 42 | 43 | // Set this to a lower value if your pod cidr is constrained. 44 | variable max_pods_per_node { 45 | default = 110 46 | } 47 | 48 | # Default node pool counts per zone 49 | variable default_pool_machine_type { 50 | default = "e2-standard-4" 51 | } 52 | variable default_pool_initial_node_count { 53 | default = 2 54 | } 55 | variable default_pool_min_node_count { 56 | default = 2 57 | } 58 | variable default_pool_max_node_count { 59 | default = 10 60 | } 61 | variable default_pool_preemptive_nodes { 62 | default = false 63 | } 64 | variable default_pool_disk_size_gb { 65 | default = 100 66 | } 67 | variable default_pool_disk_type { 68 | default = "pd-standard" 69 | } 70 | 71 | variable "ip_cidr_range" { 72 | description = "Custom IP CIDR ranges" 73 | type = map(string) 74 | default = { 75 | nodes = "" 76 | pods = "" 77 | services = "" 78 | } 79 | } -------------------------------------------------------------------------------- /setup/infra/cluster/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 0.12" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "~> 3.19" 23 | google-beta = ">= 3.23.0, <4.0.0, != 3.29.0" 24 | kubernetes = "~> 1.11.0" 25 | template = "~> 2.1" 26 | null = "~> 2.1" 27 | random = "~> 2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /setup/infra/create_cloudep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | cleanup() { 18 | rm -f cloudep.yaml 19 | } 20 | trap cleanup EXIT 21 | 22 | function log() { 23 | level=$1 24 | msg=$2 25 | echo "${level}: $msg" >&2 26 | } 27 | 28 | [[ -z "$NAME" || -z "$TARGET" || -z "$PROJECT" ]] && echo "ERRO: required env vars: NAME TARGET PROJECT" && exit 1 29 | 30 | SVC="${NAME}.endpoints.${PROJECT}.cloud.goog" 31 | 32 | cat - > cloudep.yaml <&2 48 | 49 | log "INFO" "Forcing undelete of Endpoint Service" 50 | gcloud --project ${PROJECT} endpoints services undelete ${SVC} >/dev/null 2>&1 || true 51 | 52 | gcloud --project ${PROJECT} -q endpoints services deploy cloudep.yaml 1>&2 53 | 54 | log "INFO" "Cloud Endpoint config ID ${PROJECT}:" 55 | gcloud --project ${PROJECT} endpoints services describe ${SVC} --format='value(serviceConfig.id)' 56 | 57 | log "INFO" "Created Cloud Endpoint: ${SVC}" -------------------------------------------------------------------------------- /setup/infra/gcr.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 The Selkies Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_pubsub_topic" "gcr" { 18 | name = "gcr" 19 | } 20 | 21 | resource "google_pubsub_topic_iam_member" "broker" { 22 | project = var.project_id 23 | topic = google_pubsub_topic.gcr.name 24 | role = "roles/editor" 25 | member = "serviceAccount:${google_service_account.cluster_service_account.email}" 26 | } -------------------------------------------------------------------------------- /setup/infra/networks.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_compute_network" "broker" { 18 | name = var.name 19 | auto_create_subnetworks = false 20 | depends_on = [ 21 | google_project_service.compute 22 | ] 23 | } 24 | 25 | resource "google_compute_firewall" "iap" { 26 | name = "allow-ingress-from-iap" 27 | network = replace( 28 | google_compute_network.broker.self_link, 29 | "https://www.googleapis.com/compute/v1/", 30 | "", 31 | ) 32 | 33 | allow { 34 | protocol = "tcp" 35 | ports = ["22", "3389"] 36 | } 37 | source_ranges = ["35.235.240.0/20"] 38 | } 39 | 40 | -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | _REGION: us-central1 20 | _TIER1: "true" 21 | _TIER2: "true" 22 | tags: 23 | - selkies-node-pool-apps 24 | - selkies-setup 25 | steps: 26 | ### 27 | # Apply terraform to create cluster 28 | ### 29 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 30 | id: "terraform-apply" 31 | entrypoint: "/workspace/deploy.sh" 32 | env: 33 | - TF_VAR_project_id=${PROJECT_ID} 34 | - TF_VAR_name=${_NAME} 35 | - TF_VAR_region=${_REGION} 36 | - TF_VAR_tier1_pool_enabled=${_TIER1} 37 | - TF_VAR_tier2_pool_enabled=${_TIER2} 38 | - TERRAFORM_WORKSPACE_NAME=node-pool-apps-${_REGION} 39 | - ACTION=${_ACTION} 40 | -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | data "google_container_cluster" "broker" { 18 | name = "${var.name}-${var.region}" 19 | location = var.region 20 | } 21 | 22 | data "google_service_account" "broker_cluster" { 23 | account_id = var.name 24 | } 25 | -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/tier1-ubuntu.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_container_node_pool" "tier1-ubuntu" { 18 | provider = google-beta 19 | count = var.tier1_ubuntu_pool_enabled ? 1 : 0 20 | name = "tier1-ubuntu" 21 | location = var.region 22 | cluster = data.google_container_cluster.broker.name 23 | initial_node_count = var.tier1_ubuntu_pool_initial_node_count 24 | 25 | node_config { 26 | preemptible = var.tier1_ubuntu_pool_preemptive_nodes 27 | machine_type = var.tier1_ubuntu_pool_machine_type 28 | 29 | service_account = data.google_service_account.broker_cluster.email 30 | 31 | disk_size_gb = var.tier1_ubuntu_pool_disk_size_gb 32 | disk_type = var.tier1_ubuntu_pool_disk_type 33 | 34 | ephemeral_storage_config { 35 | local_ssd_count = var.tier1_ubuntu_pool_ephemeral_storage_ssd_count 36 | } 37 | 38 | image_type = "UBUNTU" 39 | 40 | oauth_scopes = [ 41 | "https://www.googleapis.com/auth/cloud-platform", 42 | ] 43 | 44 | metadata = { 45 | cluster_name = data.google_container_cluster.broker.name 46 | node_pool = "tier1-ubuntu" 47 | disable-legacy-endpoints = "true" 48 | } 49 | 50 | labels = { 51 | cluster_name = data.google_container_cluster.broker.name 52 | node_pool = "tier1-ubuntu" 53 | 54 | # updated by node init daemonset when finished. 55 | "app.broker/initialized" = "false" 56 | 57 | # Used to set pod affinity 58 | "app.broker/tier" = "tier1-ubuntu" 59 | } 60 | 61 | taint = [ 62 | { 63 | # Taint to be removed when node init daemonset completes. 64 | key = "app.broker/node-init" 65 | value = true 66 | effect = "NO_SCHEDULE" 67 | }, 68 | { 69 | # Repel pods without the tier toleration. 70 | key = "app.broker/tier" 71 | value = "tier1-ubuntu" 72 | effect = "NO_SCHEDULE" 73 | }, 74 | ] 75 | } 76 | 77 | management { 78 | auto_repair = true 79 | auto_upgrade = true 80 | } 81 | 82 | autoscaling { 83 | min_node_count = var.tier1_ubuntu_pool_min_node_count 84 | max_node_count = var.tier1_ubuntu_pool_max_node_count 85 | } 86 | 87 | // node labels and taints are modified dynamically by the node init containers 88 | // ignore changes so that Terraform doesn't try to undo their modifications. 89 | lifecycle { 90 | ignore_changes = [ 91 | node_config[0].labels, 92 | node_config[0].taint 93 | ] 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/tier1.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_container_node_pool" "tier1" { 18 | provider = google-beta 19 | count = var.tier1_pool_enabled ? 1 : 0 20 | name = "tier1" 21 | location = var.region 22 | cluster = data.google_container_cluster.broker.name 23 | initial_node_count = var.tier1_pool_initial_node_count 24 | 25 | node_config { 26 | preemptible = var.tier1_pool_preemptive_nodes 27 | machine_type = var.tier1_pool_machine_type 28 | 29 | service_account = data.google_service_account.broker_cluster.email 30 | 31 | disk_size_gb = var.tier1_pool_disk_size_gb 32 | disk_type = var.tier1_pool_disk_type 33 | 34 | ephemeral_storage_config { 35 | local_ssd_count = var.tier1_pool_ephemeral_storage_ssd_count 36 | } 37 | 38 | image_type = "COS" 39 | 40 | oauth_scopes = [ 41 | "https://www.googleapis.com/auth/cloud-platform", 42 | ] 43 | 44 | metadata = { 45 | cluster_name = data.google_container_cluster.broker.name 46 | node_pool = "tier1" 47 | disable-legacy-endpoints = "true" 48 | } 49 | 50 | labels = { 51 | cluster_name = data.google_container_cluster.broker.name 52 | node_pool = "tier1" 53 | 54 | # updated by node init daemonset when finished. 55 | "app.broker/initialized" = "false" 56 | 57 | # Used to set pod affinity 58 | "app.broker/tier" = "tier1" 59 | } 60 | 61 | taint = [ 62 | { 63 | # Taint to be removed when node init daemonset completes. 64 | key = "app.broker/node-init" 65 | value = true 66 | effect = "NO_SCHEDULE" 67 | }, 68 | { 69 | # Repel pods without the tier toleration. 70 | key = "app.broker/tier" 71 | value = "tier1" 72 | effect = "NO_SCHEDULE" 73 | }, 74 | ] 75 | } 76 | 77 | management { 78 | auto_repair = true 79 | auto_upgrade = true 80 | } 81 | 82 | autoscaling { 83 | min_node_count = var.tier1_pool_min_node_count 84 | max_node_count = var.tier1_pool_max_node_count 85 | } 86 | 87 | // node labels and taints are modified dynamically by the node init containers 88 | // ignore changes so that Terraform doesn't try to undo their modifications. 89 | lifecycle { 90 | ignore_changes = [ 91 | node_config[0].labels, 92 | node_config[0].taint 93 | ] 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/tier2.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_container_node_pool" "tier2" { 18 | provider = google-beta 19 | count = var.tier2_pool_enabled ? 1 : 0 20 | name = "tier2" 21 | location = var.region 22 | cluster = data.google_container_cluster.broker.name 23 | initial_node_count = var.tier2_pool_initial_node_count 24 | 25 | node_config { 26 | preemptible = var.tier2_pool_preemptive_nodes 27 | machine_type = var.tier2_pool_machine_type 28 | 29 | service_account = data.google_service_account.broker_cluster.email 30 | 31 | disk_size_gb = var.tier2_pool_disk_size_gb 32 | disk_type = var.tier2_pool_disk_type 33 | 34 | ephemeral_storage_config { 35 | local_ssd_count = var.tier2_pool_ephemeral_storage_ssd_count 36 | } 37 | 38 | image_type = "COS" 39 | 40 | oauth_scopes = [ 41 | "https://www.googleapis.com/auth/cloud-platform", 42 | ] 43 | 44 | metadata = { 45 | cluster_name = data.google_container_cluster.broker.name 46 | node_pool = "tier2" 47 | disable-legacy-endpoints = "true" 48 | } 49 | 50 | labels = { 51 | cluster_name = data.google_container_cluster.broker.name 52 | node_pool = "tier2" 53 | 54 | # updated by node init daemonset when finished. 55 | "app.broker/initialized" = "false" 56 | 57 | # Used to set pod affinity 58 | "app.broker/tier" = "tier2" 59 | } 60 | 61 | taint = [ 62 | { 63 | # Taint to be removed when node init daemonset completes. 64 | key = "app.broker/node-init" 65 | value = true 66 | effect = "NO_SCHEDULE" 67 | }, 68 | { 69 | # Repel pods without the tier toleration. 70 | key = "app.broker/tier" 71 | value = "tier2" 72 | effect = "NO_SCHEDULE" 73 | }, 74 | ] 75 | } 76 | 77 | management { 78 | auto_repair = true 79 | auto_upgrade = true 80 | } 81 | 82 | autoscaling { 83 | min_node_count = var.tier2_pool_min_node_count 84 | max_node_count = var.tier2_pool_max_node_count 85 | } 86 | 87 | // node labels and taints are modified dynamically by the node init containers 88 | // ignore changes so that Terraform doesn't try to undo their modifications. 89 | lifecycle { 90 | ignore_changes = [ 91 | node_config[0].labels, 92 | node_config[0].taint 93 | ] 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable project_id {} 18 | variable region {} 19 | 20 | variable name { 21 | default = "broker" 22 | } 23 | 24 | # Tier 1 COS node pool parameters 25 | variable tier1_pool_enabled { 26 | default = true 27 | } 28 | variable tier1_pool_machine_type { 29 | default = "e2-standard-4" 30 | } 31 | variable tier1_pool_initial_node_count { 32 | default = 1 33 | } 34 | variable tier1_pool_min_node_count { 35 | default = 0 36 | } 37 | variable tier1_pool_max_node_count { 38 | default = 10 39 | } 40 | variable tier1_pool_preemptive_nodes { 41 | default = false 42 | } 43 | variable tier1_pool_disk_size_gb { 44 | default = 100 45 | } 46 | variable tier1_pool_disk_type { 47 | default = "pd-balanced" 48 | } 49 | variable tier1_pool_ephemeral_storage_ssd_count { 50 | description = "use local-ssd for ephemeral container storage. NOTE: requires either n1, n2 or n2d instance types." 51 | default = 0 52 | } 53 | 54 | # Tier 1 Ubuntu node pool parameters 55 | variable tier1_ubuntu_pool_enabled { 56 | default = false 57 | } 58 | variable tier1_ubuntu_pool_machine_type { 59 | default = "e2-standard-4" 60 | } 61 | variable tier1_ubuntu_pool_initial_node_count { 62 | default = 1 63 | } 64 | variable tier1_ubuntu_pool_min_node_count { 65 | default = 0 66 | } 67 | variable tier1_ubuntu_pool_max_node_count { 68 | default = 10 69 | } 70 | variable tier1_ubuntu_pool_preemptive_nodes { 71 | default = false 72 | } 73 | variable tier1_ubuntu_pool_disk_size_gb { 74 | default = 100 75 | } 76 | variable tier1_ubuntu_pool_disk_type { 77 | default = "pd-balanced" 78 | } 79 | variable tier1_ubuntu_pool_ephemeral_storage_ssd_count { 80 | description = "use local-ssd for ephemeral container storage. NOTE: requires either n1, n2 or n2d instance types." 81 | default = 0 82 | } 83 | 84 | # Tier 2 node pool parameters 85 | variable tier2_pool_enabled { 86 | default = false 87 | } 88 | variable tier2_pool_machine_type { 89 | default = "e2-highcpu-32" 90 | } 91 | variable tier2_pool_initial_node_count { 92 | default = 0 93 | } 94 | variable tier2_pool_min_node_count { 95 | default = 0 96 | } 97 | variable tier2_pool_max_node_count { 98 | default = 10 99 | } 100 | variable tier2_pool_preemptive_nodes { 101 | default = false 102 | } 103 | variable tier2_pool_disk_size_gb { 104 | default = 100 105 | } 106 | variable tier2_pool_disk_type { 107 | default = "pd-balanced" 108 | } 109 | variable tier2_pool_ephemeral_storage_ssd_count { 110 | description = "use local-ssd for ephemeral container storage. NOTE: requires either n1, n2 or n2d instance types." 111 | default = 0 112 | } -------------------------------------------------------------------------------- /setup/infra/node-pool-apps/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 1.2.3" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "~> 4.25.0, <4.25.6" 23 | google-beta ="~> 4.25.0" 24 | kubernetes = "~> 2.11.0" 25 | template = "~> 2.1" 26 | null = "~> 2.1" 27 | random = "~> 2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /setup/infra/node-pool-gpu/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | _REGION: us-central1 20 | _COS: "true" 21 | _UBUNTU: "true" 22 | tags: 23 | - selkies-node-pool-gpu 24 | - selkies-setup 25 | steps: 26 | ### 27 | # Apply terraform to create cluster 28 | ### 29 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 30 | id: "terraform-apply" 31 | entrypoint: "/workspace/deploy.sh" 32 | env: 33 | - TF_VAR_project_id=${PROJECT_ID} 34 | - TF_VAR_name=${_NAME} 35 | - TF_VAR_region=${_REGION} 36 | - TF_VAR_gpu_cos_pool_enabled=${_COS} 37 | - TF_VAR_gpu_ubuntu_pool_enabled=${_UBUNTU} 38 | - TERRAFORM_WORKSPACE_NAME=node-pool-gpu-${_REGION} 39 | - ACTION=${_ACTION} 40 | -------------------------------------------------------------------------------- /setup/infra/node-pool-gpu/gpu-cos.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_container_node_pool" "gpu-cos" { 18 | provider = google-beta 19 | count = var.gpu_cos_pool_enabled ? 1 : 0 20 | name = "gpu-cos" 21 | location = var.region 22 | cluster = data.google_container_cluster.broker.name 23 | initial_node_count = var.gpu_cos_pool_initial_node_count 24 | 25 | node_config { 26 | preemptible = var.gpu_cos_pool_preemptive_nodes 27 | machine_type = var.gpu_cos_pool_machine_type 28 | 29 | service_account = data.google_service_account.broker_cluster.email 30 | 31 | disk_size_gb = var.gpu_cos_pool_disk_size_gb 32 | disk_type = var.gpu_cos_pool_disk_type 33 | 34 | ephemeral_storage_config { 35 | local_ssd_count = var.gpu_cos_pool_ephemeral_storage_ssd_count 36 | } 37 | 38 | image_type = "COS" 39 | 40 | guest_accelerator { 41 | count = 1 42 | type = length(var.gpu_cos_accelerator_type) == 0 ? lookup(local.accelerator_type_regions, var.region) : var.gpu_cos_accelerator_type 43 | } 44 | 45 | oauth_scopes = [ 46 | "https://www.googleapis.com/auth/cloud-platform", 47 | ] 48 | 49 | metadata = { 50 | cluster_name = data.google_container_cluster.broker.name 51 | node_pool = "gpu-cos" 52 | disable-legacy-endpoints = "true" 53 | } 54 | 55 | labels = { 56 | cluster_name = data.google_container_cluster.broker.name 57 | node_pool = "gpu-cos" 58 | 59 | # updated by node init daemonset when finished. 60 | "app.broker/initialized" = "false" 61 | 62 | # Used to set pod affinity 63 | "app.broker/tier" = "gpu-cos" 64 | 65 | # updated by gpu driver installer to true when finished. 66 | "cloud.google.com/gke-accelerator-initialized" = "false" 67 | } 68 | 69 | taint = [ 70 | { 71 | # Taint to be removed when node init daemonset completes. 72 | key = "app.broker/node-init" 73 | value = true 74 | effect = "NO_SCHEDULE" 75 | }, 76 | { 77 | # Repel pods without the tier toleration. 78 | key = "app.broker/tier" 79 | value = "gpu-cos" 80 | effect = "NO_SCHEDULE" 81 | }, 82 | { 83 | # Removed when GPU driver installer daemonset completes. 84 | key = "cloud.google.com/gke-accelerator-init" 85 | value = "true" 86 | effect = "NO_SCHEDULE" 87 | }, 88 | ] 89 | } 90 | 91 | management { 92 | auto_repair = true 93 | auto_upgrade = true 94 | } 95 | 96 | autoscaling { 97 | min_node_count = var.gpu_cos_pool_min_node_count 98 | max_node_count = var.gpu_cos_pool_max_node_count 99 | } 100 | 101 | // node labels and taints are modified dynamically by the node init containers 102 | // ignore changes so that Terraform doesn't try to undo their modifications. 103 | lifecycle { 104 | ignore_changes = [ 105 | node_config[0].labels, 106 | node_config[0].taint 107 | ] 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /setup/infra/node-pool-gpu/locals.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Built from: 18 | // gcloud compute accelerator-types list |grep nvidia-tesla- | grep -v vws | sort -k2 -r 19 | locals { 20 | accelerator_type_regions = { 21 | "us-west1" = "nvidia-tesla-t4", 22 | "us-west2" = "nvidia-tesla-p4", 23 | "us-central1" = "nvidia-tesla-t4", 24 | "us-east1" = "nvidia-tesla-t4", 25 | "us-east4" = "nvidia-tesla-p4", 26 | "northamerica-northeast1" = "nvidia-tesla-p4", 27 | "southamerica-east1" = "nvidia-tesla-t4", 28 | "europe-west1" = "nvidia-tesla-p100", 29 | "europe-west2" = "nvidia-tesla-t4", 30 | "europe-west3" = "nvidia-tesla-t4", 31 | "europe-west4" = "nvidia-tesla-t4", 32 | "asia-east1" = "nvidia-tesla-p100", 33 | "asia-northeast1" = "nvidia-tesla-t4", 34 | "asia-northeast3" = "nvidia-tesla-t4", 35 | "asia-south1" = "nvidia-tesla-t4", 36 | "asia-southeast1" = "nvidia-tesla-t4", 37 | "australia-southeast1" = "nvidia-tesla-p4", 38 | } 39 | } -------------------------------------------------------------------------------- /setup/infra/node-pool-gpu/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | data "google_container_cluster" "broker" { 18 | name = "${var.name}-${var.region}" 19 | location = var.region 20 | } 21 | 22 | data "google_service_account" "broker_cluster" { 23 | account_id = var.name 24 | } 25 | -------------------------------------------------------------------------------- /setup/infra/node-pool-gpu/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable project_id {} 18 | variable region {} 19 | 20 | variable name { 21 | default = "broker" 22 | } 23 | 24 | # GPU COS node pool parameters 25 | variable gpu_cos_pool_enabled { 26 | default = true 27 | } 28 | variable gpu_cos_accelerator_type { 29 | # will use value from accelerator_type_regions if not given. 30 | default = "" 31 | } 32 | variable gpu_cos_pool_machine_type { 33 | default = "n1-standard-8" 34 | } 35 | variable gpu_cos_pool_initial_node_count { 36 | default = 1 37 | } 38 | variable gpu_cos_pool_min_node_count { 39 | default = 0 40 | } 41 | variable gpu_cos_pool_max_node_count { 42 | default = 10 43 | } 44 | variable gpu_cos_pool_preemptive_nodes { 45 | default = false 46 | } 47 | variable gpu_cos_pool_disk_size_gb { 48 | default = 100 49 | } 50 | variable gpu_cos_pool_disk_type { 51 | default = "pd-balanced" 52 | } 53 | variable gpu_cos_pool_ephemeral_storage_ssd_count { 54 | description = "use local-ssd for ephemeral container storage. NOTE: requires either n1, n2 or n2d instance types." 55 | default = 0 56 | } 57 | 58 | # GPU Ubuntu node pool parameters 59 | variable gpu_ubuntu_pool_enabled { 60 | default = true 61 | } 62 | variable gpu_ubuntu_pool_machine_type { 63 | default = "n1-standard-8" 64 | } 65 | variable gpu_ubuntu_accelerator_type { 66 | # will use value from accelerator_type_regions if not given. 67 | default = "" 68 | } 69 | variable gpu_ubuntu_pool_initial_node_count { 70 | default = 0 71 | } 72 | variable gpu_ubuntu_pool_min_node_count { 73 | default = 0 74 | } 75 | variable gpu_ubuntu_pool_max_node_count { 76 | default = 10 77 | } 78 | variable gpu_ubuntu_pool_preemptive_nodes { 79 | default = false 80 | } 81 | variable gpu_ubuntu_pool_disk_size_gb { 82 | default = 100 83 | } 84 | variable gpu_ubuntu_pool_disk_type { 85 | default = "pd-balanced" 86 | } 87 | variable gpu_ubuntu_pool_ephemeral_storage_ssd_count { 88 | description = "use local-ssd for ephemeral container storage. NOTE: requires either n1, n2 or n2d instance types." 89 | default = 0 90 | } -------------------------------------------------------------------------------- /setup/infra/node-pool-gpu/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 1.2.3" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "~> 4.25.0, <4.25.6" 23 | google-beta ="~> 4.25.0" 24 | kubernetes = "~> 2.11.0" 25 | template = "~> 2.1" 26 | null = "~> 2.1" 27 | random = "~> 2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /setup/infra/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "project_id" { 18 | description = "Project ID" 19 | value = var.project_id 20 | } 21 | 22 | output "name" { 23 | description = "Name of infrastructure deployment" 24 | value = var.name 25 | } 26 | 27 | output "cloud-ep-endpoint" { 28 | description = "Cloud Endpoint DNS" 29 | value = local.cloud_endpoint 30 | } 31 | 32 | output "cloud-dns" { 33 | description = "Cloud DNS Endpoint, if enabled." 34 | // TODO 35 | value = "" 36 | } 37 | 38 | output "backend-service" { 39 | description = "name of backend service that NEGs should attach to" 40 | value = google_compute_backend_service.ingress.name 41 | } 42 | 43 | output "static-ip-name" { 44 | description = "Name of static external IP for ingress" 45 | value = google_compute_global_address.ingress.name 46 | } 47 | 48 | output "static-ip-address" { 49 | description = "Address of static external IP for ingress" 50 | value = google_compute_global_address.ingress.address 51 | } 52 | 53 | output "node-service-account" { 54 | description = "Service account used by the node" 55 | value = google_service_account.cluster_service_account.name 56 | } 57 | 58 | output "cnrm-system-service-account" { 59 | description = "Service account for CNRM" 60 | value = google_service_account.cnrm-system.name 61 | } 62 | 63 | output "autoneg-system-service-account" { 64 | description = "Service account for AutoNEG controller" 65 | value = google_service_account.autoneg-system.name 66 | } 67 | -------------------------------------------------------------------------------- /setup/infra/private-cluster/.gitignore: -------------------------------------------------------------------------------- 1 | .terraform 2 | *.tfstate -------------------------------------------------------------------------------- /setup/infra/private-cluster/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | _REGION: us-central1 20 | tags: 21 | - selkies-private-cluster 22 | - selkies-setup 23 | steps: 24 | ### 25 | # Apply terraform to create cluster 26 | ### 27 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 28 | id: "terraform-apply" 29 | entrypoint: "/workspace/deploy.sh" 30 | env: 31 | - TF_VAR_project_id=${PROJECT_ID} 32 | - TF_VAR_name=${_NAME} 33 | - TF_VAR_region=${_REGION} 34 | - TERRAFORM_WORKSPACE_NAME=${_REGION} 35 | - ACTION=${_ACTION} 36 | -------------------------------------------------------------------------------- /setup/infra/private-cluster/locals.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Built from: 18 | // gcloud compute accelerator-types list |grep nvidia-tesla- | grep -v vws | sort -k2 -r 19 | locals { 20 | // NOTE: if you plan on using accelerators, choose regions that contain the desired accelerator type 21 | 22 | // Map used to create subnets, integer values are used for CIDR range offsets. 23 | // NOTE: If using a cluster with an index greater than 15, 24 | // a ConfigMap in the kube-system namespaced named ip-masq-agent must be created like the one below: 25 | // 26 | // nonMasqueradeCIDRs: 27 | // - 172.0.0.0/10 28 | // resyncInterval: 60s 29 | 30 | cluster_regions = { 31 | "us-west1" = 0, # The Dalles, Oregon, USA 32 | "us-west2" = 1, # Los Angeles, California, USA 33 | "us-central1" = 2, # Council Bluffs, Iowa, USA 34 | "us-east1" = 3, # Moncks Corner, South Carolina, USA 35 | "us-east4" = 4, # Ashburn, Northern Virginia, USA 36 | "northamerica-northeast1" = 5, # Montréal, Québec, Canada 37 | "southamerica-east1" = 6, # Osasco (São Paulo), Brazil 38 | "europe-west1" = 7, # St. Ghislain, Belgium 39 | "europe-west2" = 8, # London, England, UK 40 | "europe-west3" = 16, # Frankfurt, Germany 41 | "europe-west4" = 9, # Eemshaven, Netherlands 42 | "asia-east1" = 10, # Changhua County, Taiwan 43 | "asia-northeast1" = 11, # Tokyo, Japan 44 | "asia-northeast3" = 12, # Seoul, South Korea 45 | "asia-south1" = 13, # Mumbai, India 46 | "asia-southeast1" = 14, # Jurong West, Singapore 47 | "australia-southeast1" = 15, # Sydney, Australia 48 | } 49 | 50 | // Map of regions to zones that have accelerators available. 51 | cluster_node_zones = { 52 | "us-west1" = ["us-west1-a"], 53 | "us-west2" = ["us-west2-b"], 54 | "us-central1" = ["us-central1-a"], 55 | "us-east1" = ["us-east1-c"], 56 | "us-east4" = ["us-east4-a"], 57 | "northamerica-northeast1" = ["northamerica-northeast1-a"], 58 | "southamerica-east1" = ["southamerica-east1-c"], 59 | "europe-west1" = ["europe-west1-b"], 60 | "europe-west2" = ["europe-west2-a"], 61 | "europe-west3" = ["europe-west3-b"], 62 | "europe-west4" = ["europe-west4-b"], 63 | "asia-east1" = ["asia-east1-a"], 64 | "asia-northeast1" = ["asia-northeast1-a"], 65 | "asia-northeast3" = ["asia-northeast3-b"], 66 | "asia-south1" = ["asia-south1-a"], 67 | "asia-southeast1" = ["asia-southeast1-b"], 68 | "australia-southeast1" = ["australia-southeast1-a"], 69 | } 70 | 71 | // Placeholder that can be overriden in _override.tf file. 72 | pod_subnets = [] 73 | 74 | // Default subnet values computed from region indices. 75 | default_ip_cidr_range = { 76 | "nodes" = "10.${2 + lookup(local.cluster_regions, var.region)}.0.0/16" 77 | "master" = "172.${2 + lookup(local.cluster_regions, var.region)}.0.0/28" 78 | "services" = "192.168.${lookup(local.cluster_regions, var.region)}.0/24" 79 | "pods" = "172.${16 + lookup(local.cluster_regions, var.region)}.0.0/18" 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /setup/infra/private-cluster/network.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | data "google_compute_network" "broker" { 18 | name = var.name 19 | project = var.project_id 20 | } 21 | 22 | resource "google_compute_subnetwork" "broker" { 23 | name = "${var.name}-${var.region}" 24 | ip_cidr_range = var.ip_cidr_range.nodes != "" ? var.ip_cidr_range.nodes : local.default_ip_cidr_range.nodes 25 | region = var.region 26 | network = data.google_compute_network.broker.self_link 27 | private_ip_google_access = true 28 | 29 | secondary_ip_range = [ 30 | { 31 | range_name = "${var.region}-pods" 32 | ip_cidr_range = var.ip_cidr_range.pods != "" ? var.ip_cidr_range.pods : local.default_ip_cidr_range.pods 33 | }, 34 | { 35 | range_name = "${var.region}-services" 36 | ip_cidr_range = var.ip_cidr_range.services != "" ? var.ip_cidr_range.services : local.default_ip_cidr_range.services 37 | }, 38 | ] 39 | } 40 | 41 | resource "google_compute_address" "nat-address" { 42 | count = 2 43 | name = "broker-nat-${var.region}-${count.index}" 44 | region = var.region 45 | } 46 | 47 | resource "google_compute_router" "router-nat" { 48 | provider = google-beta 49 | name = "broker-nat-${var.region}" 50 | network = data.google_compute_network.broker.id 51 | region = var.region 52 | 53 | bgp { 54 | asn = var.router_asn 55 | } 56 | } 57 | 58 | module "cloud-nat" { 59 | source = "terraform-google-modules/cloud-nat/google" 60 | version = "~> 1.4" 61 | project_id = var.project_id 62 | region = var.region 63 | router = google_compute_router.router-nat.name 64 | nat_ips = google_compute_address.nat-address.*.self_link 65 | } -------------------------------------------------------------------------------- /setup/infra/private-cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "project_id" { 18 | description = "Project ID" 19 | value = var.project_id 20 | } 21 | 22 | output "region" { 23 | description = "Region" 24 | value = var.region 25 | } -------------------------------------------------------------------------------- /setup/infra/private-cluster/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable project_id {} 18 | variable region { 19 | default = "us-west1" 20 | } 21 | 22 | variable release_channel { 23 | description = "Configuration options for the Release channels https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels" 24 | default = "REGULAR" 25 | } 26 | 27 | variable service_account { 28 | # If not specified, will use: broker@${var.project_id}.iam.gserviceaccount.com 29 | default = "" 30 | } 31 | 32 | variable zones { 33 | type = list 34 | default = [] 35 | } 36 | 37 | variable name { 38 | default = "broker" 39 | } 40 | 41 | variable network_policy { 42 | default = true 43 | } 44 | 45 | // Set this to a lower value if your pod cidr is constrained. 46 | variable max_pods_per_node { 47 | default = 100 48 | } 49 | 50 | variable router_asn { 51 | default = 65010 52 | } 53 | 54 | # Default node pool counts per zone 55 | variable default_pool_machine_type { 56 | default = "n1-standard-4" 57 | } 58 | variable default_pool_initial_node_count { 59 | default = 2 60 | } 61 | variable default_pool_min_node_count { 62 | default = 2 63 | } 64 | variable default_pool_max_node_count { 65 | default = 10 66 | } 67 | variable default_pool_preemptive_nodes { 68 | default = false 69 | } 70 | variable default_pool_disk_size_gb { 71 | default = 200 72 | } 73 | variable default_pool_disk_type { 74 | default = "pd-standard" 75 | } 76 | 77 | variable "ip_cidr_range" { 78 | description = "Custom IP CIDR ranges" 79 | type = map(string) 80 | default = { 81 | nodes = "" 82 | pods = "" 83 | services = "" 84 | master = "" 85 | } 86 | } -------------------------------------------------------------------------------- /setup/infra/private-cluster/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 1.2.0" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "~> 4.25.0, <4.25.6" 23 | google-beta ="~> 4.25.0" 24 | kubernetes = "~> 2.11.0" 25 | template = "~> 2.1" 26 | null = "~> 2.1" 27 | random = "~> 2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /setup/infra/regional-lb/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Selkies Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | _REGION: us-central1 20 | tags: 21 | - selkies-cluster 22 | - selkies-setup 23 | steps: 24 | ### 25 | # Apply terraform to create cluster 26 | ### 27 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 28 | id: "terraform-apply" 29 | entrypoint: "/workspace/deploy.sh" 30 | env: 31 | - TF_VAR_project_id=${PROJECT_ID} 32 | - TF_VAR_name=${_NAME} 33 | - TF_VAR_region=${_REGION} 34 | - TERRAFORM_WORKSPACE_NAME=lb-${_REGION} 35 | - ACTION=${_ACTION} -------------------------------------------------------------------------------- /setup/infra/regional-lb/create_cloudep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | cleanup() { 18 | rm -f cloudep.yaml 19 | } 20 | trap cleanup EXIT 21 | 22 | function log() { 23 | level=$1 24 | msg=$2 25 | echo "${level}: $msg" >&2 26 | } 27 | 28 | [[ -z "$NAME" || -z "$TARGET" || -z "$PROJECT" ]] && echo "ERRO: required env vars: NAME TARGET PROJECT" && exit 1 29 | 30 | SVC="${NAME}.endpoints.${PROJECT}.cloud.goog" 31 | 32 | cat - > cloudep.yaml <&2 48 | 49 | log "INFO" "Forcing undelete of Endpoint Service" 50 | gcloud --project ${PROJECT} endpoints services undelete ${SVC} >/dev/null 2>&1 || true 51 | 52 | gcloud --project ${PROJECT} -q endpoints services deploy cloudep.yaml 1>&2 53 | 54 | log "INFO" "Cloud Endpoint config ID ${PROJECT}:" 55 | gcloud --project ${PROJECT} endpoints services describe ${SVC} --format='value(serviceConfig.id)' 56 | 57 | log "INFO" "Created Cloud Endpoint: ${SVC}" -------------------------------------------------------------------------------- /setup/infra/regional-lb/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2021 The Selkies Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | export RED='\033[1;31m' 20 | export CYAN='\033[1;36m' 21 | export GREEN='\033[1;32m' 22 | export NC='\033[0m' # No Color 23 | function log_red() { echo -e "${RED}$@${NC}"; } 24 | function log_cyan() { echo -e "${CYAN}$@${NC}"; } 25 | function log_green() { echo -e "${GREEN}$@${NC}"; } 26 | 27 | SCRIPT_DIR=$(dirname $(readlink -f $0 2>/dev/null) 2>/dev/null || echo "${PWD}/$(dirname $0)") 28 | 29 | cd "${SCRIPT_DIR}" 30 | 31 | # Fetch any Secret Manager secrets named broker-tfvars* and same them to .auto.tfvars files. 32 | for secret in $(gcloud -q secrets list --filter=name~broker-${TF_VAR_region?}-tfvars- --format="value(name)"); do 33 | latest=$(gcloud secrets versions list ${secret} --sort-by=created --format='value(name)' --limit=1) 34 | dest="${secret/broker-tfvars-/}.auto.tfvars" 35 | log_cyan "Creating ${dest} from secret: ${secret}" 36 | gcloud -q secrets versions access ${latest} --secret ${secret} > ${dest} 37 | done 38 | 39 | export TF_IN_AUTOMATION=1 40 | 41 | # Set default project for google provider. 42 | export GOOGLE_PROJECT=${TF_VAR_project_id?} 43 | 44 | # Initialize backend and select workspace 45 | terraform init -upgrade=true -input=false \ 46 | -backend-config="bucket=${TF_VAR_project_id?}-${TF_VAR_name?}-tf-state" \ 47 | -backend-config="prefix=${TF_VAR_name?}" || true 48 | terraform workspace select ${TERRAFORM_WORKSPACE_NAME?} || terraform workspace new ${TERRAFORM_WORKSPACE_NAME?} 49 | terraform init -input=false \ 50 | -backend-config="bucket=${TF_VAR_project_id?}-${TF_VAR_name?}-tf-state" \ 51 | -backend-config="prefix=${TF_VAR_name?}" || true 52 | 53 | if [[ "${ACTION?}" == "destroy" ]]; then 54 | log_cyan "Running terraform destroy..." 55 | terraform destroy -auto-approve -input=false 56 | elif [[ "${ACTION?}" == "plan" ]]; then 57 | log_cyan "Running terraform plan..." 58 | terraform plan -out terraform.tfplan -input=false 59 | elif [[ "${ACTION?}" == "apply" ]]; then 60 | log_cyan "Running terraform plan..." 61 | terraform plan -out terraform.tfplan -input=false 62 | 63 | log_cyan "Running terraform apply..." 64 | terraform apply -input=false terraform.tfplan 65 | fi 66 | 67 | log_green "Done" -------------------------------------------------------------------------------- /setup/infra/regional-lb/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 The Selkies Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output cloud-ep-endpoint { 18 | description = "Cloud Endpoint DNS" 19 | value = local.cloud_endpoint 20 | } 21 | 22 | output backend-service { 23 | description = "name of backend service that NEGs should attach to" 24 | value = google_compute_backend_service.ingress.name 25 | } -------------------------------------------------------------------------------- /setup/infra/regional-lb/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 The Selkies Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "The project ID to host the cluster in" 19 | } 20 | variable region {} 21 | 22 | variable "name" { 23 | default = "broker" 24 | } 25 | 26 | variable "additional_ssl_certificate_domains" { 27 | description = "list of additional domains to add to the managed certificate." 28 | type = list 29 | default = [] 30 | } 31 | -------------------------------------------------------------------------------- /setup/infra/regional-lb/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 The Selkies Authors 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 0.12" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "<3" 23 | google-beta = "~> 3" 24 | kubernetes = "~> 1.11.0" 25 | template = "~> 2.1" 26 | null = "~> 2.1" 27 | random = "~> 2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /setup/infra/services.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | resource "google_project_service" "compute" { 18 | project = var.project_id 19 | service = "compute.googleapis.com" 20 | 21 | disable_dependent_services = true 22 | disable_on_destroy = false 23 | depends_on = [google_project_service.cloudresourcemanager] 24 | } 25 | 26 | resource "google_project_service" "container" { 27 | project = var.project_id 28 | service = "container.googleapis.com" 29 | 30 | disable_dependent_services = true 31 | disable_on_destroy = false 32 | } 33 | 34 | resource "google_project_service" "cloudresourcemanager" { 35 | project = var.project_id 36 | service = "cloudresourcemanager.googleapis.com" 37 | 38 | disable_dependent_services = true 39 | disable_on_destroy = false 40 | } 41 | 42 | resource "google_project_service" "iam" { 43 | project = var.project_id 44 | service = "iam.googleapis.com" 45 | 46 | disable_dependent_services = true 47 | disable_on_destroy = false 48 | } -------------------------------------------------------------------------------- /setup/infra/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 1.2.3" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "~> 4.25.0, <4.25.6" 23 | google-beta ="~> 4.25.0" 24 | kubernetes = "~> 2.11.0" 25 | template = "~> 2.1" 26 | null = "~> 2.1" 27 | random = "~> 2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /setup/infra/wi-sa/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 10800s 16 | substitutions: 17 | _ACTION: apply 18 | _NAME: broker 19 | tags: 20 | - selkies-wi-sa 21 | - selkies-setup 22 | steps: 23 | ### 24 | # Apply terraform to create cluster 25 | ### 26 | - name: "gcr.io/${PROJECT_ID}/${_NAME}-installer" 27 | id: "terraform-apply" 28 | entrypoint: "/workspace/deploy.sh" 29 | env: 30 | - TF_VAR_project_id=${PROJECT_ID} 31 | - TF_VAR_name=${_NAME} 32 | - TERRAFORM_WORKSPACE_NAME=wi 33 | - ACTION=${_ACTION} 34 | -------------------------------------------------------------------------------- /setup/infra/wi-sa/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | export RED='\033[1;31m' 20 | export CYAN='\033[1;36m' 21 | export GREEN='\033[1;32m' 22 | export NC='\033[0m' # No Color 23 | function log_red() { echo -e "${RED}$@${NC}"; } 24 | function log_cyan() { echo -e "${CYAN}$@${NC}"; } 25 | function log_green() { echo -e "${GREEN}$@${NC}"; } 26 | 27 | SCRIPT_DIR=$(dirname $(readlink -f $0 2>/dev/null) 2>/dev/null || echo "${PWD}/$(dirname $0)") 28 | 29 | cd "${SCRIPT_DIR}" 30 | 31 | # Fetch any Secret Manager secrets named broker-tfvars* and same them to .auto.tfvars files. 32 | for secret in $(gcloud -q secrets list --filter=name~broker-tfvars- --format="value(name)"); do 33 | latest=$(gcloud secrets versions list ${secret} --sort-by=created --format='value(name)' --limit=1) 34 | dest="${secret/broker-tfvars-/}.auto.tfvars" 35 | log_cyan "Creating ${dest} from secret: ${secret}" 36 | gcloud -q secrets versions access ${latest} --secret ${secret} > ${dest} 37 | done 38 | 39 | export TF_IN_AUTOMATION=1 40 | 41 | # Set default project for google provider. 42 | export GOOGLE_PROJECT=${TF_VAR_project_id?} 43 | 44 | # Initialize backend and select workspace 45 | terraform init -upgrade=true -input=false \ 46 | -backend-config="bucket=${TF_VAR_project_id?}-${TF_VAR_name?}-tf-state" \ 47 | -backend-config="prefix=${TF_VAR_name?}" || true 48 | terraform workspace select ${TERRAFORM_WORKSPACE_NAME?} || terraform workspace new ${TERRAFORM_WORKSPACE_NAME?} 49 | terraform init -input=false \ 50 | -backend-config="bucket=${TF_VAR_project_id?}-${TF_VAR_name?}-tf-state" \ 51 | -backend-config="prefix=${TF_VAR_name?}" || true 52 | 53 | if [[ "${ACTION?}" == "destroy" ]]; then 54 | log_cyan "Running terraform destroy..." 55 | terraform destroy -auto-approve -input=false 56 | elif [[ "${ACTION?}" == "plan" ]]; then 57 | log_cyan "Running terraform plan..." 58 | terraform plan -out terraform.tfplan -input=false 59 | elif [[ "${ACTION?}" == "apply" ]]; then 60 | log_cyan "Running terraform plan..." 61 | terraform plan -out terraform.tfplan -input=false 62 | 63 | log_cyan "Running terraform apply..." 64 | terraform apply -input=false terraform.tfplan 65 | fi 66 | 67 | log_green "Done" -------------------------------------------------------------------------------- /setup/infra/wi-sa/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | backend "gcs" {} 19 | required_version = ">= 0.12" 20 | required_providers { 21 | external = "~> 1.2.0" 22 | google = "~> 4.25.0, <4.25.6" 23 | google-beta = "~> 4.25.0" 24 | template = "~> 2.1" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /setup/infra/wi-sa/wi.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable project_id {} 18 | 19 | data "terraform_remote_state" "broker" { 20 | backend = "gcs" 21 | 22 | config = { 23 | bucket = "${var.project_id}-broker-tf-state" 24 | prefix = "broker" 25 | } 26 | 27 | workspace = "default" 28 | } 29 | 30 | # Workload Identity IAM binding for CNRM. 31 | resource "google_service_account_iam_member" "cnrm-sa-workload-identity" { 32 | service_account_id = data.terraform_remote_state.broker.outputs.cnrm-system-service-account 33 | role = "roles/iam.workloadIdentityUser" 34 | member = "serviceAccount:${var.project_id}.svc.id.goog[cnrm-system/cnrm-controller-manager]" 35 | } 36 | 37 | # Workload Identity IAM binding for broker in pod-broker-system namespace. 38 | resource "google_service_account_iam_member" "broker-default-sa-workload-identity" { 39 | service_account_id = data.terraform_remote_state.broker.outputs.node-service-account 40 | role = "roles/iam.workloadIdentityUser" 41 | member = "serviceAccount:${var.project_id}.svc.id.goog[pod-broker-system/pod-broker]" 42 | } 43 | 44 | # Workload Identity IAM binding for broker node init in kube-system namespace. 45 | resource "google_service_account_iam_member" "broker-init-sa-workload-identity" { 46 | service_account_id = data.terraform_remote_state.broker.outputs.node-service-account 47 | role = "roles/iam.workloadIdentityUser" 48 | member = "serviceAccount:${var.project_id}.svc.id.goog[kube-system/pod-broker-node-init]" 49 | } 50 | 51 | # Workload Identity IAM binding for AutoNEG controller. 52 | resource "google_service_account_iam_member" "autoneg-sa-workload-identity" { 53 | service_account_id = data.terraform_remote_state.broker.outputs.autoneg-system-service-account 54 | role = "roles/iam.workloadIdentityUser" 55 | member = "serviceAccount:${var.project_id}.svc.id.goog[autoneg-system/autoneg-system]" 56 | } -------------------------------------------------------------------------------- /setup/manifests/base/autoneg-system/autoneg-serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: ServiceAccount 17 | metadata: 18 | name: autoneg-system 19 | namespace: autoneg-system 20 | annotations: 21 | iam.gke.io/gcp-service-account: autoneg-system@${PROJECT_ID}.iam.gserviceaccount.com 22 | -------------------------------------------------------------------------------- /setup/manifests/base/autoneg-system/autoneg-system-namespace.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Namespace 17 | metadata: 18 | labels: 19 | control-plane: controller-manager 20 | name: autoneg-system 21 | -------------------------------------------------------------------------------- /setup/manifests/base/autoneg-system/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | 18 | namespace: autoneg-system 19 | 20 | resources: 21 | - autoneg-system-namespace.yaml 22 | - autoneg-serviceaccount.yaml 23 | - autoneg.yaml 24 | -------------------------------------------------------------------------------- /setup/manifests/base/custom-metrics/jsonpatch-add-wi.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | - op: add 16 | path: /metadata/annotations 17 | value: 18 | iam.gke.io/gcp-service-account: broker@${PROJECT_ID}.iam.gserviceaccount.com -------------------------------------------------------------------------------- /setup/manifests/base/custom-metrics/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | 18 | resources: 19 | # Upstream source: https://raw.githubusercontent.com/GoogleCloudPlatform/k8s-stackdriver/cm-sd-adapter-v0.12.1/custom-metrics-stackdriver-adapter/deploy/production/adapter_new_resource_model.yaml 20 | - custom-metrics-adapter.yaml 21 | - wi-sa-iampolicymember.yaml 22 | 23 | patchesJson6902: 24 | - target: 25 | version: v1 26 | kind: ServiceAccount 27 | name: custom-metrics-stackdriver-adapter 28 | namespace: custom-metrics 29 | path: jsonpatch-add-wi.yaml 30 | -------------------------------------------------------------------------------- /setup/manifests/base/custom-metrics/wi-sa-iampolicymember.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 16 | kind: IAMPolicyMember 17 | metadata: 18 | name: custom-metrics-wi 19 | namespace: custom-metrics 20 | annotations: 21 | cnrm.cloud.google.com/deletion-policy: "abandon" 22 | spec: 23 | resourceRef: 24 | apiVersion: iam.cnrm.cloud.google.com/v1beta1 25 | kind: IAMServiceAccount 26 | external: projects/${PROJECT_ID}/serviceAccounts/broker@${PROJECT_ID}.iam.gserviceaccount.com 27 | member: serviceAccount:${PROJECT_ID}.svc.id.goog[custom-metrics/custom-metrics-stackdriver-adapter] 28 | role: roles/iam.workloadIdentityUser -------------------------------------------------------------------------------- /setup/manifests/base/ingress/default-gateway.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.istio.io/v1beta1 16 | kind: Gateway 17 | metadata: 18 | name: default 19 | spec: 20 | selector: 21 | istio: ingressgateway 22 | servers: 23 | - port: 24 | number: 80 25 | name: http 26 | protocol: HTTP 27 | # Default gateway handles all hosts 28 | hosts: 29 | - "*" 30 | -------------------------------------------------------------------------------- /setup/manifests/base/ingress/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: kustomize.config.k8s.io/v1beta1 16 | kind: Kustomization 17 | 18 | namespace: istio-system 19 | 20 | resources: 21 | - default-gateway.yaml 22 | - virtualservice.yaml 23 | -------------------------------------------------------------------------------- /setup/manifests/base/ingress/virtualservice.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.istio.io/v1beta1 16 | kind: VirtualService 17 | metadata: 18 | name: default 19 | spec: 20 | hosts: 21 | - "*" 22 | gateways: 23 | - default 24 | http: 25 | - match: 26 | - uri: 27 | prefix: / 28 | # Rewrite / to readiness path 29 | rewrite: 30 | uri: /healthz 31 | # Route to GCLB default http backend in the kube-system namespace. 32 | route: 33 | - destination: 34 | host: default-http-backend.kube-system.svc.cluster.local 35 | port: 36 | number: 80 37 | -------------------------------------------------------------------------------- /setup/manifests/base/istio/istiooperator-1.7.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: install.istio.io/v1alpha2 16 | kind: IstioOperator 17 | metadata: 18 | namespace: istio-operator 19 | name: broker-istiocontrolplane 20 | spec: 21 | # Profiles: https://istio.io/docs/setup/additional-setup/config-profiles/ 22 | profile: default 23 | 24 | components: 25 | policy: 26 | enabled: false 27 | 28 | telemetry: 29 | enabled: false 30 | 31 | pilot: 32 | k8s: 33 | hpaSpec: 34 | # Increasing the min replica count helps with node upgrades. 35 | maxReplicas: 5 36 | minReplicas: 2 37 | resources: 38 | requests: 39 | cpu: 100m 40 | memory: 128Mi 41 | 42 | ingressGateways: 43 | - name: istio-ingressgateway 44 | enabled: true 45 | k8s: 46 | hpaSpec: 47 | # Increasing the min replica count helps with node upgrades. 48 | maxReplicas: 5 49 | minReplicas: 2 50 | overlays: 51 | - kind: Deployment 52 | name: istio-ingressgateway 53 | patches: 54 | # Enable NEG support to work with GCLB and IAP 55 | - path: spec.template.spec.readinessGates 56 | value: 57 | - conditionType: "cloud.google.com/load-balancer-neg-ready" 58 | 59 | - kind: Service 60 | name: istio-ingressgateway 61 | patches: 62 | # Add annotation to service to use NEG and IAP 63 | # Full metadata replacement is a workaround for: https://github.com/istio/istio/issues/19463 64 | - path: metadata 65 | value: 66 | name: istio-ingressgateway 67 | namespace: istio-system 68 | labels: 69 | app: istio-ingressgateway 70 | istio: ingressgateway 71 | release: istio 72 | annotations: 73 | cloud.google.com/neg: '{"exposed_ports": {"80":{}}}' 74 | 75 | # Note that the deployment script updates the value of this annotation to match the correct backend service name. 76 | anthos.cft.dev/autoneg: '{"name":"istio-ingressgateway", "max_rate_per_endpoint":100}' 77 | 78 | # Change type from LoadBalancer to NodePort 79 | # This is to support GCLB and IAP 80 | - path: spec.type 81 | value: NodePort 82 | 83 | # Clear the externalTrafficPolicy for GCLB + NEG support. 84 | - path: spec.externalTrafficPolicy 85 | value: "" 86 | -------------------------------------------------------------------------------- /setup/manifests/base/node/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | namespace: kube-system 16 | 17 | resources: 18 | # Node init daemonset that pulls images 19 | - node-init.yaml 20 | # Storage class to use SSD persistent disks. 21 | - pd-ssd-storage-class.yaml 22 | - pd-balanced-storage-class.yaml 23 | # Storage class to use standard persistent disks with volume expansion enabled. 24 | - pd-standard-storage-class.yaml 25 | # Node termination handler (for preemptible nodes) 26 | - node-termination-handler-ds.yaml 27 | - node-termination-handler-rbac.yaml 28 | -------------------------------------------------------------------------------- /setup/manifests/base/node/node-termination-handler-ds.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | labels: 19 | name: node-termination-handler 20 | name: node-termination-handler 21 | namespace: kube-system 22 | spec: 23 | selector: 24 | matchLabels: 25 | name: node-termination-handler 26 | updateStrategy: 27 | type: RollingUpdate 28 | template: 29 | metadata: 30 | labels: 31 | name: node-termination-handler 32 | spec: 33 | # Necessary to hit the node's metadata server when using Workload Identity 34 | hostNetwork: true 35 | # Necessary to reboot node 36 | hostPID: true 37 | serviceAccountName: node-termination-handler 38 | affinity: 39 | nodeAffinity: 40 | # Restrict to GPU nodes or preemptible nodes 41 | requiredDuringSchedulingIgnoredDuringExecution: 42 | nodeSelectorTerms: 43 | - matchExpressions: 44 | - key: cloud.google.com/gke-accelerator 45 | operator: Exists 46 | - matchExpressions: 47 | - key: cloud.google.com/gke-preemptible 48 | operator: Exists 49 | containers: 50 | # TODO: Update to an official image once the build is automated via GCB. 51 | - image: k8s.gcr.io/gke-node-termination-handler@sha256:aca12d17b222dfed755e28a44d92721e477915fb73211d0a0f8925a1fa847cca 52 | name: node-termination-handler 53 | command: ["./node-termination-handler"] 54 | args: ["--logtostderr", "--exclude-pods=$(POD_NAME):$(POD_NAMESPACE)", "-v=10", "--taint=cloud.google.com/impending-node-termination::NoSchedule"] 55 | securityContext: 56 | capabilities: 57 | # Necessary to reboot node 58 | add: ["SYS_BOOT"] 59 | env: 60 | - name: POD_NAME 61 | valueFrom: 62 | fieldRef: 63 | fieldPath: metadata.name 64 | - name: POD_NAMESPACE 65 | valueFrom: 66 | fieldRef: 67 | fieldPath: metadata.namespace 68 | - name: SLACK_WEBHOOK_URL 69 | value: "" 70 | resources: 71 | limits: 72 | cpu: 150m 73 | memory: 30Mi 74 | tolerations: 75 | # Run regardless of any existing taints. 76 | - effect: NoSchedule 77 | operator: Exists 78 | - effect: NoExecute 79 | operator: Exists -------------------------------------------------------------------------------- /setup/manifests/base/node/node-termination-handler-rbac.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http:#www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | apiVersion: v1 17 | kind: ServiceAccount 18 | metadata: 19 | labels: 20 | k8s-app: node-termination-handler 21 | name: node-termination-handler 22 | namespace: kube-system 23 | --- 24 | kind: ClusterRole 25 | apiVersion: rbac.authorization.k8s.io/v1 26 | metadata: 27 | labels: 28 | k8s-app: node-termination-handler 29 | name: node-termination-handler 30 | namespace: kube-system 31 | rules: 32 | # Allow Node Termination Handler to get and update nodes (for posting taints). 33 | - apiGroups: [""] 34 | resources: ["nodes"] 35 | verbs: ["get", "update"] 36 | # Allow Node Termination Handler to create events 37 | - apiGroups: [""] 38 | resources: ["events"] 39 | verbs: ["create"] 40 | # Allow Node Termination Handler to list and delete pods (for draining nodes) 41 | - apiGroups: [""] 42 | resources: ["pods"] 43 | verbs: ["get", "list", "delete"] 44 | --- 45 | apiVersion: rbac.authorization.k8s.io/v1 46 | kind: ClusterRoleBinding 47 | metadata: 48 | name: node-termination-handler 49 | namespace: kube-system 50 | labels: 51 | k8s-app: node-termination-handler 52 | roleRef: 53 | apiGroup: rbac.authorization.k8s.io 54 | kind: ClusterRole 55 | name: node-termination-handler 56 | subjects: 57 | - kind: ServiceAccount 58 | name: node-termination-handler 59 | namespace: kube-system -------------------------------------------------------------------------------- /setup/manifests/base/node/pd-balanced-storage-class.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: storage.k8s.io/v1 16 | kind: StorageClass 17 | metadata: 18 | name: pd-balanced 19 | provisioner: pd.csi.storage.gke.io 20 | volumeBindingMode: Immediate 21 | allowVolumeExpansion: true 22 | parameters: 23 | type: pd-balanced -------------------------------------------------------------------------------- /setup/manifests/base/node/pd-ssd-storage-class.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: storage.k8s.io/v1 16 | kind: StorageClass 17 | metadata: 18 | name: pd-ssd 19 | provisioner: pd.csi.storage.gke.io 20 | volumeBindingMode: Immediate 21 | allowVolumeExpansion: true 22 | parameters: 23 | type: pd-ssd 24 | -------------------------------------------------------------------------------- /setup/manifests/base/node/pd-standard-storage-class.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: storage.k8s.io/v1 16 | kind: StorageClass 17 | metadata: 18 | name: pd-standard 19 | provisioner: pd.csi.storage.gke.io 20 | volumeBindingMode: Immediate 21 | allowVolumeExpansion: true 22 | parameters: 23 | type: pd-standard 24 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/app-publisher/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | namespace: pod-broker-system 16 | 17 | configMapGenerator: 18 | - name: app-publish-job-template 19 | files: 20 | - config/app-publish-job.yaml.tmpl 21 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/image-puller/config/image-pull-job.yaml.tmpl: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: batch/v1 16 | kind: Job 17 | metadata: 18 | name: "image-pull-{{.NameSuffix}}" 19 | annotations: 20 | pod.broker/image-pull: "{{.NodeName}},{{.Image}}" 21 | spec: 22 | # 1 hour max runtime to pull image. 23 | activeDeadlineSeconds: 3600 24 | # Delete job after it hass been complete for 10 minutes. 25 | ttlSecondsAfterFinished: 600 26 | completions: 1 27 | template: 28 | metadata: 29 | labels: 30 | app: image-pull 31 | app.kubernetes.io/managed-by: pod-broker-image-puller 32 | spec: 33 | restartPolicy: OnFailure 34 | serviceAccountName: pod-broker 35 | nodeSelector: 36 | kubernetes.io/hostname: "{{.NodeName}}" 37 | tolerations: 38 | - effect: "NoSchedule" 39 | operator: "Exists" 40 | volumes: 41 | ### 42 | # Local docker socket for pulling images 43 | ### 44 | - name: docker 45 | hostPath: 46 | path: /var/run/docker.sock 47 | type: Socket 48 | containers: 49 | ### 50 | # Pull image and then exit 51 | ### 52 | - name: image-puller 53 | image: google/cloud-sdk:alpine 54 | command: ["/bin/bash"] 55 | args: 56 | - -ec 57 | - | 58 | echo "Pulling image: ${IMAGE}" 59 | mkdir -p ${HOME}/.docker 60 | echo "${DOCKER_CONFIG_JSON64}" | base64 -d > ${HOME}/.docker/config.json 61 | docker pull ${IMAGE} 62 | if [[ -n "${TAG}" ]]; then 63 | echo "Tagging image ${IMAGE} with ${TAG}" 64 | STRIPTAG=${IMAGE//:*/} 65 | STRIPSHA=${STRIPTAG//@*/} 66 | docker tag ${IMAGE} ${STRIPSHA}:${TAG} 67 | fi 68 | env: 69 | - name: IMAGE 70 | value: {{.Image|quote}} 71 | - name: TAG 72 | value: {{.Tag|quote}} 73 | - name: DOCKER_CONFIG_JSON64 74 | value: {{.DockerConfigJSON64|quote}} 75 | volumeMounts: 76 | - name: docker 77 | mountPath: /var/run/docker.sock 78 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/image-puller/image-puller-daemonset.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: pod-broker-image-puller 19 | spec: 20 | selector: 21 | matchLabels: 22 | app: pod-broker-image-puller 23 | updateStrategy: 24 | type: RollingUpdate 25 | rollingUpdate: 26 | maxUnavailable: 100% 27 | template: 28 | metadata: 29 | labels: 30 | app: pod-broker-image-puller 31 | spec: 32 | serviceAccountName: pod-broker 33 | affinity: 34 | nodeAffinity: 35 | requiredDuringSchedulingIgnoredDuringExecution: 36 | nodeSelectorTerms: 37 | - matchExpressions: 38 | - key: app.broker/initialized 39 | operator: In 40 | values: ["true"] 41 | - key: app.broker/tier 42 | operator: Exists 43 | tolerations: 44 | - effect: "NoSchedule" 45 | operator: "Exists" 46 | volumes: 47 | ### 48 | # Local docker socket for pulling images 49 | ### 50 | - name: docker 51 | hostPath: 52 | path: /var/run/docker.sock 53 | type: Socket 54 | ### 55 | # Job template 56 | ### 57 | - name: image-pull-job-template 58 | configMap: 59 | name: image-pull-job-template 60 | containers: 61 | ### 62 | # Pull images 63 | # This helps to speed up container startup with large images. 64 | ### 65 | - name: image-puller 66 | image: gcr.io/cloud-solutions-images/kube-pod-broker-controller:latest 67 | command: ["/usr/local/bin/image-puller"] 68 | workingDir: /run/image-puller 69 | env: 70 | - name: NAMESPACE 71 | valueFrom: 72 | fieldRef: 73 | fieldPath: metadata.namespace 74 | - name: NODE_NAME 75 | valueFrom: 76 | fieldRef: 77 | fieldPath: spec.nodeName 78 | volumeMounts: 79 | - name: docker 80 | mountPath: /var/run/docker.sock 81 | - name: image-pull-job-template 82 | mountPath: /run/image-puller/template 83 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/image-puller/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | namespace: pod-broker-system 16 | 17 | configMapGenerator: 18 | - name: image-pull-job-template 19 | files: 20 | - config/image-pull-job.yaml.tmpl 21 | 22 | resources: 23 | - image-puller-daemonset.yaml 24 | - subscription-reaper-cronjob.yaml 25 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/image-puller/patch-wait-for-image-cache.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: DaemonSet 17 | metadata: 18 | name: pod-broker-image-puller 19 | spec: 20 | template: 21 | spec: 22 | affinity: 23 | nodeAffinity: 24 | requiredDuringSchedulingIgnoredDuringExecution: 25 | nodeSelectorTerms: 26 | - matchExpressions: 27 | - key: app.broker/initialized 28 | operator: In 29 | values: ["true"] 30 | - key: app.broker/tier 31 | operator: Exists 32 | - key: app.broker/image-load-initialized 33 | operator: In 34 | values: ["true"] 35 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/image-puller/subscription-reaper-cronjob.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Selkies Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | kind: CronJob 16 | apiVersion: batch/v1beta1 17 | metadata: 18 | name: image-puller-subscription-reaper 19 | namespace: pod-broker-system 20 | labels: 21 | k8s-app: image-puller-subscription-reaper 22 | spec: 23 | # UTC time, 24 | schedule: "0 0 * * *" 25 | startingDeadlineSeconds: 3600 26 | concurrencyPolicy: Replace 27 | successfulJobsHistoryLimit: 0 28 | jobTemplate: 29 | spec: 30 | activeDeadlineSeconds: 1800 31 | template: 32 | spec: 33 | serviceAccount: pod-broker 34 | restartPolicy: OnFailure 35 | containers: 36 | ### 37 | # per-node gcr topic subscription reaper script 38 | ### 39 | - image: gcr.io/cloud-solutions-images/kube-pod-broker-controller:latest 40 | name: subscription-reaper 41 | command: ["/bin/bash"] 42 | args: 43 | - "-ec" 44 | - | 45 | SUBSCRIPTIONS=$(gcloud pubsub subscriptions list --format='value(name)') 46 | NODES=$(kubectl get node -o jsonpath='{..metadata.name}') 47 | REGION=$(kubectl get node -o jsonpath='{.items[0].metadata.labels.topology\.kubernetes\.io/region}') 48 | 49 | for sub in $SUBSCRIPTIONS; do 50 | if [[ "$sub" =~ pod-broker-image-puller && "$sub" =~ ${REGION} ]]; then 51 | submatch=$(echo $sub | cut -d'/' -f4 | sed 's/pod-broker-image-puller-//g') 52 | if [[ ! "$NODES" =~ $submatch ]]; then 53 | echo "deleting stale node subscription: $sub" 54 | gcloud pubsub subscriptions delete $sub 55 | fi 56 | fi 57 | 58 | if [[ "$sub" =~ pod-broker-image-finder && "$sub" =~ ${REGION} ]]; then 59 | submatch=$(echo $sub | cut -d'/' -f4 | sed 's/pod-broker-image-finder-//g') 60 | if [[ ! "$NODES" =~ $submatch ]]; then 61 | echo "deleting stale node subscription: $sub" 62 | gcloud pubsub subscriptions delete $sub 63 | fi 64 | fi 65 | done 66 | resources: 67 | requests: 68 | cpu: 10m 69 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | namespace: pod-broker-system 16 | 17 | generatorOptions: 18 | # Generate stable ConfigMap and Secret names. 19 | disableNameSuffixHash: true 20 | 21 | bases: 22 | - app-publisher/ 23 | - image-puller/ 24 | 25 | resources: 26 | - namespace.yaml 27 | - crd.yaml 28 | - broker-deployment.yaml 29 | - network-policy.yaml 30 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/namespace.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Namespace 17 | metadata: 18 | name: pod-broker-system 19 | annotations: 20 | cnrm.cloud.google.com/project-id: ${PROJECT_ID} 21 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/network-policy.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.k8s.io/v1 16 | kind: NetworkPolicy 17 | metadata: 18 | name: deny-all 19 | spec: 20 | podSelector: {} 21 | policyTypes: 22 | - Ingress 23 | ingress: 24 | - from: 25 | - namespaceSelector: 26 | matchLabels: 27 | install.operator.istio.io/owner-kind: IstioControlPlane 28 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/redirect/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Selkies Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | namespace: pod-broker-system 16 | 17 | resources: 18 | - redirect-gateway.yaml 19 | - redirect-serviceentry.yaml 20 | - redirect-virtualservice.yaml 21 | -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/redirect/redirect-gateway.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Selkies Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.istio.io/v1beta1 16 | kind: Gateway 17 | metadata: 18 | name: pod-broker-redirect-gateway 19 | spec: 20 | selector: 21 | istio: ingressgateway 22 | servers: 23 | - port: 24 | number: 80 25 | name: http 26 | protocol: HTTP 27 | hosts: 28 | - "{{OLD_DOMAIN}}" 29 | - "*.{{OLD_DOMAIN}}" -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/redirect/redirect-serviceentry.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Selkies Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ### 16 | # ServiceEntry for external broker redirect URL. 17 | # Required to do route to destinations outside of the cluster. 18 | ### 19 | apiVersion: networking.istio.io/v1beta1 20 | kind: ServiceEntry 21 | metadata: 22 | name: pod-broker-redirect 23 | spec: 24 | hosts: 25 | - "{{NEW_DOMAIN}}" 26 | location: MESH_EXTERNAL 27 | ports: 28 | - number: 80 29 | name: http 30 | protocol: HTTP 31 | resolution: DNS -------------------------------------------------------------------------------- /setup/manifests/base/pod-broker/redirect/redirect-virtualservice.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 The Selkies Authors 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: networking.istio.io/v1beta1 16 | kind: VirtualService 17 | metadata: 18 | name: pod-broker-redirect 19 | spec: 20 | hosts: 21 | - "{{OLD_DOMAIN}}" 22 | gateways: 23 | - pod-broker-system/pod-broker-redirect-gateway 24 | http: 25 | ### 26 | # Redirect to new URL 27 | ### 28 | - match: 29 | - headers: 30 | host: 31 | exact: "{{OLD_DOMAIN}}" 32 | headers: 33 | response: 34 | set: 35 | location: https://{{NEW_DOMAIN}}/ 36 | route: 37 | - destination: 38 | host: "{{NEW_DOMAIN}}" 39 | -------------------------------------------------------------------------------- /setup/manifests/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | timeout: 3600s 16 | substitutions: 17 | _INFRA_NAME: broker 18 | _REGION: us-central1 19 | tags: 20 | - selkies-setup 21 | - selkies-manifests 22 | steps: 23 | ### 24 | # Waits for the cluster to become ready. 25 | ### 26 | - name: "gcr.io/${PROJECT_ID}/${_INFRA_NAME}-installer" 27 | id: "wait-for-infra" 28 | entrypoint: "/workspace/wait_for_clusters.sh" 29 | args: 30 | - ${_INFRA_NAME} 31 | - ${_REGION} 32 | ### 33 | # Deploys manifests to cluster. 34 | ### 35 | - name: "gcr.io/${PROJECT_ID}/${_INFRA_NAME}-installer" 36 | id: "deploy-manifests" 37 | entrypoint: "/workspace/deploy.sh" 38 | env: 39 | - PROJECT_ID=${PROJECT_ID} 40 | - INFRA_NAME=${_INFRA_NAME} 41 | - REGION=${_REGION} 42 | - NODE_SERVICE_ACCOUNT=broker@${PROJECT_ID}.iam.gserviceaccount.com 43 | -------------------------------------------------------------------------------- /setup/manifests/fix_pod_cidr_masq.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2021 The Selkies Authors 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | NODE_POD_CIDR=$(kubectl get node -o jsonpath='{.items[0].spec.podCIDR}') 20 | [[ -z "${NODE_POD_CIDR}" ]] && echo "ERROR: failed to get node pod cidr" && exit 1 21 | 22 | IFS="." read -ra octets <<< ${NODE_POD_CIDR} 23 | 24 | #if [[ "${octets[1]}" -lt 32 ]]; then 25 | # echo "INFO: Pod CIDR range (${NODE_POD_CIDR}) fits within the RFC1918 range and does not need fixing." 26 | # exit 0 27 | #fi 28 | 29 | POD_CIDR="${octets[0]}.${octets[1]}.0.0/18" 30 | 31 | set +e 32 | read -r -d '' MASQ_CONFIG </dev/null || true) 43 | if [[ -n "${CURR_CONFIG_MAP}" ]]; then 44 | CURR_CONFIG=$(echo "$CURR_CONFIG_MAP" | jq -r '.data.config|fromjson') 45 | echo "INFO: Merging ip-masq-agent config with existing nonMasqueradeCIDRs" 46 | MASQ_CONFIG=$(echo ${CURR_CONFIG} | jq --arg cidr "${POD_CIDR}" -r '.nonMasqueradeCIDRs=(.nonMasqueradeCIDRs+[$cidr]|unique)') 47 | else 48 | echo "INFO: Creating ip-masq-agent config for Pod CIDR range: ${POD_CIDR}" 49 | fi 50 | 51 | kubectl create configmap ip-masq-agent -n kube-system --from-file config=<(echo "${MASQ_CONFIG}") --dry-run -o yaml | \ 52 | kubectl apply -f - 53 | -------------------------------------------------------------------------------- /setup/manifests/install_istio_1.14.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | export CYAN='\033[1;36m' 20 | export GREEN='\033[1;32m' 21 | export RED='\033[1;31m' 22 | export NC='\033[0m' # No Color 23 | function log_cyan() { echo -e "${CYAN}$@${NC}"; } 24 | function log_green() { echo -e "${GREEN}$@${NC}"; } 25 | function log_red() { echo -e "${RED}$@${NC}"; } 26 | 27 | [[ -z "${ISTIOCTL}" ]] && log_red "Missing ISTIOCTL env var." && exit 1 28 | 29 | log_cyan "Installing Istio control plane..." 30 | ${ISTIOCTL} manifest generate --set profile=default | kubectl apply -f - 31 | ## wait for EnvoyFilter crd to be ready and reapply 32 | sleep 60 33 | ${ISTIOCTL} manifest generate --set profile=default | kubectl apply -f - 34 | kubectl label ns istio-system install.operator.istio.io/owner-kind=IstioControlPlane --overwrite=true 35 | log_cyan "Waiting for namespace 'istio-system'" 36 | until [[ -n $(kubectl get namespace istio-system -oname 2>/dev/null) ]]; do sleep 2; done 37 | log_cyan "Namespace 'istio-system' is ready" 38 | 39 | log_cyan "Waiting for istio controlplane crds" 40 | until [[ -n $(kubectl get crd gateways.networking.istio.io -oname 2>/dev/null) ]]; do sleep 2; done 41 | until [[ -n $(kubectl get crd virtualservices.networking.istio.io -oname 2>/dev/null) ]]; do sleep 2; done 42 | log_cyan "Istio control plane crds are ready" 43 | kubectl annotate svc istio-ingressgateway cloud.google.com/neg='{"exposed_ports": {"80":{}}}' -n istio-system --overwrite 44 | kubectl annotate svc istio-ingressgateway anthos.cft.dev/autoneg='{"name":"istio-ingressgateway", "max_rate_per_endpoint":100}' -n istio-system --overwrite -------------------------------------------------------------------------------- /setup/manifests/install_istio_1.4.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | export CYAN='\033[1;36m' 20 | export GREEN='\033[1;32m' 21 | export RED='\033[1;31m' 22 | export NC='\033[0m' # No Color 23 | function log_cyan() { echo -e "${CYAN}$@${NC}"; } 24 | function log_green() { echo -e "${GREEN}$@${NC}"; } 25 | function log_red() { echo -e "${RED}$@${NC}"; } 26 | 27 | # Install istio operator 28 | log_cyan "Installing Istio operator..." 29 | kubectl apply -k /opt/istio-operator/deploy/ 30 | 31 | # Create istio control plane 32 | log_cyan "Creating Istio control plane..." 33 | kubectl apply -f base/istio/istiocontrolplane.yaml 34 | 35 | # Wait for operator to create istio control plane objects 36 | # Objects created async by the operator may not exist yet. 37 | # Manual wait for object creation until this is merged: https://github.com/kubernetes/kubernetes/pull/83335 38 | log_cyan "Waiting for namespace 'istio-system'" 39 | until [[ -n $(kubectl get namespace istio-system -oname 2>/dev/null) ]]; do sleep 2; done 40 | log_cyan "Namespace 'istio-system' is ready" 41 | 42 | log_cyan "Waiting for istio controlplane crds" 43 | until [[ -n $(kubectl get crd gateways.networking.istio.io -oname 2>/dev/null) ]]; do sleep 2; done 44 | until [[ -n $(kubectl get crd virtualservices.networking.istio.io -oname 2>/dev/null) ]]; do sleep 2; done 45 | log_cyan "Istio control plane crds are ready" 46 | -------------------------------------------------------------------------------- /setup/manifests/install_istio_1.7.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | export CYAN='\033[1;36m' 20 | export GREEN='\033[1;32m' 21 | export RED='\033[1;31m' 22 | export NC='\033[0m' # No Color 23 | function log_cyan() { echo -e "${CYAN}$@${NC}"; } 24 | function log_green() { echo -e "${GREEN}$@${NC}"; } 25 | function log_red() { echo -e "${RED}$@${NC}"; } 26 | 27 | [[ -z "${ISTIOCTL}" ]] && log_red "Missing ISTIOCTL env var." && exit 1 28 | 29 | log_cyan "Installing Istio control plane..." 30 | ${ISTIOCTL} install -f base/istio/istiooperator-1.7.yaml \ 31 | --skip-confirmation \ 32 | --readiness-timeout 5m0s $@ 33 | kubectl label ns istio-system install.operator.istio.io/owner-kind=IstioControlPlane --overwrite=true 34 | log_cyan "Istio control plane is ready" 35 | -------------------------------------------------------------------------------- /setup/manifests/wait_for_clusters.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | [[ $# -lt 2 ]] && echo "USAGE: $0 ..." && exit 1 20 | 21 | INFRA_NAME=$1 22 | shift 23 | REGIONS=$@ 24 | 25 | # Wait for all clusters to become RUNNING 26 | for REGION in $REGIONS; do 27 | echo "Waiting for cluster in '${REGION}'..." 28 | until [[ $(gcloud container clusters describe ${INFRA_NAME}-${REGION} --region ${REGION} --format="value(status)" 2>/dev/null) == "RUNNING" ]]; do 29 | sleep 2 30 | done 31 | echo "Cluster in '${REGION}' is ready." 32 | done 33 | 34 | echo "Done. All clusters ready." -------------------------------------------------------------------------------- /setup/scripts/add_iap_user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | [[ -z "$1" || -z "$2" || -z "$3" ]] && echo "USAGE: $0 " && exit 1 20 | 21 | [[ ! "$1" =~ user|group|domain|serviceAccount ]] && echo "ERROR: invalid member type '$1', must be one of user|group|domain|serviceAccount" && exit 1 22 | 23 | MEMBER_TYPE=${1} 24 | MEMBER=$2 25 | PROJECT=$3 26 | 27 | SCRIPT_DIR=$(dirname $(readlink -f $0 2>/dev/null) 2>/dev/null || echo "${PWD}/$(dirname $0)") 28 | 29 | # Get project from terraform outputs 30 | PROJECT=${GOOGLE_CLOUD_PROJECT:-$PROJECT}; 31 | 32 | TMPDIR=$(mktemp -d) 33 | CLOUDBUILD="${TMPDIR}/cloudbuild.yaml" 34 | 35 | cat - > $CLOUDBUILD <<'EOF' 36 | timeout: 300s 37 | substitutions: 38 | _MEMBER_TYPE: 39 | _MEMBER: 40 | tags: 41 | - add-iap-user 42 | steps: 43 | - name: "gcr.io/cloud-builders/gcloud" 44 | id: "add-iap-user" 45 | entrypoint: "bash" 46 | args: 47 | - "-exc" 48 | - | 49 | curl -sfL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/bin/jq 50 | chmod +x /usr/bin/jq 51 | 52 | echo "${_MEMBER_TYPE},${_MEMBER},${PROJECT_ID}" 53 | TMP=$(mktemp -p /tmp -t policy.json.XXXXXXX) 54 | gcloud projects get-iam-policy ${PROJECT_ID} --format=json > $${TMP} 55 | 56 | if [[ -z "$(jq '.bindings[] | select(.role=="roles/iap.httpsResourceAccessor")' $${TMP})" ]]; then 57 | # Create new binding. 58 | echo "INFO: Adding IAM policy binding" 59 | gcloud projects add-iam-policy-binding ${PROJECT_ID} \ 60 | --member="${_MEMBER_TYPE}:${_MEMBER}" \ 61 | --role='roles/iap.httpsResourceAccessor' >/dev/null 62 | else 63 | # Append to existing binding. 64 | echo "INFO: Updating IAM policy binding" 65 | gcloud projects get-iam-policy ${PROJECT_ID} --format=json | \ 66 | jq '(.bindings[] | select(.role=="roles/iap.httpsResourceAccessor").members) += ["'${_MEMBER_TYPE}':'${_MEMBER}'"]' > $${TMP} 67 | gcloud projects set-iam-policy --format=json ${PROJECT_ID} $${TMP} > /dev/null 68 | fi 69 | rm -f $${TMP} 70 | EOF 71 | 72 | gcloud builds submit --config $CLOUDBUILD --project ${PROJECT} --substitutions=_MEMBER_TYPE="${MEMBER_TYPE?}",_MEMBER="${MEMBER?}" -------------------------------------------------------------------------------- /setup/scripts/create_oauth_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Font colors 18 | export RED='\033[1;31m' 19 | export CYAN='\033[1;36m' 20 | export GREEN='\033[1;32m' 21 | export NC='\033[0m' # No Color 22 | 23 | export GCLOUD=${GCLOUD:-"gcloud -q"} 24 | 25 | function log_cyan() { 26 | echo -e "${CYAN}$@${NC}" >&2 27 | } 28 | 29 | APP_NAME=$1 30 | 31 | [[ -z "${APP_NAME}" ]] && echo "USAGE: $0 " && exit 1 32 | 33 | # Ensure that the API is enabled. 34 | $GCLOUD services enable iap.googleapis.com >/dev/null 35 | 36 | # Helper function that retries failed alpha commands up to 5 times. 37 | function gcloud_alpha() { 38 | local count=0 39 | while [[ $count -lt 5 ]]; do 40 | gcloud -q alpha "$@" 41 | [[ $? -eq 0 ]] && return 0 42 | ((count=count+1)) 43 | sleep 2 44 | done 45 | return 1 46 | } 47 | 48 | # Check to see if OAuth brand already exists 49 | log_cyan "INFO: Creating OAuth Brand" 50 | BRAND_ID=$(gcloud_alpha iap oauth-brands list --format='value(name)' --limit=1) 51 | if [[ -z "${BRAND_ID}" ]]; then 52 | # Create the OAuth Brand 53 | BRAND_ID=$(gcloud_alpha iap oauth-brands create --application_title="${APP_NAME?}" --support_email=$($GCLOUD config get-value account) --format='value(name)') 54 | [[ $? -ne 0 ]] && echo "ERROR: Failed to create oauth brand." && exit 1 55 | sleep 2 56 | else 57 | log_cyan "INFO: Using existing brand: ${BRAND_ID}" 58 | fi 59 | 60 | # Check to see if OAuth client already exists. 61 | CLIENT_ID= 62 | CLIENT_SECRET= 63 | IFS=',' read -ra toks < <(gcloud_alpha iap oauth-clients list ${BRAND_ID?} --filter displayName~"${APP_NAME?}" --limit=1 --format 'csv[no-heading](name,secret)') 64 | if [[ ${#toks[@]} -eq 0 ]]; then 65 | log_cyan "INFO: Creating OAuth client" 66 | IFS=',' read -ra toks < <(gcloud_alpha iap oauth-clients create ${BRAND_ID?} --display_name="${APP_NAME?}" --format 'csv[no-heading](name,secret)') 67 | [[ $? -ne 0 ]] && echo "ERROR: Failed to create oauth client." && exit 1 68 | CLIENT_ID=$(basename ${toks[0]}) 69 | CLIENT_SECRET=${toks[1]} 70 | else 71 | log_cyan "INFO: Using existing client: ${toks[0]}" 72 | CLIENT_ID=$(basename ${toks[0]}) 73 | CLIENT_SECRET=${toks[1]} 74 | fi 75 | 76 | [[ -z "${CLIENT_ID}" || -z "${CLIENT_SECRET}" ]] && echo "ERROR: Failed to get oauth client" && exit 1 77 | 78 | echo "export CLIENT_ID=${CLIENT_ID}" 79 | echo "export CLIENT_SECRET=${CLIENT_SECRET}" 80 | 81 | log_cyan "INFO: exported CLIENT_ID" 82 | log_cyan "INFO: exported CLIENT_SECRET" 83 | log_cyan "INFO: Done" 84 | -------------------------------------------------------------------------------- /setup/scripts/fix_autoneg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | echo "INFO: Removing neg-status annotation on istio-ingressgateway service" 18 | kubectl annotate service istio-ingressgateway -n istio-system cloud.google.com/neg-status- 19 | 20 | echo "INFO: Removing autoneg-status annotation on istio-ingressgateway service" 21 | kubectl annotate service istio-ingressgateway -n istio-system anthos.cft.dev/autoneg-status- 22 | 23 | echo "Done. Annotations should be auto-added by the controllers." -------------------------------------------------------------------------------- /setup/scripts/gcping.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # config file from gcping containing region map. 18 | GCPING_SRC_URL="https://raw.githubusercontent.com/ImJasonH/gcping/master/config.js" 19 | 20 | function usage() { 21 | echo "USAGE: $0 " >&2 22 | } 23 | 24 | function parseRegionMap() { 25 | src_url=$1 26 | declare -n region_map="$2" 27 | 28 | data=$(curl -sf ${src_url}) || return 1 29 | 30 | while read -r line; do 31 | [[ ! "${line}" =~ .*http.* || "${line}" =~ global ]] && continue 32 | IFS=' ' read -ra toks <<< "$line" 33 | region=${toks[0]//[\":]/} 34 | region_url=${toks[1]//[\",]/} 35 | region_map[${region}]=${region_url} 36 | done <<< "$data" 37 | } 38 | 39 | function gcping() { 40 | declare -n region_map="$1" 41 | declare -n rtt_map="$2" 42 | 43 | for region in "${!region_map[@]}"; do 44 | url=${region_map[$region]} 45 | rtt_ms=30000 46 | rtt=$(curl --connect-timeout 2 -sf -o /dev/null -w "%{time_starttransfer}\n" ${url}) 47 | [[ $? -eq 0 ]] && rtt_ms=$(awk -vp=${rtt} 'BEGIN{printf "%d" ,p * 1000}') 48 | rtt_map[$region]=${rtt_ms} 49 | echo "PING ${region}: ${rtt_ms}ms" >&2 50 | done 51 | } 52 | 53 | function findNearestRegion() { 54 | declare -n rtt_map="$1" 55 | 56 | local nearest_rtt=100000 57 | local nearest_region="" 58 | 59 | for region in "${!rtt_map[@]}"; do 60 | rtt=${rtt_map[$region]} 61 | if [[ "${rtt}" -lt "${nearest_rtt}" ]]; then 62 | nearest_rtt=${rtt} 63 | nearest_region=$region 64 | fi 65 | done 66 | echo "${nearest_region}" 67 | } 68 | 69 | function findFurthestRegion() { 70 | declare -n rtt_map="$1" 71 | 72 | local furthest_rtt=0 73 | local furthest_region="" 74 | 75 | for region in "${!rtt_map[@]}"; do 76 | rtt=${rtt_map[$region]} 77 | if [[ "${rtt}" -gt "${furthest_rtt}" ]]; then 78 | furthest_rtt=${rtt} 79 | furthest_region=$region 80 | fi 81 | done 82 | echo "${furthest_region}" 83 | } 84 | 85 | if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then 86 | [[ -z "${1}" ]] && usage && exit 1 87 | 88 | declare -A REGION_MAP 89 | if ! parseRegionMap $GCPING_SRC_URL REGION_MAP; then 90 | echo "ERROR: Failed to parse region map from: ${GCPING_SRC_URL}" >&2 91 | exit 1 92 | fi 93 | 94 | declare -A RTT_MAP 95 | if ! gcping REGION_MAP RTT_MAP; then 96 | echo "ERROR: Failed to ping regions" >&2 97 | exit 1 98 | fi 99 | 100 | case ${1,,} in 101 | "nearest") findNearestRegion RTT_MAP ;; 102 | "furthest") findFurthestRegion RTT_MAP ;; 103 | *) usage && exit 1 ;; 104 | esac 105 | fi 106 | 107 | -------------------------------------------------------------------------------- /setup/scripts/pod-broker-gitops-init.sh: -------------------------------------------------------------------------------- 1 | gitops-init.sh -------------------------------------------------------------------------------- /setup/scripts/stream_logs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | PROJECT=$1 20 | shift || true 21 | 22 | [[ -z "${PROJECT}" ]] && echo "USAGE: $0 " && exit 1 23 | 24 | SCRIPT_DIR=$(dirname $(readlink -f $0 2>/dev/null) 2>/dev/null || echo "${PWD}/$(dirname $0)") 25 | 26 | BUILD_ID=$(gcloud builds list --project ${PROJECT} --sort-by=startTime --limit=1 --format='value(id)' | cut -f1) 27 | 28 | gcloud builds log --stream --project ${PROJECT} ${BUILD_ID} $@ -------------------------------------------------------------------------------- /setup/scripts/wait_for_iap.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | SA_EMAIL=$1 18 | CLIENT_ID=$2 19 | ENDPOINT=$3 20 | 21 | [[ -z "${SA_EMAIL}" || -z "${CLIENT_ID}" || -z "${ENDPOINT}" ]] && echo "USAGE: $0 " && exit 1 22 | 23 | TOKEN=$(curl -f -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token" | jq -r '.access_token') 24 | 25 | ID_TOKEN=$(curl -f -s -H "Authorization: Bearer ${TOKEN?}" --header 'Content-Type: application/json' -d '{ "audience": "'${CLIENT_ID?}'", "includeEmail": "true" }' "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${SA_EMAIL?}:generateIdToken" | jq -r '.token') 26 | 27 | [[ -z "${ID_TOKEN}" ]] && echo "ERROR: Failed to obtain ID token" && exit 1 28 | 29 | echo "INFO: Waiting for: ${ENDPOINT}" 30 | count=0 31 | while [[ "${count}" -le 5 ]]; do 32 | STATUS=$(curl --connect-timeout 1 -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${ID_TOKEN?}" "${ENDPOINT?}") 33 | [[ "$STATUS" -eq 200 ]] && ((count=count+1)) 34 | [[ "$STATUS" -eq 403 ]] && echo "ERROR: failed to get valid IAP credentials." && exit 1 35 | [[ "$STATUS" -eq 302 ]] && echo "ERROR: unexpected 302 from IAP endpoint, possible invalid ID token." && exit 1 36 | sleep 2 37 | done 38 | echo "INFO: Done" -------------------------------------------------------------------------------- /setup/scripts/workload_identity_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | NAMESPACE=$1 20 | SERVICE_ACCOUNT=$2 21 | 22 | [[ $# -lt 2 ]] && echo "USAGE: $0 ..." && exit 1 23 | 24 | # From: https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity 25 | kubectl run --rm -it \ 26 | --generator=run-pod/v1 \ 27 | --image google/cloud-sdk:slim \ 28 | --namespace ${NAMESPACE} \ 29 | --serviceaccount ${SERVICE_ACCOUNT} \ 30 | workload-identity-test 31 | --------------------------------------------------------------------------------