├── .gitignore ├── CONTRIBUTING.md ├── Jenkinsfile ├── LICENSE ├── Makefile ├── OWNERS ├── README-QWIKLABS.md ├── README.md ├── containers ├── app.go ├── cloudbuild.yaml ├── root.txt ├── root_Dockerfile ├── user.txt └── user_Dockerfile ├── enable-apis.sh ├── generate-tfvars.sh ├── img └── terraform_fingerprint_error.png ├── renovate.json ├── terraform ├── iam.tf ├── main.tf ├── modules │ ├── firewall │ │ ├── firewall.tf │ │ └── variables.tf │ ├── instance │ │ ├── main.tf │ │ ├── manifests │ │ │ ├── apparmor-configmap.yaml │ │ │ ├── apparmor-loader-ds.yaml │ │ │ ├── apparmor-namespace.yaml │ │ │ ├── armored-run-as-user-denied.yaml │ │ │ ├── armored-run-as-user.yaml │ │ │ ├── override-root-with-user.yaml │ │ │ ├── run-as-root.yaml │ │ │ └── run-as-user.yaml │ │ ├── outputs.tf │ │ ├── scripts │ │ │ ├── deploy.sh │ │ │ ├── teardown.sh │ │ │ ├── utils.sh │ │ │ └── validate.sh │ │ └── variables.tf │ └── network │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── variables.tf ├── provider.tf ├── variables.tf └── versions.tf ├── test ├── boilerplate │ ├── boilerplate.BUILD.txt │ ├── boilerplate.Dockerfile.txt │ ├── boilerplate.Makefile.txt │ ├── boilerplate.WORKSPACE.txt │ ├── boilerplate.bazel.txt │ ├── boilerplate.bzl.txt │ ├── boilerplate.css.txt │ ├── boilerplate.go.preamble │ ├── boilerplate.go.txt │ ├── boilerplate.html.preamble │ ├── boilerplate.html.txt │ ├── boilerplate.java.txt │ ├── boilerplate.js.txt │ ├── boilerplate.py.preamble │ ├── boilerplate.py.txt │ ├── boilerplate.scss.txt │ ├── boilerplate.sh.preamble │ ├── boilerplate.sh.txt │ ├── boilerplate.tf.txt │ ├── boilerplate.ts.txt │ ├── boilerplate.xml.preamble │ ├── boilerplate.xml.txt │ └── boilerplate.yaml.txt ├── make.sh └── verify_boilerplate.py └── waitfor_svc_deleted.sh /.gitignore: -------------------------------------------------------------------------------- 1 | containers/app 2 | 3 | # OSX leaves these everywhere on SMB shares 4 | ._* 5 | 6 | # OSX trash 7 | .DS_Store 8 | 9 | # Emacs save files 10 | *~ 11 | \#*\# 12 | .\#* 13 | 14 | # Vim-related files 15 | [._]*.s[a-w][a-z] 16 | [._]s[a-w][a-z] 17 | *.un~ 18 | Session.vim 19 | .netrwhist 20 | 21 | # Local .terraform directories 22 | **/.terraform/* 23 | 24 | # .tfstate files 25 | *.tfstate 26 | *.tfstate.* 27 | 28 | # Ignore any .tfvars files that are generated automatically for each Terraform 29 | # run. Most .tfvars files are not managed as part of configuration and so should 30 | # not be included in version control. 31 | terraform.tfvars 32 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | Contributions to this project must be accompanied by a Contributor License 8 | Agreement. You (or your employer) retain the copyright to your contribution; 9 | this simply gives us permission to use and redistribute your contributions as 10 | part of the project. Head over to https://cla.developers.google.com/ to see your 11 | current agreements on file or to sign a new one. 12 | 13 | You generally only need to submit a CLA once, so if you've already submitted one 14 | (even if it was for a different project), you probably don't need to do it again. 15 | 16 | ## Code reviews 17 | All submissions, including submissions by project members, require review. We 18 | use GitHub pull requests for this purpose. Consult GitHub Help for more 19 | information on using pull requests. 20 | 21 | ## Community Guidelines 22 | This project follows 23 | [Google's Open Source Community Guidelines](CODE-OF-CONDUCT.md). 24 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env groovy 2 | /* 3 | Copyright 2018 Google LLC 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | https://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | // Reference: https://github.com/jenkinsci/kubernetes-plugin 19 | // set up pod label and GOOGLE_APPLICATION_CREDENTIALS (for Terraform) 20 | def label = "k8s-infra" 21 | def containerName = "k8s-node" 22 | def GOOGLE_APPLICATION_CREDENTIALS = '/home/jenkins/dev/jenkins-deploy-dev-infra.json' 23 | def jenkins_container_version = env.JENKINS_CONTAINER_VERSION 24 | 25 | podTemplate(label: label, 26 | containers: [ 27 | containerTemplate(name: "${containerName}", 28 | image: "gcr.io/pso-helmsman-cicd/jenkins-k8s-node:${jenkins_container_version}", 29 | command: 'tail -f /dev/null', 30 | resourceRequestCpu: '1000m', 31 | resourceLimitCpu: '2000m', 32 | resourceRequestMemory: '1Gi', 33 | resourceLimitMemory: '2Gi' 34 | ) 35 | ], 36 | volumes: [secretVolume(mountPath: '/home/jenkins/dev', 37 | secretName: 'jenkins-deploy-dev-infra' 38 | )] 39 | ) { 40 | node(label) { 41 | try { 42 | // set env variable GOOGLE_APPLICATION_CREDENTIALS for Terraform 43 | env.GOOGLE_APPLICATION_CREDENTIALS = GOOGLE_APPLICATION_CREDENTIALS 44 | 45 | stage('Setup') { 46 | container(containerName) { 47 | // checkout code from scm i.e. commits related to the PR 48 | checkout scm 49 | 50 | // Setup gcloud service account access 51 | sh "gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}" 52 | sh "gcloud config set compute/zone ${env.CLUSTER_ZONE}" 53 | sh "gcloud config set core/project ${env.PROJECT_ID}" 54 | sh "gcloud config set compute/region ${env.REGION}" 55 | } 56 | } 57 | stage('Lint') { 58 | container(containerName) { 59 | sh "make lint" 60 | } 61 | } 62 | stage('Create') { 63 | container(containerName) { 64 | sh 'make setup-project' 65 | sh 'make tf-apply' 66 | sh 'gcloud compute scp --recurse terraform/modules/instance/manifests jenkins-deploy-dev-infra@gke-application-security-bastion:' 67 | sh 'gcloud compute scp --recurse terraform/modules/instance/scripts jenkins-deploy-dev-infra@gke-application-security-bastion:' 68 | sh 'gcloud compute ssh jenkins-deploy-dev-infra@gke-application-security-bastion --command \'source /etc/profile\'' 69 | sh 'gcloud compute ssh jenkins-deploy-dev-infra@gke-application-security-bastion --command \'scripts/deploy.sh\'' 70 | } 71 | } 72 | stage('Validate') { 73 | container(containerName) { 74 | sh 'gcloud compute ssh jenkins-deploy-dev-infra@gke-application-security-bastion --command \'scripts/validate.sh\'' 75 | } 76 | } 77 | } catch (err) { 78 | // if any exception occurs, mark the build as failed 79 | // and display a detailed message on the Jenkins console output 80 | currentBuild.result = 'FAILURE' 81 | echo "FAILURE caught echo ${err}" 82 | throw err 83 | } finally { 84 | stage('Teardown') { 85 | container(containerName) { 86 | sh 'gcloud compute ssh jenkins-deploy-dev-infra@gke-application-security-bastion --command \'scripts/teardown.sh\'' 87 | sh "make tf-destroy" 88 | } 89 | } 90 | } 91 | } 92 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Make will use bash instead of sh 16 | SHELL := /usr/bin/env bash 17 | 18 | # All is the first target in the file so it will get picked up when you just run 'make' on its own 19 | lint: check_shell check_python check_golang check_terraform check_docker check_base_files check_headers check_trailing_whitespace 20 | 21 | # The .PHONY directive tells make that this isn't a real target and so 22 | # the presence of a file named 'check_shell' won't cause this target to stop 23 | # working 24 | .PHONY: check_shell 25 | check_shell: 26 | @source test/make.sh && check_shell 27 | 28 | .PHONY: check_python 29 | check_python: 30 | @source test/make.sh && check_python 31 | 32 | .PHONY: check_golang 33 | check_golang: 34 | @source test/make.sh && golang 35 | 36 | .PHONY: check_terraform 37 | check_terraform: 38 | @source test/make.sh && check_terraform 39 | 40 | .PHONY: check_docker 41 | check_docker: 42 | @source test/make.sh && docker 43 | 44 | .PHONY: check_base_files 45 | check_base_files: 46 | @source test/make.sh && basefiles 47 | 48 | .PHONY: check_shebangs 49 | check_shebangs: 50 | @source test/make.sh && check_bash 51 | 52 | .PHONY: check_trailing_whitespace 53 | check_trailing_whitespace: 54 | @source test/make.sh && check_trailing_whitespace 55 | 56 | .PHONY: check_headers 57 | check_headers: 58 | @echo "Checking file headers" 59 | @python3.7 test/verify_boilerplate.py 60 | 61 | .PHONY: build_app 62 | build_app: 63 | docker run --rm --mount type=bind,source="$$(pwd)"/containers,target=/gosrc -w /gosrc golang:1.10.3 go build -v -o app 64 | 65 | .PHONY: build_root_image 66 | build_root_image: 67 | cd containers && docker build -t hello-run-as-root:1.0.0 -f root_Dockerfile . 68 | 69 | .PHONY: build_user_image 70 | build_user_image: 71 | cd containers && docker build -t hello-run-as-user:1.0.0 -f user_Dockerfile . 72 | 73 | .PHONY: push_root_image 74 | push_root_image: 75 | docker tag hello-run-as-root:1.0.0 gcr.io/"$$(gcloud config get-value project)"/hello-run-as-root:1.0.0 76 | docker push gcr.io/"$$(gcloud config get-value project)"/hello-run-as-root:1.0.0 77 | 78 | .PHONY: push_user_image 79 | push_user_image: 80 | docker tag hello-run-as-user:1.0.0 gcr.io/"$$(gcloud config get-value project)"/hello-run-as-user:1.0.0 81 | docker push gcr.io/"$$(gcloud config get-value project)"/hello-run-as-user:1.0.0 82 | 83 | 84 | .PHONY: setup-project 85 | setup-project: 86 | # Enables the Google Cloud APIs needed 87 | ./enable-apis.sh 88 | # Runs generate-tfvars.sh 89 | ./generate-tfvars.sh 90 | 91 | .PHONY: tf-apply 92 | tf-apply: 93 | # Downloads the terraform providers and applies the configuration 94 | cd terraform && terraform init && terraform apply -auto-approve 95 | 96 | .PHONY: tf-destroy 97 | tf-destroy: 98 | # Wait for the related backend services to be deleted and then 99 | # delete all the resources created by terraform 100 | ./waitfor_svc_deleted.sh && cd terraform && terraform destroy -auto-approve 101 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - chrislovecnm 3 | - robinpercy 4 | - geojaz 5 | - techgnosis 6 | - erkolson 7 | labels: 8 | - gke-helmsman 9 | -------------------------------------------------------------------------------- /README-QWIKLABS.md: -------------------------------------------------------------------------------- 1 | # Best Practices for Securing Containerized Applications in Kubernetes Engine 2 | 3 | ## Table of Contents 4 | 5 | 6 | 7 | * [Introduction](#introduction) 8 | * [Architecture](#architecture) 9 | * [Containers](#containers) 10 | * [AppArmor](#apparmor) 11 | * [The container configurations](#the-container-configurations) 12 | * [Initial Setup](#initial-setup) 13 | * [Configure gcloud](#configure-gcloud) 14 | * [Tools](#tools) 15 | * [Deployment](#deployment) 16 | * [Authenticate gcloud](#authenticate-gcloud) 17 | * [Configure gcloud settings](#configure-gcloud-settings) 18 | * [Setup this project](#setup-this-project) 19 | * [Provisioning the Kubernetes Engine Cluster](#provisioning-the-kubernetes-engine-cluster) 20 | * [Validation](#validation) 21 | * [Tear Down](#tear-down) 22 | * [Troubleshooting](#troubleshooting) 23 | * [Relevant Material](#relevant-material) 24 | 25 | 26 | 27 | ## Introduction 28 | 29 | This guide demonstrates a series of best practices that will allow the user to improve the security of their containerized applications deployed to Kubernetes Engine. 30 | 31 | The [principle of least privilege]((https://en.wikipedia.org/wiki/Principle_of_least_privilege)) 32 | is widely recognized as an important design consideration in enhancing the protection of critical systems from faults and malicious behavior. It suggests that every component must be able to access **only** the information and resources that are necessary for its legitimate purpose. This guide will go about showing the user how to improve a container's security by providing a systematic approach to effectively remove unnecessary privileges. 33 | 34 | ## Architecture 35 | 36 | ### Containers 37 | 38 | At their core, containers help make implementing security best practices easier by providing the user with an easy interface to run processes in a chroot environment as an unprivileged user and removing all but the kernel [capabilities](http://man7.org/linux/man-pages/man7/capabilities.7.html) needed to run the application. By default, all containers are run in the root user namespace so running containers as a non-root user is important. 39 | 40 | ### AppArmor 41 | 42 | On occasion, an application will need to access a kernel resource that requires special privileges normally granted only to the root user. However, running the application as a user with root privileges is a bad solution as it provides the application with access to the entire system. Instead, the kernel provides a set of capabilities that can be granted to a process to allow it coarse-grained access to only the kernel resources it needs and nothing more. 43 | 44 | Using kernel modules such as AppArmor, Kubernetes provides an easy interface to both run the containerized application as a non-root user in the process namespace and restrict the set of capabilities granted to the process. 45 | 46 | ### The container configurations 47 | 48 | This demonstration will deploy five containers in a private cluster: 49 | 50 | 1. A container run as the root user in the container in the Dockerfile 51 | 1. A container run as a user created in the container in the Dockerfile 52 | 1. A container that Kubernetes started as a non-root user despite the Dockerfile not specifying it be run as a non-root user 53 | 1. A container with a lenient AppArmor profile that allows all non-root permissions. 54 | 1. A container with an AppArmor profile applied to disallow the `/proc/cpuinfo` endpoint from being properly read 55 | 56 | Each container will be exposed outside the clusters as an internal load balancer. 57 | 58 | The containers themselves are running a simple Go web server with five endpoints. The endpoints differ in terms of the privileges they need to complete the request. A non-root user cannot read a file owned by root. The `nobody` user cannot read `/proc/cpuinfo` when that privilege is being blocked by AppArmor. 59 | 60 | 1. An endpoint to get the container's hostname 61 | 1. An endpoint to get the username, UID, and GID of identity running the server 62 | 1. An endpoint to read a file owned by the `root` user 63 | 1. An endpoint to read a file owned by the `nobody` user 64 | 1. An endpoint to read the `/proc/cpuinfo` file 65 | 66 | ## Initial Setup 67 | 68 | ### Configure gcloud 69 | 70 | All the tools for the demo are installed. When using Cloud Shell execute the following 71 | command in order to setup gcloud cli. When executing this command please setup your region and zone. 72 | 73 | ```console 74 | gcloud init 75 | ``` 76 | 77 | ### Tools 78 | 1. [Terraform >= 0.11.7](https://www.terraform.io/downloads.html) 79 | 2. [Google Cloud SDK version >= 204.0.0](https://cloud.google.com/sdk/docs/downloads-versioned-archives) 80 | 3. [kubectl matching the latest GKE version](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 81 | 4. bash or bash compatible shell 82 | 5. [GNU Make 3.x or later](https://www.gnu.org/software/make/) 83 | 6. A Google Cloud Platform project where you have permission to create 84 | networks 85 | 86 | #### Install Cloud SDK 87 | The Google Cloud SDK is used to interact with your GCP resources. 88 | [Installation instructions](https://cloud.google.com/sdk/downloads) for multiple platforms are available online. 89 | 90 | #### Install kubectl CLI 91 | 92 | The kubectl CLI is used to interteract with both Kubernetes Engine and kubernetes in general. 93 | [Installation instructions](https://cloud.google.com/kubernetes-engine/docs/quickstart) 94 | for multiple platforms are available online. 95 | 96 | #### Install Terraform 97 | 98 | Terraform is used to automate the manipulation of cloud infrastructure. Its 99 | [installation instructions](https://www.terraform.io/intro/getting-started/install.html) are also available online. 100 | 101 | ## Deployment 102 | 103 | The steps below will walk you through using terraform to deploy a Kubernetes Engine cluster that you will then use for exploring multiple types of container security configurations. 104 | 105 | ### Authenticate gcloud 106 | 107 | Prior to running this demo, ensure you have authenticated your gcloud client by running the following command: 108 | 109 | ```console 110 | gcloud auth application-default login 111 | ``` 112 | 113 | ### Configure gcloud settings 114 | 115 | Run `gcloud config list` and make sure that `compute/zone`, `compute/region` and `core/project` are populated with values that work for you. You can set their values with the following commands: 116 | 117 | ```console 118 | # Where the region is us-east1 119 | gcloud config set compute/region us-east1 120 | 121 | Updated property [compute/region]. 122 | ``` 123 | 124 | ```console 125 | # Where the zone inside the region is us-east1-c 126 | gcloud config set compute/zone us-east1-c 127 | 128 | Updated property [compute/zone]. 129 | ``` 130 | 131 | ```console 132 | # Where the project name is my-project-name 133 | gcloud config set project my-project-name 134 | 135 | Updated property [core/project]. 136 | ``` 137 | 138 | ### Setup this project 139 | 140 | This project requires the following Google Cloud Service APIs to be enabled: 141 | 142 | * `compute.googleapis.com` 143 | * `container.googleapis.com` 144 | * `cloudbuild.googleapis.com` 145 | 146 | In addition, the terraform configuration takes three parameters to determine where the Kubernetes Engine cluster should be created: 147 | 148 | * `project` 149 | * `region` 150 | * `zone` 151 | 152 | For simplicity, these parameters are to be specified in a file named `terraform.tfvars`, in the `terraform` directory. To ensure the appropriate APIs are enabled and to generate the `terraform/terraform.tfvars` file based on your gcloud defaults, run: 153 | 154 | ```console 155 | make setup-project 156 | ``` 157 | 158 | This will enable the necessary Service APIs, and it will also generate a `terraform/terraform.tfvars` file with the following keys. The values themselves will match the output of `gcloud config list`: 159 | 160 | ```console 161 | $ cat terraform/terraform.tfvars 162 | 163 | project="YOUR_PROJECT" 164 | region="YOUR_REGION" 165 | zone="YOUR_ZONE" 166 | ``` 167 | 168 | If you need to override any of the defaults, simply replace the desired value(s) to the right of the equals sign(s). Be sure your replacement values are still double-quoted. 169 | 170 | ### Provisioning the Kubernetes Engine Cluster 171 | 172 | Next, apply the terraform configuration with: 173 | 174 | ```console 175 | # From within the project root, use make to apply the terraform 176 | make tf-apply 177 | ``` 178 | 179 | This will take a few minutes to complete. The following is the last few lines of successful output. 180 | 181 | ```console 182 | ...snip... 183 | google_container_cluster.primary: Still creating... (2m20s elapsed) 184 | google_container_cluster.primary: Still creating... (2m30s elapsed) 185 | google_container_cluster.primary: Still creating... (2m40s elapsed) 186 | google_container_cluster.primary: Still creating... (2m50s elapsed) 187 | google_container_cluster.primary: Still creating... (3m0s elapsed) 188 | google_container_cluster.primary: Still creating... (3m10s elapsed) 189 | google_container_cluster.primary: Still creating... (3m20s elapsed) 190 | google_container_cluster.primary: Still creating... (3m30s elapsed) 191 | google_container_cluster.primary: Still creating... (3m40s elapsed) 192 | google_container_cluster.primary: Creation complete after 3m44s (ID: gke-security-best-practices) 193 | 194 | Apply complete! Resources: 7 added, 0 changed, 0 destroyed. 195 | ``` 196 | 197 | Once that has completed, remote into the bastion instance using SSH: 198 | 199 | ```console 200 | gcloud compute ssh gke-application-security-bastion 201 | ``` 202 | 203 | Apply the manifests for the cluster using the deployment script: 204 | 205 | ```console 206 | ./scripts/deploy.sh 207 | ``` 208 | 209 | This will take a minute or two to complete. The final output should be similar to: 210 | 211 | ```console 212 | namespace/apparmor created 213 | configmap/apparmor-profiles created 214 | daemonset.apps/apparmor-loader created 215 | deployment.apps/armored-hello-user created 216 | service/armored-hello-user created 217 | deployment.apps/armored-hello-denied created 218 | service/armored-hello-denied created 219 | deployment.apps/hello-override created 220 | service/hello-override created 221 | deployment.apps/hello-root created 222 | service/hello-root created 223 | deployment.apps/hello-user created 224 | service/hello-user created 225 | 226 | ...snip... 227 | 228 | Service hello-root has not allocated an IP yet. 229 | Service hello-root has not allocated an IP yet. 230 | Service hello-root IP has been allocated 231 | Service hello-user has not allocated an IP yet. 232 | Service hello-user has not allocated an IP yet. 233 | Service hello-user has not allocated an IP yet. 234 | Service hello-user has not allocated an IP yet. 235 | Service hello-user IP has been allocated 236 | Service hello-override IP has been allocated 237 | Service armored-hello-user IP has been allocated 238 | Service armored-hello-denied IP has been allocated 239 | ``` 240 | 241 | At this point, the environment should be completely set up. 242 | 243 | ## Validation 244 | 245 | To test all of the services in one command, run the validation script from the scripts directory of the bastion host: 246 | 247 | ```console 248 | ./scripts/validate.sh 249 | ``` 250 | 251 | This script queries each of the services to get: 252 | 253 | * the hostname of the pod being queried 254 | * the username, UID, and GID of the process the pod's web server is running as 255 | * the contents of a file owned by root 256 | * the contents of a file owned by a non-root user 257 | * the first 5 lines of content from `/proc/cpuinfo` 258 | 259 | The first service, `hello-root`, has an output similar to: 260 | 261 | ```console 262 | Querying service running natively as root 263 | You are querying host hello-root-54fdf49bf7-8bjmm 264 | User: root 265 | UID: 0 266 | GID: 0 267 | You have read the root.txt file. 268 | You have read the user.txt file. 269 | processor : 0 270 | vendor_id : GenuineIntel 271 | cpu family : 6 272 | model : 63 273 | model name : Intel(R) Xeon(R) CPU @ 2.30GHz 274 | ``` 275 | 276 | and it clearly shows that it is running as `root` and can perform all actions. 277 | 278 | The third service, `hello-user`, has an output similar to: 279 | 280 | ```console 281 | Querying service containers running natively as user 282 | You are querying host hello-user-76957b5645-hvfw2 283 | User: nobody 284 | UID: 65534 285 | GID: 65534 286 | unable to open root.txt: open root.txt: permission denied 287 | You have read the user.txt file. 288 | % Total % Received % Xferd Average Speed Time Time Time Current 289 | Dload Upload Total Spent Left Speed 290 | 100 804 100 804 0 0 156k 0 --:--:-- --:--:-- --:--:-- 196k 291 | processor : 0 292 | vendor_id : GenuineIntel 293 | cpu family : 6 294 | model : 45 295 | model name : Intel(R) Xeon(R) CPU @ 2.60GHz 296 | ``` 297 | 298 | which shows that it is running as `nobody` (65534) and therefore can read user.txt but not root.txt. 299 | 300 | The third service, `hello-override`, has an output similar to: 301 | 302 | ```console 303 | Querying service containers normally running as root but overridden by Kubernetes 304 | You are querying host hello-override-7c6c4b6c4-szmrh 305 | User: nobody 306 | UID: 65534 307 | GID: 65534 308 | unable to open root.txt: open root.txt: permission denied 309 | You have read the user.txt file. 310 | % Total % Received % Xferd Average Speed Time Time Time Current 311 | Dload Upload Total Spent Left Speed 312 | 100 804 100 804 0 0 144k 0 --:--:-- --:--:-- --:--:-- 157k 313 | processor : 0 314 | vendor_id : GenuineIntel 315 | cpu family : 6 316 | model : 45 317 | model name : Intel(R) Xeon(R) CPU @ 2.60GHz 318 | ``` 319 | 320 | and it shows that the container is running as the `nobody` user of id `65534`. Therefore, it can again read the `user.txt` file and read from `/proc/cpuinfo`. 321 | 322 | The fourth service, `armored-hello-user`, has an output similar to: 323 | 324 | ```console 325 | Querying service containers with an AppArmor profile allowing reading /proc/cpuinfo 326 | You are querying host armored-hello-user-5645cd4496-qls6q 327 | User: nobody 328 | UID: 65534 329 | GID: 65534 330 | unable to open root.txt: open root.txt: permission denied 331 | You have read the user.txt file. 332 | % Total % Received % Xferd Average Speed Time Time Time Current 333 | Dload Upload Total Spent Left Speed 334 | 100 804 100 804 0 0 148k 0 --:--:-- --:--:-- --:--:-- 157k 335 | processor : 0 336 | vendor_id : GenuineIntel 337 | cpu family : 6 338 | model : 45 339 | model name : Intel(R) Xeon(R) CPU @ 2.60GHz 340 | ``` 341 | 342 | and it shows that the leniently armored container still has the default access of the 343 | `nobody` user. 344 | 345 | The fifth and final service, `armored-hello-denied`, has an output similar to: 346 | 347 | ```console 348 | Querying service containers with an AppArmor profile blocking the reading of /proc/cpuinfo 349 | You are querying host armored-hello-denied-6fccb988dd-sxhmz 350 | User: nobody 351 | UID: 65534 352 | GID: 65534 353 | unable to open root.txt: open root.txt: permission denied 354 | You have read the user.txt file. 355 | % Total % Received % Xferd Average Speed Time Time Time Current 356 | Dload Upload Total Spent Left Speed 357 | 100 63 100 63 0 0 12162 0 --:--:-- --:--:-- --:--:-- 12600 358 | unable to open root.txt: open /proc/cpuinfo: permission denied 359 | ``` 360 | 361 | and it shows that the container is prohibited by AppArmor policy from reading the `user.txt` and `/proc/cpuinfo`. 362 | 363 | ## Tear Down 364 | 365 | To tear down the environment, use : 366 | 367 | ```console 368 | ./scripts/teardown.sh 369 | ``` 370 | 371 | It's output should look like the following: 372 | 373 | ```console 374 | daemonset.apps "apparmor-loader" deleted 375 | configmap "apparmor-profiles" deleted 376 | namespace "apparmor" deleted 377 | deployment.apps "armored-hello-user" deleted 378 | service "armored-hello-user" deleted 379 | deployment.apps "armored-hello-denied" deleted 380 | service "armored-hello-denied" deleted 381 | deployment.apps "hello-override" deleted 382 | service "hello-override" deleted 383 | deployment.apps "hello-root" deleted 384 | service "hello-root" deleted 385 | deployment.apps "hello-user" deleted 386 | service "hello-user" deleted 387 | ``` 388 | 389 | After that script completes, log out of the bastion host and run the following to destroy the environment: 390 | 391 | ```console 392 | make tf-destroy 393 | ``` 394 | 395 | Terraform will destroy the environment and indicate when it has completed: 396 | 397 | ```console 398 | ...snip... 399 | module.network.google_compute_subnetwork.cluster-subnet: Destroying... (ID: us-east1/kube-net-subnet) 400 | google_service_account.admin: Destruction complete after 0s 401 | module.network.google_compute_subnetwork.cluster-subnet: Still destroying... (ID: us-east1/kube-net-subnet, 10s elapsed) 402 | module.network.google_compute_subnetwork.cluster-subnet: Still destroying... (ID: us-east1/kube-net-subnet, 20s elapsed) 403 | module.network.google_compute_subnetwork.cluster-subnet: Destruction complete after 25s 404 | module.network.google_compute_network.gke-network: Destroying... (ID: kube-net) 405 | module.network.google_compute_network.gke-network: Still destroying... (ID: kube-net, 10s elapsed) 406 | module.network.google_compute_network.gke-network: Still destroying... (ID: kube-net, 20s elapsed) 407 | module.network.google_compute_network.gke-network: Destruction complete after 25s 408 | 409 | Destroy complete! Resources: 7 destroyed. 410 | ``` 411 | 412 | ## Troubleshooting 413 | 414 | ### Terraform destroy does not finish cleanly. The error will look something like 415 | 416 | ```console 417 | Error: Error applying plan: 418 | 419 | 1 error(s) occurred: 420 | 421 | * module.network.google_compute_network.gke-network (destroy): 1 error(s) occurred: 422 | 423 | * google_compute_network.gke-network: The network resource 'projects/seymourd-sandbox/global/networks/kube-net' is already being used by 'projects/seymourd-sandbox/global/firewalls/k8s-29e43f3a2accf594-node-hc' 424 | 425 | 426 | Terraform does not automatically rollback in the face of errors. Instead, your Terraform state file has been partially updated with any resources that successfully completed. Please address the error above and apply again to incrementally change your infrastructure. 427 | 428 | ``` 429 | 430 | Solution: the cluster does not always cleanly remove all of the GCP resources associated with a service before the cluster is deleted. You will need to manually clean up the remaining resources using either the Cloud Console or gcloud. 431 | 432 | ### The install script fails with a `Permission denied` when running Terraform 433 | 434 | The credentials that Terraform is using do not provide the necessary permissions to create resources in the selected projects. Ensure that the account listed in `gcloud config list` has necessary permissions to create resources. If it does, regenerate the application default credentials using `gcloud auth application-default login`. 435 | 436 | ### Invalid fingerprint error during Terraform operations 437 | 438 | Terraform occasionally complains about an invalid fingerprint, when updating certain resources. If you see the error below, simply re-run the command. ![terraform fingerprint error](./img/terraform_fingerprint_error.png) 439 | 440 | ## Relevant Material 441 | 442 | * [Capabilities documentation](http://man7.org/linux/man-pages/man7/capabilities.7.html) 443 | * [Docker security documentation](https://docs.docker.com/engine/security/security/) 444 | * [AppArmor](https://wiki.ubuntu.com/AppArmor) 445 | * [Kubernetes Engine Release Notes](https://cloud.google.com/kubernetes-engine/release-notes) 446 | 447 | **This is not an officially supported Google product** 448 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Best Practices for Securing Containerized Applications in Kubernetes Engine 2 | 3 | ## Table of Contents 4 | 5 | 6 | 7 | * [Introduction](#introduction) 8 | * [Architecture](#architecture) 9 | * [Containers](#containers) 10 | * [AppArmor](#apparmor) 11 | * [The container configurations](#the-container-configurations) 12 | * [Prerequisites](#prerequisites) 13 | * [Tools](#tools) 14 | * [Versions](#versions) 15 | * [Deployment](#deployment) 16 | * [Authenticate gcloud](#authenticate-gcloud) 17 | * [Configure gcloud settings](#configuring-gcloud-settings) 18 | * [Setup this project](#setup-this-project) 19 | * [Provisioning the Kubernetes Engine Cluster](#provisioning-the-kubernetes-engine-cluster) 20 | * [Validation](#validation) 21 | * [Tear Down](#tear-down) 22 | * [Troubleshooting](#troubleshooting) 23 | * [Relevant Material](#relevant-material) 24 | 25 | 26 | 27 | ## Introduction 28 | 29 | This guide demonstrates a series of best practices that will allow the user to improve the security of their containerized applications deployed to Kubernetes Engine. 30 | 31 | The [principle of least privilege]((https://en.wikipedia.org/wiki/Principle_of_least_privilege)) 32 | is widely recognized as an important design consideration in enhancing the protection of critical systems from faults and malicious behavior. It suggests that every component must be able to access **only** the information and resources that are necessary for its legitimate purpose. This guide will go about showing the user how to improve a container's security by providing a systematic approach to effectively remove unnecessary privileges. 33 | 34 | ## Architecture 35 | 36 | ### Containers 37 | 38 | At their core, containers help make implementing security best practices easier by providing the user with an easy interface to run processes in a chroot environment as an unprivileged user and removing all but the kernel [capabilities](http://man7.org/linux/man-pages/man7/capabilities.7.html) needed to run the application. By default, all containers are run in the root user namespace so running containers as a non-root user is important. 39 | 40 | ### AppArmor 41 | 42 | On occasion, an application will need to access a kernel resource that requires special privileges normally granted only to the root user. However, running the application as a user with root privileges is a bad solution as it provides the application with access to the entire system. Instead, the kernel provides a set of capabilities that can be granted to a process to allow it coarse-grained access to only the kernel resources it needs and nothing more. 43 | 44 | Using kernel modules such as AppArmor, Kubernetes provides an easy interface to both run the containerized application as a non-root user in the process namespace and restrict the set of capabilities granted to the process. 45 | 46 | ### The container configurations 47 | 48 | This demonstration will deploy five containers in a private cluster: 49 | 50 | 1. A container run as the root user in the container in the Dockerfile 51 | 1. A container run as a user created in the container in the Dockerfile 52 | 1. A container that Kubernetes started as a non-root user despite the Dockerfile not specifying it be run as a non-root user 53 | 1. A container with a lenient AppArmor profile that allows all non-root permissions. 54 | 1. A container with an AppArmor profile applied to disallow the `/proc/cpuinfo` endpoint from being properly read 55 | 56 | Each container will be exposed outside the clusters as an internal load balancer. 57 | 58 | The containers themselves are running a simple Go web server with five endpoints. The endpoints differ in terms of the privileges they need to complete the request. A non-root user cannot read a file owned by root. The `nobody` user cannot read `/proc/cpuinfo` when that privilege is being blocked by AppArmor. 59 | 60 | 1. An endpoint to get the container's hostname 61 | 1. An endpoint to get the username, UID, and GID of identity running the server 62 | 1. An endpoint to read a file owned by the `root` user 63 | 1. An endpoint to read a file owned by the `nobody` user 64 | 1. An endpoint to read the `/proc/cpuinfo` file 65 | 66 | ## Prerequisites 67 | 68 | ### Run Demo in a Google Cloud Shell 69 | 70 | Click the button below to run the demo in a [Google Cloud Shell](https://cloud.google.com/shell/docs/). 71 | 72 | [![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/open?cloudshell_git_repo=https://github.com/GoogleCloudPlatform/gke-application-security-demo.git&cloudshell_image=gcr.io/graphite-cloud-shell-images/terraform:latest&cloudshell_tutorial=README.md) 73 | 74 | 75 | All the tools for the demo are installed. When using Cloud Shell execute the following 76 | command in order to setup gcloud cli. When executing this command please setup your region and zone. 77 | 78 | ```console 79 | gcloud init 80 | ``` 81 | 82 | ### Tools 83 | 1. [Terraform >= 0.12.3](https://www.terraform.io/downloads.html) 84 | 2. [Google Cloud SDK version >= 253.0.0](https://cloud.google.com/sdk/docs/downloads-versioned-archives) 85 | 3. [kubectl matching the latest GKE version](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 86 | 4. bash or bash compatible shell 87 | 5. [GNU Make 3.x or later](https://www.gnu.org/software/make/) 88 | 6. A Google Cloud Platform project where you have permission to create 89 | networks 90 | 91 | #### Install Cloud SDK 92 | The Google Cloud SDK is used to interact with your GCP resources. 93 | [Installation instructions](https://cloud.google.com/sdk/downloads) for multiple platforms are available online. 94 | 95 | #### Install kubectl CLI 96 | 97 | The kubectl CLI is used to interteract with both Kubernetes Engine and kubernetes in general. 98 | [Installation instructions](https://cloud.google.com/kubernetes-engine/docs/quickstart) 99 | for multiple platforms are available online. 100 | 101 | #### Install Terraform 102 | 103 | Terraform is used to automate the manipulation of cloud infrastructure. Its 104 | [installation instructions](https://www.terraform.io/intro/getting-started/install.html) are also available online. 105 | 106 | ## Deployment 107 | 108 | The steps below will walk you through using terraform to deploy a Kubernetes Engine cluster that you will then use for exploring multiple types of container security configurations. 109 | 110 | ### Authenticate gcloud 111 | 112 | Prior to running this demo, ensure you have authenticated your gcloud client by running the following command: 113 | 114 | ```console 115 | gcloud auth application-default login 116 | ``` 117 | 118 | ### Configure gcloud settings 119 | 120 | Run `gcloud config list` and make sure that `compute/zone`, `compute/region` and `core/project` are populated with values that work for you. You can set their values with the following commands: 121 | 122 | ```console 123 | # Where the region is us-east1 124 | gcloud config set compute/region us-east1 125 | 126 | Updated property [compute/region]. 127 | ``` 128 | 129 | ```console 130 | # Where the zone inside the region is us-east1-c 131 | gcloud config set compute/zone us-east1-c 132 | 133 | Updated property [compute/zone]. 134 | ``` 135 | 136 | ```console 137 | # Where the project name is my-project-name 138 | gcloud config set project my-project-name 139 | 140 | Updated property [core/project]. 141 | ``` 142 | 143 | ### Setup this project 144 | 145 | This project requires the following Google Cloud Service APIs to be enabled: 146 | 147 | * `compute.googleapis.com` 148 | * `container.googleapis.com` 149 | * `cloudbuild.googleapis.com` 150 | 151 | In addition, the terraform configuration takes three parameters to determine where the Kubernetes Engine cluster should be created: 152 | 153 | * `project` 154 | * `region` 155 | * `zone` 156 | 157 | For simplicity, these parameters are to be specified in a file named `terraform.tfvars`, in the `terraform` directory. To ensure the appropriate APIs are enabled and to generate the `terraform/terraform.tfvars` file based on your gcloud defaults, run: 158 | 159 | ```console 160 | make setup-project 161 | ``` 162 | 163 | This will enable the necessary Service APIs, and it will also generate a `terraform/terraform.tfvars` file with the following keys. The values themselves will match the output of `gcloud config list`: 164 | 165 | ```console 166 | $ cat terraform/terraform.tfvars 167 | 168 | project="YOUR_PROJECT" 169 | region="YOUR_REGION" 170 | zone="YOUR_ZONE" 171 | ``` 172 | 173 | If you need to override any of the defaults, simply replace the desired value(s) to the right of the equals sign(s). Be sure your replacement values are still double-quoted. 174 | 175 | ### Provisioning the Kubernetes Engine Cluster 176 | 177 | Next, apply the terraform configuration with: 178 | 179 | ```console 180 | # From within the project root, use make to apply the terraform 181 | make tf-apply 182 | ``` 183 | 184 | This will take a few minutes to complete. The following is the last few lines of successful output. 185 | 186 | ```console 187 | ...snip... 188 | google_container_cluster.primary: Still creating... (2m20s elapsed) 189 | google_container_cluster.primary: Still creating... (2m30s elapsed) 190 | google_container_cluster.primary: Still creating... (2m40s elapsed) 191 | google_container_cluster.primary: Still creating... (2m50s elapsed) 192 | google_container_cluster.primary: Still creating... (3m0s elapsed) 193 | google_container_cluster.primary: Still creating... (3m10s elapsed) 194 | google_container_cluster.primary: Still creating... (3m20s elapsed) 195 | google_container_cluster.primary: Still creating... (3m30s elapsed) 196 | google_container_cluster.primary: Still creating... (3m40s elapsed) 197 | google_container_cluster.primary: Creation complete after 3m44s (ID: gke-security-best-practices) 198 | 199 | Apply complete! Resources: 7 added, 0 changed, 0 destroyed. 200 | ``` 201 | 202 | Once that has completed, remote into the bastion instance using SSH: 203 | 204 | ```console 205 | gcloud compute ssh gke-application-security-bastion 206 | ``` 207 | 208 | Apply the manifests for the cluster using the deployment script: 209 | 210 | ```console 211 | ./scripts/deploy.sh 212 | ``` 213 | 214 | This will take a minute or two to complete. The final output should be similar to: 215 | 216 | ```console 217 | namespace/apparmor created 218 | configmap/apparmor-profiles created 219 | daemonset.apps/apparmor-loader created 220 | deployment.apps/armored-hello-user created 221 | service/armored-hello-user created 222 | deployment.apps/armored-hello-denied created 223 | service/armored-hello-denied created 224 | deployment.apps/hello-override created 225 | service/hello-override created 226 | deployment.apps/hello-root created 227 | service/hello-root created 228 | deployment.apps/hello-user created 229 | service/hello-user created 230 | 231 | ...snip... 232 | 233 | Service hello-root has not allocated an IP yet. 234 | Service hello-root has not allocated an IP yet. 235 | Service hello-root IP has been allocated 236 | Service hello-user has not allocated an IP yet. 237 | Service hello-user has not allocated an IP yet. 238 | Service hello-user has not allocated an IP yet. 239 | Service hello-user has not allocated an IP yet. 240 | Service hello-user IP has been allocated 241 | Service hello-override IP has been allocated 242 | Service armored-hello-user IP has been allocated 243 | Service armored-hello-denied IP has been allocated 244 | ``` 245 | 246 | At this point, the environment should be completely set up. 247 | 248 | ## Validation 249 | 250 | To test all of the services in one command, run the validation script from the scripts directory of the bastion host: 251 | 252 | ```console 253 | ./scripts/validate.sh 254 | ``` 255 | 256 | This script queries each of the services to get: 257 | 258 | * the hostname of the pod being queried 259 | * the username, UID, and GID of the process the pod's web server is running as 260 | * the contents of a file owned by root 261 | * the contents of a file owned by a non-root user 262 | * the first 5 lines of content from `/proc/cpuinfo` 263 | 264 | The first service, `hello-root`, has an output similar to: 265 | 266 | ```console 267 | Querying service running natively as root 268 | You are querying host hello-root-54fdf49bf7-8bjmm 269 | User: root 270 | UID: 0 271 | GID: 0 272 | You have read the root.txt file. 273 | You have read the user.txt file. 274 | processor : 0 275 | vendor_id : GenuineIntel 276 | cpu family : 6 277 | model : 63 278 | model name : Intel(R) Xeon(R) CPU @ 2.30GHz 279 | ``` 280 | 281 | and it clearly shows that it is running as `root` and can perform all actions. 282 | 283 | The third service, `hello-user`, has an output similar to: 284 | 285 | ```console 286 | Querying service containers running natively as user 287 | You are querying host hello-user-76957b5645-hvfw2 288 | User: nobody 289 | UID: 65534 290 | GID: 65534 291 | unable to open root.txt: open root.txt: permission denied 292 | You have read the user.txt file. 293 | % Total % Received % Xferd Average Speed Time Time Time Current 294 | Dload Upload Total Spent Left Speed 295 | 100 804 100 804 0 0 156k 0 --:--:-- --:--:-- --:--:-- 196k 296 | processor : 0 297 | vendor_id : GenuineIntel 298 | cpu family : 6 299 | model : 45 300 | model name : Intel(R) Xeon(R) CPU @ 2.60GHz 301 | ``` 302 | 303 | which shows that it is running as `nobody` (65534) and therefore can read user.txt but not root.txt. 304 | 305 | The third service, `hello-override`, has an output similar to: 306 | 307 | ```console 308 | Querying service containers normally running as root but overridden by Kubernetes 309 | You are querying host hello-override-7c6c4b6c4-szmrh 310 | User: nobody 311 | UID: 65534 312 | GID: 65534 313 | unable to open root.txt: open root.txt: permission denied 314 | You have read the user.txt file. 315 | % Total % Received % Xferd Average Speed Time Time Time Current 316 | Dload Upload Total Spent Left Speed 317 | 100 804 100 804 0 0 144k 0 --:--:-- --:--:-- --:--:-- 157k 318 | processor : 0 319 | vendor_id : GenuineIntel 320 | cpu family : 6 321 | model : 45 322 | model name : Intel(R) Xeon(R) CPU @ 2.60GHz 323 | ``` 324 | 325 | and it shows that the container is running as the `nobody` user of id `65534`. Therefore, it can again read the `user.txt` file and read from `/proc/cpuinfo`. 326 | 327 | The fourth service, `armored-hello-user`, has an output similar to: 328 | 329 | ```console 330 | Querying service containers with an AppArmor profile allowing reading /proc/cpuinfo 331 | You are querying host armored-hello-user-5645cd4496-qls6q 332 | User: nobody 333 | UID: 65534 334 | GID: 65534 335 | unable to open root.txt: open root.txt: permission denied 336 | You have read the user.txt file. 337 | % Total % Received % Xferd Average Speed Time Time Time Current 338 | Dload Upload Total Spent Left Speed 339 | 100 804 100 804 0 0 148k 0 --:--:-- --:--:-- --:--:-- 157k 340 | processor : 0 341 | vendor_id : GenuineIntel 342 | cpu family : 6 343 | model : 45 344 | model name : Intel(R) Xeon(R) CPU @ 2.60GHz 345 | ``` 346 | 347 | and it shows that the leniently armored container still has the default access of the 348 | `nobody` user. 349 | 350 | The fifth and final service, `armored-hello-denied`, has an output similar to: 351 | 352 | ```console 353 | Querying service containers with an AppArmor profile blocking the reading of /proc/cpuinfo 354 | You are querying host armored-hello-denied-6fccb988dd-sxhmz 355 | User: nobody 356 | UID: 65534 357 | GID: 65534 358 | unable to open root.txt: open root.txt: permission denied 359 | You have read the user.txt file. 360 | % Total % Received % Xferd Average Speed Time Time Time Current 361 | Dload Upload Total Spent Left Speed 362 | 100 63 100 63 0 0 12162 0 --:--:-- --:--:-- --:--:-- 12600 363 | unable to open root.txt: open /proc/cpuinfo: permission denied 364 | ``` 365 | 366 | and it shows that the container is prohibited by AppArmor policy from reading the `user.txt` and `/proc/cpuinfo`. 367 | 368 | ## Tear Down 369 | 370 | To tear down the environment, use : 371 | 372 | ```console 373 | ./scripts/teardown.sh 374 | ``` 375 | 376 | It's output should look like the following: 377 | 378 | ```console 379 | daemonset.apps "apparmor-loader" deleted 380 | configmap "apparmor-profiles" deleted 381 | namespace "apparmor" deleted 382 | deployment.apps "armored-hello-user" deleted 383 | service "armored-hello-user" deleted 384 | deployment.apps "armored-hello-denied" deleted 385 | service "armored-hello-denied" deleted 386 | deployment.apps "hello-override" deleted 387 | service "hello-override" deleted 388 | deployment.apps "hello-root" deleted 389 | service "hello-root" deleted 390 | deployment.apps "hello-user" deleted 391 | service "hello-user" deleted 392 | ``` 393 | 394 | After that script completes, log out of the bastion host and run the following to destroy the environment: 395 | 396 | ```console 397 | make tf-destroy 398 | ``` 399 | 400 | Terraform will destroy the environment and indicate when it has completed: 401 | 402 | ```console 403 | ...snip... 404 | module.network.google_compute_subnetwork.cluster-subnet: Destroying... (ID: us-east1/kube-net-subnet) 405 | google_service_account.admin: Destruction complete after 0s 406 | module.network.google_compute_subnetwork.cluster-subnet: Still destroying... (ID: us-east1/kube-net-subnet, 10s elapsed) 407 | module.network.google_compute_subnetwork.cluster-subnet: Still destroying... (ID: us-east1/kube-net-subnet, 20s elapsed) 408 | module.network.google_compute_subnetwork.cluster-subnet: Destruction complete after 25s 409 | module.network.google_compute_network.gke-network: Destroying... (ID: kube-net) 410 | module.network.google_compute_network.gke-network: Still destroying... (ID: kube-net, 10s elapsed) 411 | module.network.google_compute_network.gke-network: Still destroying... (ID: kube-net, 20s elapsed) 412 | module.network.google_compute_network.gke-network: Destruction complete after 25s 413 | 414 | Destroy complete! Resources: 7 destroyed. 415 | ``` 416 | 417 | ## Troubleshooting 418 | 419 | ### Terraform destroy does not finish cleanly. The error will look something like 420 | 421 | ```console 422 | Error: Error applying plan: 423 | 424 | 1 error(s) occurred: 425 | 426 | * module.network.google_compute_network.gke-network (destroy): 1 error(s) occurred: 427 | 428 | * google_compute_network.gke-network: The network resource 'projects/seymourd-sandbox/global/networks/kube-net' is already being used by 'projects/seymourd-sandbox/global/firewalls/k8s-29e43f3a2accf594-node-hc' 429 | 430 | 431 | Terraform does not automatically rollback in the face of errors. Instead, your Terraform state file has been partially updated with any resources that successfully completed. Please address the error above and apply again to incrementally change your infrastructure. 432 | 433 | ``` 434 | 435 | Solution: the cluster does not always cleanly remove all of the GCP resources associated with a service before the cluster is deleted. You will need to manually clean up the remaining resources using either the Cloud Console or gcloud. 436 | 437 | ### The install script fails with a `Permission denied` when running Terraform 438 | 439 | The credentials that Terraform is using do not provide the necessary permissions to create resources in the selected projects. Ensure that the account listed in `gcloud config list` has necessary permissions to create resources. If it does, regenerate the application default credentials using `gcloud auth application-default login`. 440 | 441 | ### Invalid fingerprint error during Terraform operations 442 | 443 | Terraform occasionally complains about an invalid fingerprint, when updating certain resources. If you see the error below, simply re-run the command. ![terraform fingerprint error](./img/terraform_fingerprint_error.png) 444 | 445 | ## Relevant Material 446 | 447 | * [Capabilities documentation](http://man7.org/linux/man-pages/man7/capabilities.7.html) 448 | * [Docker security documentation](https://docs.docker.com/engine/security/security/) 449 | * [AppArmor](https://wiki.ubuntu.com/AppArmor) 450 | * [Kubernetes Engine Release Notes](https://cloud.google.com/kubernetes-engine/release-notes) 451 | 452 | **This is not an officially supported Google product** -------------------------------------------------------------------------------- /containers/app.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | /* 18 | When compiled, this application provides endpoints to make demonstrating 19 | various security best practices easier. 20 | */ 21 | 22 | package main 23 | 24 | import ( 25 | "fmt" 26 | "io/ioutil" 27 | "log" 28 | "net/http" 29 | "os" 30 | "os/user" 31 | ) 32 | 33 | // hostnameHandler handles /hostname and returns the result of os.Hostname() 34 | func hostnameHandler(w http.ResponseWriter, r *http.Request) { 35 | h, err := os.Hostname() 36 | if err != nil { 37 | fmt.Fprintf(w, "unable to get hostname: %s", err) 38 | } 39 | fmt.Fprintf(w, "You are querying host %s\n", h) 40 | } 41 | 42 | // getUserHandler handles /getuser and returns the result of user.Current() 43 | func getUserHandler(w http.ResponseWriter, r *http.Request) { 44 | user, err := user.Current() 45 | if err != nil { 46 | fmt.Fprintf(w, "unable to get user: %s", err) 47 | } 48 | 49 | fmt.Fprintf(w, "User: %s\nUID: %s\nGID: %s\n", user.Username, user.Uid, user.Gid) 50 | } 51 | 52 | // userFileHandler handles /userfile and returns the contents of user.txt 53 | func userFileHandler(w http.ResponseWriter, r *http.Request) { 54 | file, err := ioutil.ReadFile("user.txt") 55 | 56 | if err != nil { 57 | fmt.Fprintf(w, "unable to open user.txt: %s", err) 58 | } 59 | 60 | fmt.Fprintf(w, "%s\n", string(file)) 61 | } 62 | 63 | // rootFileHandler handles /rootfile and returns the contents of root.txt 64 | // which is owned by root in the container 65 | func rootFileHandler(w http.ResponseWriter, r *http.Request) { 66 | file, err := ioutil.ReadFile("root.txt") 67 | 68 | if err != nil { 69 | fmt.Fprintf(w, "unable to open root.txt: %s", err) 70 | } 71 | 72 | fmt.Fprintf(w, "%s\n", string(file)) 73 | } 74 | 75 | // procFileHandler handles /procfile and returns the contents of /proc/cpuinfo 76 | // which can be blocked by AppArmor 77 | func procFileHandler(w http.ResponseWriter, r *http.Request) { 78 | file, err := ioutil.ReadFile("/proc/cpuinfo") 79 | 80 | if err != nil { 81 | fmt.Fprintf(w, "unable to open root.txt: %s", err) 82 | } 83 | 84 | fmt.Fprintf(w, "%s\n", string(file)) 85 | } 86 | 87 | func main() { 88 | http.HandleFunc("/", hostnameHandler) 89 | http.HandleFunc("/hostname", hostnameHandler) 90 | http.HandleFunc("/getuser", getUserHandler) 91 | http.HandleFunc("/userfile", userFileHandler) 92 | http.HandleFunc("/rootfile", rootFileHandler) 93 | http.HandleFunc("/procfile", procFileHandler) 94 | log.Print("Starting web server on port 8080...") 95 | log.Fatal(http.ListenAndServe(":8080", nil)) 96 | } 97 | -------------------------------------------------------------------------------- /containers/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | steps: 16 | - name: 'gcr.io/cloud-builders/docker' 17 | args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/hello-run-as-root', '-f', 'root_Dockerfile', '.'] 18 | - name: 'gcr.io/cloud-builders/docker' 19 | args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/hello-run-as-user', '-f', 'user_Dockerfile', '.' ] 20 | images: 21 | - 'gcr.io/$PROJECT_ID/hello-run-as-root:latest' 22 | - 'gcr.io/$PROJECT_ID/hello-run-as-user:latest' 23 | 24 | -------------------------------------------------------------------------------- /containers/root.txt: -------------------------------------------------------------------------------- 1 | You have read the root.txt file. -------------------------------------------------------------------------------- /containers/root_Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.10.3 16 | RUN apt-get update && apt-get upgrade -y 17 | WORKDIR /go/src/app 18 | COPY . . 19 | RUN go build -o app . 20 | 21 | FROM golang:1.10.3 22 | RUN apt-get update && apt-get upgrade -y 23 | WORKDIR /usr/local/bin 24 | COPY --from=0 /go/src/app/app /go/src/app/root.txt /go/src/app/user.txt ./ 25 | RUN chown nobody:nogroup user.txt && chown root:root root.txt && \ 26 | chmod 0600 user.txt root.txt 27 | EXPOSE 8080 28 | ENTRYPOINT [ "/usr/local/bin/app" ] 29 | -------------------------------------------------------------------------------- /containers/user.txt: -------------------------------------------------------------------------------- 1 | You have read the user.txt file. -------------------------------------------------------------------------------- /containers/user_Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | FROM golang:1.10.3 16 | WORKDIR /go/src/app 17 | RUN apt-get update && apt-get upgrade -y 18 | COPY . . 19 | RUN go build -o app . 20 | 21 | FROM golang:1.10.3 22 | RUN apt-get update && apt-get upgrade -y 23 | WORKDIR /usr/local/bin/ 24 | COPY --from=0 /go/src/app/app /go/src/app/root.txt /go/src/app/user.txt ./ 25 | RUN chown nobody:nogroup user.txt && chown root:root root.txt && \ 26 | chmod 0600 user.txt root.txt 27 | EXPOSE 8080 28 | USER app 29 | ENTRYPOINT [ "/usr/local/bin/app" ] 30 | -------------------------------------------------------------------------------- /enable-apis.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # Stop immediately if something goes wrong 19 | set -euo pipefail 20 | 21 | # Validate the user would like to proceed 22 | echo 23 | echo "The following APIs will be enabled in your Google Cloud account:" 24 | echo "- compute.googleapis.com" 25 | echo "- container.googleapis.com" 26 | echo "- cloudbuild.googleapis.com" 27 | echo 28 | 29 | # Enable Compute Engine, Kubernetes Engine, and Container Builder 30 | gcloud services enable compute.googleapis.com \ 31 | container.googleapis.com \ 32 | cloudbuild.googleapis.com 33 | echo "APIs enabled successfully." 34 | -------------------------------------------------------------------------------- /generate-tfvars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # "---------------------------------------------------------" 18 | # "- -" 19 | # "- Helper script to generate terraform variables -" 20 | # "- file based on glcoud defaults. -" 21 | # "- -" 22 | # "---------------------------------------------------------" 23 | 24 | # Stop immediately if something goes wrong 25 | set -euo pipefail 26 | 27 | # This script should be run from directory that contains the terraform directory. 28 | # The purpose is to populate defaults for subsequent terraform commands. 29 | 30 | # git is required for this tutorial 31 | command -v git >/dev/null 2>&1 || { \ 32 | echo >&2 "I require git but it's not installed. Aborting."; exit 1; } 33 | 34 | # glcoud is required for this tutorial 35 | command -v gcloud >/dev/null 2>&1 || { \ 36 | echo >&2 "I require gcloud but it's not installed. Aborting."; exit 1; } 37 | 38 | 39 | # gcloud config holds values related to your environment. If you already 40 | # defined a default region we will retrieve it and use it 41 | REGION="$(gcloud config get-value compute/region)" 42 | if [[ -z "${REGION}" ]]; then 43 | echo "https://cloud.google.com/compute/docs/regions-zones/changing-default-zone-region" 1>&2 44 | echo "gcloud cli must be configured with a default region." 1>&2 45 | echo "run 'gcloud config set compute/region REGION'." 1>&2 46 | echo "replace 'REGION' with the region name like us-west1." 1>&2 47 | exit 1; 48 | fi 49 | 50 | # gcloud config holds values related to your environment. If you already 51 | # defined a default zone we will retrieve it and use it 52 | ZONE="$(gcloud config get-value compute/zone)" 53 | if [[ -z "${ZONE}" ]]; then 54 | echo "https://cloud.google.com/compute/docs/regions-zones/changing-default-zone-region" 1>&2 55 | echo "gcloud cli must be configured with a default zone." 1>&2 56 | echo "run 'gcloud config set compute/zone ZONE'." 1>&2 57 | echo "replace 'ZONE' with the zone name like us-west1-a." 1>&2 58 | exit 1; 59 | fi 60 | 61 | # gcloud config holds values related to your environment. If you already 62 | # defined a default project we will retrieve it and use it 63 | PROJECT="$(gcloud config get-value core/project)" 64 | if [[ -z "${PROJECT}" ]]; then 65 | echo "gcloud cli must be configured with a default project." 1>&2 66 | echo "run 'gcloud config set core/project PROJECT'." 1>&2 67 | echo "replace 'PROJECT' with the project name." 1>&2 68 | exit 1; 69 | fi 70 | 71 | 72 | # Use git to find the top-level directory and confirm 73 | # by looking for the 'terraform' directory 74 | PROJECT_DIR="$(git rev-parse --show-toplevel)" 75 | if [[ -d "./terraform" ]]; then 76 | PROJECT_DIR="$(pwd)" 77 | fi 78 | if [[ -z "${PROJECT_DIR}" ]]; then 79 | echo "Could not identify project base directory." 1>&2 80 | echo "Please re-run from a project directory and ensure" 1>&2 81 | echo "the .git directory exists." 1>&2 82 | exit 1; 83 | fi 84 | 85 | ( 86 | cd "${PROJECT_DIR}" 87 | 88 | TFVARS_FILE="./terraform/terraform.tfvars" 89 | 90 | # We don't want to overwrite a pre-existing tfvars file 91 | if [[ -f "${TFVARS_FILE}" ]] 92 | then 93 | rm "${TFVARS_FILE}" 94 | 95 | fi 96 | # Write out all the values we gathered into a tfvars file so you don't 97 | # have to enter the values manually 98 | cat < "${TFVARS_FILE}" 99 | project="${PROJECT}" 100 | region="${REGION}" 101 | zone="${ZONE}" 102 | vpc_name="kube-net" 103 | execution_id="$RANDOM" 104 | EOF 105 | ) 106 | 107 | -------------------------------------------------------------------------------- /img/terraform_fingerprint_error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/gke-application-security-demo/994ecafb8d035b4531ad6fd786a856dfb1be5ce5/img/terraform_fingerprint_error.png -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /terraform/iam.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Service accounts are used to provide credentials to software running in GCP 18 | // You set what privileges a service account has by defining custom roles 19 | // and binding those roles to a service account in the IAM tab 20 | resource "google_service_account" "admin" { 21 | account_id = "gke-tutorial-admin-${var.execution_id}" 22 | display_name = "GKE Tutorial Admin" 23 | } 24 | 25 | resource "google_project_iam_member" "kube-api-admin" { 26 | project = var.project 27 | role = "roles/container.admin" 28 | member = "serviceAccount:${google_service_account.admin.email}" 29 | } 30 | -------------------------------------------------------------------------------- /terraform/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Provides access to available Google Container Engine versions in a zone for a given project. 18 | // https://www.terraform.io/docs/providers/google/d/google_container_engine_versions.html 19 | data "google_container_engine_versions" "on-prem" { 20 | zone = var.zone 21 | project = var.project 22 | } 23 | 24 | // Create the network to be used by the cluster and bastion host 25 | module "network" { 26 | source = "./modules/network" 27 | project = var.project 28 | region = var.region 29 | vpc_name = "${var.vpc_name}-${var.execution_id}" 30 | tags = var.bastion_tags 31 | execution_id = var.execution_id 32 | } 33 | 34 | // Create the necessary firewalls to allow ssh to the bastion host 35 | module "firewall" { 36 | source = "./modules/firewall" 37 | project = var.project 38 | vpc = module.network.network_self_link 39 | net_tags = var.bastion_tags 40 | execution_id = var.execution_id 41 | } 42 | 43 | // Create and configure the bastion host used to access the private cluster 44 | module "bastion" { 45 | source = "./modules/instance" 46 | project = var.project 47 | hostname = "gke-application-security-bastion" 48 | machine_type = var.bastion_machine_type 49 | zone = var.zone 50 | tags = var.bastion_tags 51 | cluster_subnet = module.network.subnet_self_link 52 | cluster_name = "${var.cluster_name}-${var.execution_id}" 53 | service_account_email = google_service_account.admin.email 54 | grant_cluster_admin = "1" 55 | vpc_name = "${var.vpc_name}-${var.execution_id}" 56 | execution_id = var.execution_id 57 | } 58 | 59 | // Create the Kubernetes engine cluster that we will launch the demonstration 60 | // pods in 61 | resource "google_container_cluster" "primary" { 62 | name = "${var.cluster_name}-${var.execution_id}" 63 | project = var.project 64 | zone = var.zone 65 | network = module.network.network_self_link 66 | subnetwork = module.network.subnet_self_link 67 | min_master_version = data.google_container_engine_versions.on-prem.latest_master_version 68 | initial_node_count = var.initial_node_count 69 | 70 | lifecycle { 71 | ignore_changes = [ip_allocation_policy.0.services_secondary_range_name] 72 | } 73 | 74 | additional_zones = [] 75 | 76 | // Scopes necessary for the nodes to function correctly 77 | node_config { 78 | oauth_scopes = [ 79 | "https://www.googleapis.com/auth/compute", 80 | "https://www.googleapis.com/auth/devstorage.read_only", 81 | "https://www.googleapis.com/auth/logging.write", 82 | "https://www.googleapis.com/auth/monitoring", 83 | ] 84 | 85 | machine_type = var.node_machine_type 86 | image_type = "COS" 87 | 88 | // (Optional) The Kubernetes labels (key/value pairs) to be applied to each node. 89 | labels = { 90 | status = "poc" 91 | } 92 | 93 | // (Optional) The list of instance tags applied to all nodes. 94 | // Tags are used to identify valid sources or targets for network firewalls. 95 | tags = ["poc"] 96 | } 97 | 98 | // (Required for private cluster, optional otherwise) Configuration for cluster IP allocation. 99 | // As of now, only pre-allocated subnetworks (custom type with 100 | // secondary ranges) are supported. This will activate IP aliases. 101 | ip_allocation_policy { 102 | cluster_secondary_range_name = "secondary-range" 103 | } 104 | 105 | // In a private cluster, the master has two IP addresses, one public and one 106 | // private. Nodes communicate to the master through this private IP address. 107 | private_cluster_config { 108 | enable_private_nodes = true 109 | master_ipv4_cidr_block = "10.0.90.0/28" 110 | } 111 | 112 | // (Required for private cluster, optional otherwise) network (cidr) from which cluster is accessible 113 | master_authorized_networks_config { 114 | cidr_blocks { 115 | display_name = "gke-application-security-bastion" 116 | cidr_block = "${module.bastion.external_ip}/32" 117 | } 118 | } 119 | 120 | // (Required for Calico, optional otherwise) Configuration options for the NetworkPolicy feature 121 | network_policy { 122 | enabled = true 123 | provider = "CALICO" 124 | } 125 | 126 | // (Required for network_policy enabled cluster, optional otherwise) 127 | // Addons config supports other options as well, see: 128 | // https://www.terraform.io/docs/providers/google/r/container_cluster.html#addons_config 129 | addons_config { 130 | network_policy_config { 131 | disabled = false 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /terraform/modules/firewall/firewall.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // https://www.terraform.io/docs/providers/google/r/compute_firewall.html 18 | // Adding a firewall rule to allow access to the bastion host from anywhere 19 | resource "google_compute_firewall" "bastion-ssh" { 20 | name = "gke-demo-bastion-fw-${var.execution_id}" 21 | network = "${var.vpc}" 22 | direction = "INGRESS" 23 | project = "${var.project}" 24 | source_ranges = ["0.0.0.0/0"] 25 | 26 | allow { 27 | protocol = "tcp" 28 | ports = ["22"] 29 | } 30 | 31 | target_tags = "${var.net_tags}" 32 | } 33 | -------------------------------------------------------------------------------- /terraform/modules/firewall/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | variable "project" { 18 | description = "the project for this network" 19 | type = "string" 20 | } 21 | 22 | variable "vpc" { 23 | description = "network for the firewall" 24 | type = "string" 25 | } 26 | 27 | variable "net_tags" { 28 | description = "tags for the firewall" 29 | type = "list" 30 | } 31 | 32 | variable "execution_id" { 33 | description = "A random string used as as suffix for resources." 34 | type = "string" 35 | } 36 | -------------------------------------------------------------------------------- /terraform/modules/instance/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This is a convenient way to define the multiline command for binding the instances's 18 | // service account to the cluster-admin role 19 | data "null_data_source" "grant_admin" { 20 | inputs = { 21 | command = < /dev/null || 23 | kubectl create clusterrolebinding gke-tutorial-admin-${var.execution_id}-binding \ 24 | --clusterrole cluster-admin --user $(gcloud config get-value account) 25 | EOF 26 | } 27 | } 28 | 29 | // https://www.terraform.io/docs/providers/template/index.html 30 | // startup script used to initialize kubectl configuration 31 | data "template_file" "startup_script" { 32 | template = <> /etc/profile 37 | echo "$${admin_binding}" >> /etc/profile 38 | EOF 39 | 40 | vars = { 41 | cluster_name = "${var.cluster_name}" 42 | zone = "${var.zone}" 43 | project = "${var.project}" 44 | 45 | // If this instance needs admin access, bind the user to the cluster-admin role, 46 | //if it hasn't already been bound 47 | admin_binding = "${var.grant_cluster_admin ? 48 | "${data.null_data_source.grant_admin.outputs["command"]}" : ""}" 49 | } 50 | } 51 | 52 | // https://www.terraform.io/docs/providers/google/r/compute_instance.html 53 | // bastion host for access and administration of a private cluster. 54 | resource "google_compute_instance" "instance" { 55 | name = "${var.hostname}" 56 | machine_type = "${var.machine_type}" 57 | zone = "${var.zone}" 58 | project = "${var.project}" 59 | tags = "${var.tags}" 60 | 61 | // Specify the Operating System Family and version. 62 | boot_disk { 63 | initialize_params { 64 | image = "debian-cloud/debian-9" 65 | } 66 | } 67 | 68 | // Define a network interface in the correct subnet. 69 | network_interface { 70 | subnetwork = "${var.cluster_subnet}" 71 | 72 | // Add an ephemeral external IP. 73 | access_config { 74 | // Ephemeral IP 75 | } 76 | } 77 | 78 | // Ensure that when the bastion host is booted, it will have kubectl. 79 | metadata_startup_script = "${data.template_file.startup_script.rendered}" 80 | 81 | // Allow the instance to be stopped by terraform when updating configuration 82 | allow_stopping_for_update = true 83 | 84 | // Necessary scopes for administering kubernetes. 85 | service_account { 86 | email = "${var.service_account_email}" 87 | scopes = ["userinfo-email", "compute-ro", "storage-ro", "cloud-platform"] 88 | } 89 | 90 | // local-exec providers may run before the host has fully initialized. However, they 91 | // are run sequentially in the order they were defined. 92 | // 93 | // This provider is used to block the subsequent providers until the instance 94 | // is available. 95 | provisioner "local-exec" { 96 | command = < 33 | 34 | 35 | profile armored-hello-user flags=(attach_disconnected,mediate_deleted) { 36 | #include 37 | 38 | 39 | 40 | network inet tcp, 41 | network inet udp, 42 | 43 | deny network raw, 44 | 45 | deny network packet, 46 | 47 | file, 48 | umount, 49 | 50 | deny /bin/** wl, 51 | deny /boot/** wl, 52 | deny /dev/** wl, 53 | deny /etc/** wl, 54 | deny /home/** wl, 55 | deny /lib/** wl, 56 | deny /lib64/** wl, 57 | deny /media/** wl, 58 | deny /mnt/** wl, 59 | deny /opt/** wl, 60 | deny /proc/** wl, 61 | deny /root/** wl, 62 | deny /sbin/** wl, 63 | deny /srv/** wl, 64 | deny /tmp/** wl, 65 | deny /sys/** wl, 66 | deny /usr/** wl, 67 | 68 | audit /** w, 69 | 70 | 71 | /app ix, 72 | 73 | deny /bin/dash mrwklx, 74 | deny /bin/sh mrwklx, 75 | deny /usr/bin/top mrwklx, 76 | 77 | 78 | 79 | deny @{PROC}/* w, # deny write for all files directly in /proc (not in a subdir) 80 | deny @{PROC}/{[^1-9],[^1-9][^0-9],[^1-9s][^0-9y][^0-9s],[^1-9][^0-9][^0-9][^0-9]*}/** w, 81 | deny @{PROC}/sys/[^k]** w, # deny /proc/sys except /proc/sys/k* (effectively /proc/sys/kernel) 82 | deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} w, # deny everything except shm* in /proc/sys/kernel/ 83 | deny @{PROC}/sysrq-trigger rwklx, 84 | deny @{PROC}/mem rwklx, 85 | deny @{PROC}/kmem rwklx, 86 | deny @{PROC}/kcore rwklx, 87 | deny mount, 88 | deny /sys/[^f]*/** wklx, 89 | deny /sys/f[^s]*/** wklx, 90 | deny /sys/fs/[^c]*/** wklx, 91 | deny /sys/fs/c[^g]*/** wklx, 92 | deny /sys/fs/cg[^r]*/** wklx, 93 | deny /sys/firmware/efi/efivars/** rwklx, 94 | deny /sys/kernel/security/** rwklx, 95 | } 96 | 97 | armored-hello-denied.apparmor: | 98 | #include 99 | 100 | profile armored-hello-denied flags=(attach_disconnected,mediate_deleted) { 101 | #include 102 | 103 | network, 104 | 105 | deny network raw, 106 | 107 | deny network packet, 108 | 109 | file, 110 | umount, 111 | 112 | deny /bin/** wl, 113 | deny /boot/** wl, 114 | deny /dev/** wl, 115 | deny /etc/** wl, 116 | deny /home/** wl, 117 | deny /lib/** wl, 118 | deny /lib64/** wl, 119 | deny /media/** wl, 120 | deny /mnt/** wl, 121 | deny /opt/** wl, 122 | deny /proc/** wl, 123 | deny /root/** wl, 124 | deny /sbin/** wl, 125 | deny /srv/** wl, 126 | deny /tmp/** wl, 127 | deny /sys/** wl, 128 | deny /usr/** wl, 129 | 130 | audit /** w, 131 | 132 | /app ix, 133 | 134 | deny /bin/dash mrwklx, 135 | deny /bin/sh mrwklx, 136 | deny /usr/bin/top mrwklx, 137 | 138 | deny @{PROC}/* w, # deny write for all files directly in /proc (not in a subdir) 139 | deny @{PROC}/cpuinfo wr, 140 | deny @{PROC}/{[^1-9],[^1-9][^0-9],[^1-9s][^0-9y][^0-9s],[^1-9][^0-9][^0-9][^0-9]*}/** w, 141 | deny @{PROC}/sys/[^k]** w, # deny /proc/sys except /proc/sys/k* (effectively /proc/sys/kernel) 142 | deny @{PROC}/sys/kernel/{?,??,[^s][^h][^m]**} w, # deny everything except shm* in /proc/sys/kernel/ 143 | deny @{PROC}/sysrq-trigger rwklx, 144 | deny @{PROC}/mem rwklx, 145 | deny @{PROC}/kmem rwklx, 146 | deny @{PROC}/kcore rwklx, 147 | deny mount, 148 | deny /sys/[^f]*/** wklx, 149 | deny /sys/f[^s]*/** wklx, 150 | deny /sys/fs/[^c]*/** wklx, 151 | deny /sys/fs/c[^g]*/** wklx, 152 | deny /sys/fs/cg[^r]*/** wklx, 153 | deny /sys/firmware/efi/efivars/** rwklx, 154 | deny /sys/kernel/security/** rwklx, 155 | } -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/apparmor-loader-ds.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The example DaemonSet demonstrating how the profile loader can be deployed onto a cluster to 16 | # automatically load AppArmor profiles from a ConfigMap. 17 | 18 | apiVersion: apps/v1 19 | kind: DaemonSet 20 | metadata: 21 | name: apparmor-loader 22 | # Namespace must match that of the ConfigMap. 23 | namespace: apparmor 24 | spec: 25 | selector: 26 | matchLabels: 27 | daemon: apparmor-loader 28 | template: 29 | metadata: 30 | name: apparmor-loader 31 | labels: 32 | daemon: apparmor-loader 33 | spec: 34 | containers: 35 | - name: apparmor-loader 36 | image: gcr.io/google-containers/apparmor-loader:0.2 37 | args: 38 | # Tell the loader to pull the /profiles directory every 30 seconds. 39 | - -poll 40 | - 30s 41 | - /profiles 42 | securityContext: 43 | # The loader requires root permissions to actually load the profiles. 44 | privileged: true 45 | volumeMounts: 46 | - name: sys 47 | mountPath: /sys 48 | readOnly: false 49 | - name: apparmor-includes 50 | mountPath: /etc/apparmor.d 51 | readOnly: true 52 | - name: profiles 53 | mountPath: /profiles 54 | readOnly: true 55 | volumes: 56 | # /sys is required since the loader depends on the apparmorfs interface 57 | # which gets mounted under /sys/kernel/security. 58 | # https://gitlab.com/apparmor/apparmor/wikis/AppArmorInterfaces#apparmorfs 59 | - name: sys 60 | hostPath: 61 | path: /sys 62 | # The /etc/apparmor.d directory is required for most apparmor include templates. 63 | - name: apparmor-includes 64 | hostPath: 65 | path: /etc/apparmor.d 66 | # Map in the profile data. 67 | - name: profiles 68 | configMap: 69 | name: apparmor-profiles 70 | -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/apparmor-namespace.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Create a new namespace for the AppArmor loader daemon to isolate its highly 16 | # privileged container in a way that makes locking it down with RBAC easy. 17 | apiVersion: v1 18 | kind: Namespace 19 | metadata: 20 | name: apparmor 21 | -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/armored-run-as-user-denied.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This pod is running app.go which can read a file from disk, read /proc/cpuinfo, 16 | # read the hostname, and a few other things 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: armored-hello-denied 21 | labels: 22 | app: armored-hello-denied 23 | annotations: 24 | container.apparmor.security.beta.kubernetes.io/armored-hello-denied: localhost/armored-hello-denied 25 | spec: 26 | selector: 27 | matchLabels: 28 | app: armored-hello-denied 29 | replicas: 1 30 | template: 31 | metadata: 32 | labels: 33 | app: armored-hello-denied 34 | # This annotation tells the spec where to find the AppArmor profile that removes unnecessary 35 | # container privileges not needed to run app.go, as well as removing privileges for the 36 | # /procfile endpoint 37 | annotations: 38 | container.apparmor.security.beta.kubernetes.io/armored-hello-denied: localhost/armored-hello-denied 39 | spec: 40 | # prevent the pod from using the default service account credentials 41 | automountServiceAccountToken: false 42 | containers: 43 | - name: armored-hello-denied 44 | image: gcr.io/pso-examples/hello-run-as-user:1.0.1 45 | imagePullPolicy: Always 46 | ports: 47 | - name: http 48 | containerPort: 8080 49 | --- 50 | apiVersion: v1 51 | kind: Service 52 | metadata: 53 | name: armored-hello-denied 54 | labels: 55 | app: armored-hello-denied 56 | annotations: 57 | cloud.google.com/load-balancer-type: "Internal" 58 | spec: 59 | ports: 60 | - port: 80 61 | protocol: TCP 62 | targetPort: http 63 | selector: 64 | app: armored-hello-denied 65 | sessionAffinity: None 66 | type: LoadBalancer 67 | -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/armored-run-as-user.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This pod is running app.go which can read a file from disk, read /proc/cpuinfo, 16 | # read the hostname, and a few other things 17 | apiVersion: apps/v1 18 | kind: Deployment 19 | metadata: 20 | name: armored-hello-user 21 | labels: 22 | app: armored-hello-user 23 | spec: 24 | selector: 25 | matchLabels: 26 | app: armored-hello-user 27 | replicas: 1 28 | template: 29 | metadata: 30 | labels: 31 | app: armored-hello-user 32 | # This annotation tells the spec where to find the AppArmor profile that removes unnecessary 33 | # container privileges not needed to run app.go 34 | annotations: 35 | container.apparmor.security.beta.kubernetes.io/armored-hello-user: localhost/armored-hello-user 36 | spec: 37 | # prevent the pod from using the default service account credentials 38 | automountServiceAccountToken: false 39 | containers: 40 | - name: armored-hello-user 41 | image: gcr.io/pso-examples/hello-run-as-user:1.0.1 42 | imagePullPolicy: Always 43 | ports: 44 | - name: http 45 | containerPort: 8080 46 | --- 47 | apiVersion: v1 48 | kind: Service 49 | metadata: 50 | name: armored-hello-user 51 | labels: 52 | app: armored-hello-user 53 | annotations: 54 | cloud.google.com/load-balancer-type: "Internal" 55 | spec: 56 | ports: 57 | - port: 80 58 | protocol: TCP 59 | targetPort: http 60 | selector: 61 | app: armored-hello-user 62 | sessionAffinity: None 63 | type: LoadBalancer 64 | 65 | -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/override-root-with-user.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1 16 | kind: Deployment 17 | metadata: 18 | name: hello-override 19 | labels: 20 | app: hello-override 21 | spec: 22 | selector: 23 | matchLabels: 24 | app: hello-override 25 | replicas: 1 26 | template: 27 | metadata: 28 | labels: 29 | app: hello-override 30 | spec: 31 | # prevent the pod from using the default service account credentials 32 | automountServiceAccountToken: false 33 | containers: 34 | - name: hello-override 35 | image: gcr.io/pso-examples/hello-run-as-root:1.0.1 36 | imagePullPolicy: Always 37 | ports: 38 | - name: http 39 | containerPort: 8080 40 | securityContext: 41 | runAsUser: 65534 42 | runAsGroup: 65534 43 | --- 44 | apiVersion: v1 45 | kind: Service 46 | metadata: 47 | name: hello-override 48 | labels: 49 | app: hello-override 50 | annotations: 51 | cloud.google.com/load-balancer-type: "Internal" 52 | spec: 53 | ports: 54 | - port: 80 55 | protocol: TCP 56 | targetPort: http 57 | selector: 58 | app: hello-override 59 | sessionAffinity: None 60 | type: LoadBalancer 61 | -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/run-as-root.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This pod is running app.go which can read a file from disk, read /proc/cpuinfo, 16 | # read the hostname, and a few other things 17 | # This image is configured to launch app.go as user 'root' and thus has all the 18 | # permissions of root 19 | apiVersion: apps/v1 20 | kind: Deployment 21 | metadata: 22 | name: hello-root 23 | labels: 24 | app: hello-root 25 | spec: 26 | selector: 27 | matchLabels: 28 | app: hello-root 29 | replicas: 1 30 | template: 31 | metadata: 32 | labels: 33 | app: hello-root 34 | spec: 35 | # prevent the pod from using the default service account credentials 36 | automountServiceAccountToken: false 37 | containers: 38 | - name: hello-root 39 | image: gcr.io/pso-examples/hello-run-as-root:1.0.1 40 | imagePullPolicy: Always 41 | ports: 42 | - name: http 43 | containerPort: 8080 44 | --- 45 | apiVersion: v1 46 | kind: Service 47 | metadata: 48 | name: hello-root 49 | labels: 50 | app: hello-root 51 | annotations: 52 | cloud.google.com/load-balancer-type: "Internal" 53 | spec: 54 | ports: 55 | - port: 80 56 | protocol: TCP 57 | targetPort: http 58 | selector: 59 | app: hello-root 60 | sessionAffinity: None 61 | type: LoadBalancer 62 | 63 | -------------------------------------------------------------------------------- /terraform/modules/instance/manifests/run-as-user.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This pod is running app.go which can read a file from disk, read /proc/cpuinfo, 16 | # read the hostname, and a few other things 17 | # This image is configured to launch app.go as user 'app' and thus has all the 18 | # permissions of 'app' 19 | apiVersion: apps/v1 20 | kind: Deployment 21 | metadata: 22 | name: hello-user 23 | labels: 24 | app: hello-user 25 | spec: 26 | selector: 27 | matchLabels: 28 | app: hello-user 29 | replicas: 1 30 | template: 31 | metadata: 32 | labels: 33 | app: hello-user 34 | spec: 35 | # prevent the pod from using the default service account credentials 36 | automountServiceAccountToken: false 37 | containers: 38 | - name: hello-user 39 | image: gcr.io/pso-examples/hello-run-as-user:1.0.1 40 | imagePullPolicy: Always 41 | ports: 42 | - name: http 43 | containerPort: 8080 44 | --- 45 | apiVersion: v1 46 | kind: Service 47 | metadata: 48 | name: hello-user 49 | labels: 50 | app: hello-user 51 | annotations: 52 | cloud.google.com/load-balancer-type: "Internal" 53 | spec: 54 | ports: 55 | - port: 80 56 | protocol: TCP 57 | targetPort: http 58 | selector: 59 | app: hello-user 60 | sessionAffinity: None 61 | type: LoadBalancer 62 | 63 | -------------------------------------------------------------------------------- /terraform/modules/instance/outputs.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // The public IP of the bastion instance 18 | output "external_ip" { 19 | value = google_compute_instance.instance.network_interface[0].access_config[0].nat_ip 20 | } 21 | -------------------------------------------------------------------------------- /terraform/modules/instance/scripts/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | # Stop immediately if something goes wrong 19 | set -euo pipefail 20 | 21 | ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 22 | 23 | # importing utils.sh which contains utility functions used in multiple files 24 | # shellcheck source=terraform/modules/instance/scripts/utils.sh 25 | source "$ROOT"/utils.sh 26 | 27 | # first we create a kubernetes namespace called 'apparmor' 28 | kubectl apply -f "$ROOT"/../manifests/apparmor-namespace.yaml 29 | # next we create a configmap containing two apparmor profiles as text 30 | kubectl apply -f "$ROOT"/../manifests/apparmor-configmap.yaml 31 | # next we use a daemonset to launch the apparmor-loader container on each 32 | # host, which uses the apparmor config maps to know what to load 33 | kubectl apply -f "$ROOT"/../manifests/apparmor-loader-ds.yaml 34 | # next we run an image as user 'app' with the 'armored-hello-user' apparmor profile 35 | kubectl apply -f "$ROOT"/../manifests/armored-run-as-user.yaml 36 | # next we run an image as user 'app' with the 'armored-hello-denied' apparmor profile 37 | kubectl apply -f "$ROOT"/../manifests/armored-run-as-user-denied.yaml 38 | # next we show off how to use a securityContext block to override the images 39 | # configured user 40 | kubectl apply -f "$ROOT"/../manifests/override-root-with-user.yaml 41 | # next we run an image as root with no apparmor profile 42 | kubectl apply -f "$ROOT"/../manifests/run-as-root.yaml 43 | # and last we run an image as 'app' with no apparmor profile 44 | kubectl apply -f "$ROOT"/../manifests/run-as-user.yaml 45 | 46 | # Wait for the services to be allocated IPs before declaring the deployment 47 | # complete. 48 | wait_for_svc hello-root 49 | wait_for_svc hello-user 50 | wait_for_svc hello-override 51 | wait_for_svc armored-hello-user 52 | wait_for_svc armored-hello-denied 53 | -------------------------------------------------------------------------------- /terraform/modules/instance/scripts/teardown.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Stop immediately if something goes wrong 18 | set -euo pipefail 19 | 20 | ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 21 | 22 | # shellcheck source=terraform/modules/instance/scripts/utils.sh 23 | 24 | source "$ROOT"/utils.sh 25 | 26 | # Clean up the apparmor namespace 27 | kubectl delete -f "$ROOT"/../manifests/apparmor-loader-ds.yaml 28 | kubectl delete -f "$ROOT"/../manifests/apparmor-configmap.yaml 29 | kubectl delete -f "$ROOT"/../manifests/apparmor-namespace.yaml 30 | 31 | # Clean up the default namespace 32 | kubectl delete -f "$ROOT"/../manifests/armored-run-as-user.yaml 33 | kubectl delete -f "$ROOT"/../manifests/armored-run-as-user-denied.yaml 34 | kubectl delete -f "$ROOT"/../manifests/override-root-with-user.yaml 35 | kubectl delete -f "$ROOT"/../manifests/run-as-root.yaml 36 | kubectl delete -f "$ROOT"/../manifests/run-as-user.yaml 37 | -------------------------------------------------------------------------------- /terraform/modules/instance/scripts/utils.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Stop immediately if something goes wrong 18 | set -euo pipefail 19 | 20 | # this function waits for a services load balancer to be assigned a public IP 21 | wait_for_svc () { 22 | RETRIES_REMAINING=30 23 | while [[ $(kubectl get svc -l app="$1" -o jsonpath='{.items[*].status.loadBalancer.ingress[*].ip}') == "" ]]; do 24 | if [[ $RETRIES_REMAINING -eq 0 ]]; then 25 | echo "Retry limit exceeded; exiting..." 26 | exit 1 27 | fi 28 | echo "Service $1 has not allocated an IP yet." 29 | RETRIES_REMAINING=$(("$RETRIES_REMAINING" - 1)) 30 | sleep 10 31 | done 32 | echo "Service $1 IP has been allocated" 33 | } 34 | 35 | # this function hits every endpoint of app.go based on the service name 36 | # passed in as the sole argument 37 | query_svc () { 38 | curl -q http://"$1"/hostname 39 | curl -q http://"$1"/getuser 40 | curl -q http://"$1"/rootfile 41 | curl -q http://"$1"/userfile 42 | curl -q http://"$1"/procfile | head -5 43 | echo "" 44 | } 45 | -------------------------------------------------------------------------------- /terraform/modules/instance/scripts/validate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 18 | 19 | # Stop immediately if something goes wrong 20 | set -euo pipefail 21 | 22 | # shellcheck source=terraform/modules/instance/scripts/utils.sh 23 | source "$ROOT"/utils.sh 24 | 25 | # wait_for_svc is a function defined in utils.sh 26 | wait_for_svc hello-root 27 | wait_for_svc hello-user 28 | wait_for_svc hello-override 29 | wait_for_svc armored-hello-user 30 | wait_for_svc armored-hello-denied 31 | 32 | RUN_AS_ROOT=$(kubectl get svc -l app=hello-root -o jsonpath='{.items[*].status.loadBalancer.ingress[*].ip}') 33 | RUN_AS_USER=$(kubectl get svc -l app=hello-user -o jsonpath='{.items[*].status.loadBalancer.ingress[*].ip}') 34 | RUN_AS_OVERRIDE=$(kubectl get svc -l app=hello-override -o jsonpath='{.items[*].status.loadBalancer.ingress[*].ip}') 35 | RUN_AS_ARMORED=$(kubectl get svc -l app=armored-hello-user -o jsonpath='{.items[*].status.loadBalancer.ingress[*].ip}') 36 | RUN_AS_DENIED=$(kubectl get svc -l app=armored-hello-denied -o jsonpath='{.items[*].status.loadBalancer.ingress[*].ip}') 37 | 38 | 39 | # the query_svc function is defined in utils.sh 40 | # Each call to query_svc hits all five endpoints of app.go 41 | # The output will depend on the privileges of the pod being queried 42 | # You can get a better idea of the expected outputs in the Validation section 43 | # of the README 44 | echo -e "===\\n\\nQuerying service running natively as root" 45 | query_svc "$RUN_AS_ROOT" 46 | 47 | echo -e "===\\n\\nQuerying service containers running natively as user 'nobody'" 48 | query_svc "$RUN_AS_USER" 49 | 50 | echo -e "===\\n\\nQuerying service containers running with user overridden to be 'nobody'" 51 | query_svc "$RUN_AS_OVERRIDE" 52 | 53 | echo -e "===\\n\\nQuerying service containers with an AppArmor profile allowing reading /proc/cpuinfo" 54 | query_svc "$RUN_AS_ARMORED" 55 | 56 | echo -e "===\\n\\nQuerying service containers with an AppArmor profile blocking the reading of /proc/cpuinfo" 57 | query_svc "$RUN_AS_DENIED" 58 | -------------------------------------------------------------------------------- /terraform/modules/instance/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | /* 18 | Required Variables 19 | These must be provided at runtime. 20 | */ 21 | 22 | variable "hostname" { 23 | description = "the hostname" 24 | type = "string" 25 | } 26 | 27 | variable "machine_type" { 28 | description = "the instance type" 29 | type = "string" 30 | } 31 | 32 | variable "project" { 33 | description = "the project for this instance" 34 | type = "string" 35 | } 36 | 37 | variable "zone" { 38 | description = "the desired zone for the host" 39 | type = "string" 40 | } 41 | 42 | variable "execution_id" { 43 | description = "A random string used as as suffix for resources." 44 | type = "string" 45 | } 46 | 47 | variable "tags" { 48 | description = "the instance tags" 49 | type = "list" 50 | } 51 | 52 | variable "cluster_subnet" { 53 | description = "the subnet in which to put the private IP address" 54 | type = "string" 55 | } 56 | 57 | variable "cluster_name" { 58 | description = "the name of the cluster for which this host will be used to connect" 59 | type = "string" 60 | } 61 | 62 | variable "service_account_email" { 63 | description = "" 64 | type = "string" 65 | default = "" 66 | } 67 | 68 | variable "grant_cluster_admin" { 69 | description = "" 70 | type = "string" 71 | default = "0" 72 | } 73 | 74 | variable "vpc_name" { 75 | type = "string" 76 | } 77 | -------------------------------------------------------------------------------- /terraform/modules/network/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This file creates a custom VPC network and subnet to contain the 18 | // GKE cluster. 19 | 20 | // https://www.terraform.io/docs/providers/google/d/datasource_compute_subnetwork.html 21 | // Subnetwork for the GKE cluster. 22 | resource "google_compute_subnetwork" "cluster-subnet" { 23 | name = "${var.vpc_name}-subnet" 24 | project = "${var.project}" 25 | ip_cidr_range = "${var.ip_range}" 26 | network = "${google_compute_network.gke-network.self_link}" 27 | region = "${var.region}" 28 | 29 | // A named secondary range is mandatory for a private cluster, this creates it. 30 | secondary_ip_range { 31 | range_name = "secondary-range" 32 | ip_cidr_range = "${var.secondary_ip_range}" 33 | } 34 | } 35 | 36 | // https://www.terraform.io/docs/providers/google/d/datasource_compute_network.html 37 | // A network to hold just the GKE cluster, not recommended for other instances. 38 | resource "google_compute_network" "gke-network" { 39 | name = "${var.vpc_name}" 40 | project = "${var.project}" 41 | auto_create_subnetworks = false 42 | } 43 | -------------------------------------------------------------------------------- /terraform/modules/network/outputs.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Adding these outputs allows other modules as well as the calling 18 | // terraform script to access the attributes of these modules. 19 | 20 | // network_self_link can be used to reference the network created by this module 21 | output "network_self_link" { 22 | value = "${google_compute_network.gke-network.self_link}" 23 | } 24 | 25 | // subnet_self_link can be used to reference the network created by this module 26 | output "subnet_self_link" { 27 | value = "${google_compute_subnetwork.cluster-subnet.self_link}" 28 | } 29 | -------------------------------------------------------------------------------- /terraform/modules/network/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | variable "project" { 18 | description = "the project for this network" 19 | type = "string" 20 | } 21 | 22 | variable "ip_range" { 23 | type = "string" 24 | default = "10.0.96.0/22" 25 | } 26 | 27 | variable "region" { 28 | type = "string" 29 | } 30 | 31 | variable "secondary_ip_range" { 32 | type = "string" 33 | default = "10.0.92.0/22" 34 | } 35 | 36 | variable "tags" { 37 | type = "list" 38 | } 39 | 40 | variable "vpc_name" { 41 | type = "string" 42 | } 43 | 44 | variable "execution_id" { 45 | description = "A random string used as as suffix for resources." 46 | type = "string" 47 | } 48 | -------------------------------------------------------------------------------- /terraform/provider.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Configures the default project and zone for underlying Google Cloud API calls 18 | provider "google" { 19 | project = var.project 20 | zone = var.zone 21 | version = "~> 2.17.0" 22 | } 23 | 24 | // Pins the version of the "template" provider 25 | provider "template" { 26 | version = "~> 2.1.2" 27 | } 28 | 29 | // Pins the version of the "null" provider 30 | provider "null" { 31 | version = "~> 2.1.2" 32 | } 33 | -------------------------------------------------------------------------------- /terraform/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | /* 18 | This file exposes variables that can be overridden to customize your cloud configuration. 19 | https://www.terraform.io/docs/configuration/variables.html 20 | */ 21 | 22 | /* 23 | Required Variables 24 | These must be provided at runtime. 25 | */ 26 | 27 | variable "zone" { 28 | description = "The zone in which to create the Kubernetes cluster. Must match the region" 29 | type = string 30 | } 31 | 32 | variable "region" { 33 | description = "The region in which to create the Kubernetes cluster." 34 | type = string 35 | } 36 | 37 | variable "project" { 38 | description = "The name of the project in which to create the Kubernetes cluster." 39 | type = string 40 | } 41 | 42 | variable "vpc_name" { 43 | description = "The name used for vps, network and subnetwork." 44 | type = string 45 | } 46 | 47 | variable "execution_id" { 48 | description = "A random string used as as suffix for resources." 49 | type = string 50 | } 51 | 52 | /* 53 | Optional Variables 54 | Defaults will be used for these, if not overridden at runtime. 55 | */ 56 | 57 | variable "bastion_machine_type" { 58 | description = "The instance size to use for your bastion instance." 59 | type = string 60 | default = "f1-micro" 61 | } 62 | 63 | variable "bastion_tags" { 64 | description = "A list of tags applied to your bastion instance." 65 | type = list(string) 66 | default = ["bastion"] 67 | } 68 | 69 | variable "cluster_name" { 70 | description = "The name to give the new Kubernetes cluster." 71 | type = string 72 | default = "gke-security-best-practices" 73 | } 74 | 75 | variable "initial_node_count" { 76 | description = "The number of nodes initially provisioned in the cluster" 77 | type = string 78 | default = "3" 79 | } 80 | 81 | variable "ip_range" { 82 | description = "The CIDR from which to allocate cluster node IPs" 83 | type = string 84 | default = "10.0.96.0/22" 85 | } 86 | 87 | variable "master_cidr_block" { 88 | description = "The CIDR from which to allocate master IPs" 89 | type = string 90 | default = "10.0.90.0/28" 91 | } 92 | 93 | variable "node_machine_type" { 94 | description = "The instance to use for your bastion instance" 95 | type = string 96 | default = "n1-standard-1" 97 | } 98 | 99 | variable "node_tags" { 100 | description = "A list of tags applied to your node instances." 101 | type = list(string) 102 | default = ["poc"] 103 | } 104 | 105 | variable "secondary_ip_range" { 106 | // See https://cloud.google.com/kubernetes-engine/docs/how-to/alias-ips 107 | description = "The CIDR from which to allocate pod IPs for IP Aliasing." 108 | type = string 109 | default = "10.0.92.0/22" 110 | } 111 | 112 | variable "secondary_subnet_name" { 113 | // See https://cloud.google.com/kubernetes-engine/docs/how-to/alias-ips 114 | description = "The name to give the secondary subnet." 115 | type = string 116 | default = "kube-net-secondary-sub" 117 | } 118 | -------------------------------------------------------------------------------- /terraform/versions.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">= 0.12" 19 | } 20 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.BUILD.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.Dockerfile.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.Makefile.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.WORKSPACE.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.bazel.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.bzl.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.css.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.go.preamble: -------------------------------------------------------------------------------- 1 | // +build 2 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.html.preamble: -------------------------------------------------------------------------------- 1 | 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.java.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.js.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.py.preamble: -------------------------------------------------------------------------------- 1 | #! 2 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.py.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.scss.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.sh.preamble: -------------------------------------------------------------------------------- 1 | #! 2 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.sh.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.tf.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | https://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.ts.txt: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // https://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.xml.preamble: -------------------------------------------------------------------------------- 1 | 16 | -------------------------------------------------------------------------------- /test/boilerplate/boilerplate.yaml.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /test/make.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This function checks to make sure that every 18 | # shebang has a '- e' flag, which causes it 19 | # to exit on error 20 | function check_bash() { 21 | find . -name "*.sh" | while IFS= read -d '' -r file; 22 | do 23 | if [[ "$file" != *"bash -e"* ]]; 24 | then 25 | echo "$file is missing shebang with -e"; 26 | exit 1; 27 | fi; 28 | done; 29 | } 30 | 31 | # This function makes sure that the required files for 32 | # releasing to OSS are present 33 | function basefiles() { 34 | echo "Checking for required files" 35 | test -f CONTRIBUTING.md || echo "Missing CONTRIBUTING.md" 36 | test -f LICENSE || echo "Missing LICENSE" 37 | test -f README.md || echo "Missing README.md" 38 | } 39 | 40 | # This function runs the hadolint linter on 41 | # every file named 'Dockerfile' 42 | function docker() { 43 | echo "Running hadolint on Dockerfiles" 44 | find . -name "Dockerfile" -exec hadolint {} \; 45 | } 46 | 47 | # This function runs 'terraform validate' against all 48 | # files ending in '.tf' 49 | function check_terraform() { 50 | echo "Running terraform validate" 51 | #shellcheck disable=SC2156 52 | find . -name "*.tf" -exec bash -c 'terraform validate --check-variables=false $(dirname "{}")' \; 53 | } 54 | 55 | # This function runs 'go fmt' and 'go vet' on eery file 56 | # that ends in '.go' 57 | function golang() { 58 | echo "Running go fmt and go vet" 59 | find . -name "*.go" -exec go fmt {} \; 60 | find . -name "*.go" -exec go vet {} \; 61 | } 62 | 63 | # This function runs the flake8 linter on every file 64 | # ending in '.py' 65 | function check_python() { 66 | echo "Running flake8" 67 | find . -name "*.py" -exec flake8 {} \; 68 | } 69 | 70 | # This function runs the shellcheck linter on every 71 | # file ending in '.sh' 72 | function check_shell() { 73 | echo "Running shellcheck" 74 | find . -name "*.sh" -exec shellcheck -x {} \; 75 | } 76 | 77 | # This function makes sure that there is no trailing whitespace 78 | # in any files in the project. 79 | # There are some exclusions 80 | function check_trailing_whitespace() { 81 | echo "The following lines have trailing whitespace" 82 | grep -r '[[:blank:]]$' --exclude-dir=".terraform" --exclude="*.png" --exclude-dir=".git" --exclude="*.pyc" . 83 | rc=$? 84 | if [ $rc = 0 ]; then 85 | exit 1 86 | fi 87 | } 88 | -------------------------------------------------------------------------------- /test/verify_boilerplate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3.7 2 | # Copyright 2018 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # https://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # Verifies that all source files contain the necessary copyright boilerplate 16 | # snippet. 17 | 18 | # This code is based on existing work 19 | # https://partner-code.googlesource.com/helmsman-cardinal/+/refs/heads/master/helmsman-template-project/test/verify_boilerplate.py 20 | 21 | 22 | """ 23 | A runnable module to test the presence of boilerplate 24 | text in files within a repo. 25 | """ 26 | 27 | from __future__ import print_function 28 | from subprocess import run, CalledProcessError 29 | import argparse 30 | import glob 31 | import os 32 | import re 33 | import sys 34 | 35 | 36 | # These directories will be omitted from header checks 37 | SKIPPED_PATHS = [ 38 | 'Godeps', 'third_party', '_gopath', '_output', 39 | '.git', 'vendor', '__init__.py', 'node_modules', 40 | 'bazel-out', 'external', '3rdparty' 41 | ] 42 | 43 | # A map of regular expressions used in boilerplate validation. 44 | # The date regex is used in validating the date referenced 45 | # is the boilerplate, by ensuring it is an acceptable year. 46 | REGEXES = { 47 | # beware the Y2100 problem 48 | "date": re.compile(r'(20\d\d)') 49 | } 50 | 51 | 52 | def get_args(): 53 | """Parses command line arguments. 54 | Configures and runs argparse.ArgumentParser to extract command line 55 | arguments. 56 | Returns: 57 | An argparse.Namespace containing the arguments parsed from the 58 | command line 59 | """ 60 | parser = argparse.ArgumentParser() 61 | 62 | parser.add_argument("filenames", 63 | help="""A list of files to check, all in repo are 64 | checked if this is unspecified.""", 65 | nargs='*') 66 | 67 | parser.add_argument("-f", "--force-extension", 68 | default="", 69 | help="""Force an extension to compare against. Useful 70 | for files without extensions, such as runnable shell 71 | scripts .""") 72 | 73 | parser.add_argument( 74 | "-r", "--rootdir", 75 | default=None, 76 | help="""Root directory of repository. If not specified, the script will 77 | attempt to draw this value from git.""") 78 | 79 | parser.add_argument("-b", "--boilerplate-dir", 80 | default=None, 81 | help="""Directory with boilerplate files. Defaults to 82 | [root]/test/boilerplate.""") 83 | 84 | args = parser.parse_args() 85 | 86 | if not args.rootdir: 87 | ask_git = run( 88 | ["git", "rev-parse", "--show-toplevel"], 89 | capture_output=True, text=True) 90 | try: 91 | ask_git.check_returncode() 92 | except CalledProcessError: 93 | print("""No root specfied and directory does not seem to be a git 94 | repository, or git is not installed.""", file=sys.stderr) 95 | sys.exit(1) 96 | args.rootdir = ask_git.stdout.strip() 97 | 98 | if not args.boilerplate_dir: 99 | args.boilerplate_dir = os.path.join(args.rootdir, "test/boilerplate") 100 | 101 | return args 102 | 103 | 104 | def get_references(args): 105 | """Reads each reference boilerplate file's contents into an array, then 106 | adds that array to a dictionary keyed by the file extension. 107 | 108 | Returns: 109 | A dictionary of boilerplate lines, keyed by file extension. 110 | For example, boilerplate.py.txt would result in the 111 | k,v pair {".py": py_lines} where py_lines is an array 112 | containing each line of the file. 113 | """ 114 | references = {} 115 | 116 | # Find all paths for boilerplate references 117 | boilerplate_paths = glob.glob( 118 | os.path.join(args.boilerplate_dir, "boilerplate.*.txt")) 119 | 120 | # Read all boilerplate references into dictionary 121 | for path in boilerplate_paths: 122 | with open(path, 'r') as ref_file: 123 | extension = os.path.basename(path).split(".")[1] 124 | ref = ref_file.read().splitlines() 125 | references[extension] = ref 126 | 127 | return references 128 | 129 | 130 | # Improvement: combine this function with `get_references` 131 | def get_preambles(args): 132 | """Reads each preamble boilerplate file's contents into an array, then 133 | adds that array to a dictionary keyed by the file extension. 134 | 135 | Returns: 136 | A dictionary of boilerplate lines, keyed by file extension. 137 | For example, boilerplate.py.preamble would result 138 | in the k,v pair {".py": py_lines} where py_lines is 139 | an array containing each line of the file 140 | (ex: "#!/usr/bin/env python3.7") 141 | """ 142 | preambles = {} 143 | 144 | # Find all paths for boilerplate preambles 145 | boilerplate_paths = glob.glob( 146 | os.path.join(args.boilerplate_dir, "boilerplate.*.preamble")) 147 | 148 | # Read all boilerplate preambles into dictionary 149 | for path in boilerplate_paths: 150 | with open(path, 'r') as ref_file: 151 | extension = os.path.basename(path).split(".")[1] 152 | ref = ref_file.read().splitlines() 153 | preambles[extension] = ref 154 | 155 | return preambles 156 | 157 | 158 | def has_valid_header(filename, references, preambles, regexs, args): 159 | """Test whether a file has the correct boilerplate header. 160 | Tests each file against the boilerplate stored in refs for that file type 161 | (based on extension), or by the entire filename (eg Dockerfile, Makefile). 162 | Some heuristics are applied to remove build tags and shebangs, but little 163 | variance in header formatting is tolerated. 164 | Args: 165 | filename: A string containing the name of the file to test 166 | references: A map of reference boilerplate text, 167 | keyed by file extension 168 | preambles: A map of preamble boilerplate text, keyed by file extension 169 | regexs: a map of compiled regex objects used in verifying boilerplate 170 | Returns: 171 | True if the file has the correct boilerplate header, otherwise returns 172 | False. 173 | """ 174 | # Read the entire file. 175 | with open(filename, 'r') as test_file: 176 | data = test_file.read() 177 | 178 | # Select the appropriate reference based on the extension, 179 | # or if none, the file name. 180 | basename, extension = get_file_parts(filename) 181 | if args.force_extension: 182 | extension = args.force_extension 183 | elif extension: 184 | extension = extension 185 | else: 186 | extension = basename 187 | ref = references[extension] 188 | print("Verifying boilerplate in file: %s as %s" % ( 189 | os.path.relpath(filename, args.rootdir), 190 | extension)) 191 | 192 | preamble = preambles.get(extension) 193 | if preamble: 194 | preamble = re.escape("\n".join(preamble)) 195 | regflags = re.MULTILINE | re.IGNORECASE 196 | regex = re.compile(r"^(%s.*\n)\n*" % preamble, regflags) 197 | (data, _) = regex.subn("", data, 1) 198 | 199 | data = data.splitlines() 200 | 201 | # if our test file is smaller than the reference it surely fails! 202 | if len(ref) > len(data): 203 | return False 204 | # truncate our file to the same number of lines as the reference file 205 | data = data[:len(ref)] 206 | 207 | # if we don't match the reference at this point, fail 208 | if ref != data: 209 | return False 210 | 211 | return True 212 | 213 | 214 | def get_file_parts(filename): 215 | """Extracts the basename and extension parts of a filename. 216 | Identifies the extension as everything after the last period in filename. 217 | Args: 218 | filename: string containing the filename 219 | Returns: 220 | A tuple of: 221 | A string containing the basename 222 | A string containing the extension in lowercase 223 | """ 224 | extension = os.path.splitext(filename)[1].split(".")[-1].lower() 225 | basename = os.path.basename(filename) 226 | return basename, extension 227 | 228 | 229 | def normalize_files(files, args): 230 | """Extracts the files that require boilerplate checking from the files 231 | argument. 232 | A new list will be built. Each path from the original files argument will 233 | be added unless it is within one of SKIPPED_DIRS. All relative paths will 234 | be converted to absolute paths by prepending the root_dir path parsed from 235 | the command line, or its default value. 236 | Args: 237 | files: a list of file path strings 238 | Returns: 239 | A modified copy of the files list where any any path in a skipped 240 | directory is removed, and all paths have been made absolute. 241 | """ 242 | newfiles = [f for f in files if not any(s in f for s in SKIPPED_PATHS)] 243 | 244 | for idx, pathname in enumerate(newfiles): 245 | if not os.path.isabs(pathname): 246 | newfiles[idx] = os.path.join(args.rootdir, pathname) 247 | return newfiles 248 | 249 | 250 | def get_files(extensions, args): 251 | """Generates a list of paths whose boilerplate should be verified. 252 | If a list of file names has been provided on the command line, it will be 253 | treated as the initial set to search. Otherwise, all paths within rootdir 254 | will be discovered and used as the initial set. 255 | Once the initial set of files is identified, it is normalized via 256 | normalize_files() and further stripped of any file name whose extension is 257 | not in extensions. 258 | Args: 259 | extensions: a list of file extensions indicating which file types 260 | should have their boilerplate verified 261 | Returns: 262 | A list of absolute file paths 263 | """ 264 | files = [] 265 | if args.filenames: 266 | files = args.filenames 267 | else: 268 | for root, dirs, walkfiles in os.walk(args.rootdir): 269 | # don't visit certain dirs. This is just a performance improvement 270 | # as we would prune these later in normalize_files(). But doing it 271 | # cuts down the amount of filesystem walking we do and cuts down 272 | # the size of the file list 273 | for dpath in SKIPPED_PATHS: 274 | if dpath in dirs: 275 | dirs.remove(dpath) 276 | for name in walkfiles: 277 | pathname = os.path.join(root, name) 278 | files.append(pathname) 279 | files = normalize_files(files, args) 280 | outfiles = [] 281 | for pathname in files: 282 | basename, extension = get_file_parts(pathname) 283 | extension_present = extension in extensions or basename in extensions 284 | if args.force_extension or extension_present: 285 | outfiles.append(pathname) 286 | return outfiles 287 | 288 | 289 | def main(args): 290 | """Identifies and verifies files that should have the desired boilerplate. 291 | Retrieves the lists of files to be validated and tests each one in turn. 292 | If all files contain correct boilerplate, this function terminates 293 | normally. Otherwise it prints the name of each non-conforming file and 294 | exists with a non-zero status code. 295 | """ 296 | refs = get_references(args) 297 | preambles = get_preambles(args) 298 | filenames = get_files(refs.keys(), args) 299 | nonconforming_files = [] 300 | for filename in filenames: 301 | if not has_valid_header(filename, refs, preambles, REGEXES, args): 302 | nonconforming_files.append(filename) 303 | if nonconforming_files: 304 | print('%d files have incorrect boilerplate headers:' % len( 305 | nonconforming_files)) 306 | for filename in sorted(nonconforming_files): 307 | print(os.path.relpath(filename, args.rootdir)) 308 | sys.exit(1) 309 | else: 310 | print('All files examined have correct boilerplate.') 311 | 312 | 313 | if __name__ == "__main__": 314 | ARGS = get_args() 315 | main(ARGS) 316 | -------------------------------------------------------------------------------- /waitfor_svc_deleted.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2018 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # Verifies that all source files contain the necessary copyright boilerplate 17 | # snippet. 18 | 19 | # this code waits for the backend services to be deleted 20 | LIST='hello-root|hello-user|hello-override|armored-hello-user|armored-hello-denied' 21 | REGION="$(gcloud config get-value compute/region)" 22 | 23 | # obtain all backend service names 24 | STR="$(gcloud compute backend-services list | awk '{print $1}' | awk '!/NAME/')" 25 | 26 | # convert the names of backend services to a list 27 | ARR="$(echo "${STR}" | tr ' ' '\n')" 28 | 29 | # loop all the backend services 30 | for ITEM in $ARR 31 | do 32 | # find a backend service created by this application 33 | RES=$(gcloud compute backend-services describe "$ITEM" --region="$REGION" | grep -E "$LIST|group") 34 | if [[ $RES == *"description"* ]]; then 35 | 36 | # get the name of the backend service "group" (ie k8s-ig--d0bf239d98a9d918) 37 | # shellcheck disable=SC2034 38 | IFS=' ' read -r var1 var2 var3 <<< "$RES" 39 | NAME=${var2##*/} 40 | 41 | # wait for all backend services to be deleted in that "group" (max time 5 minutes) 42 | RETRIES_REMAINING=30 43 | while [[ $(gcloud compute backend-services list | grep "$NAME") != "" ]]; do 44 | if [[ $RETRIES_REMAINING -eq 0 ]]; then 45 | echo "Retry limit exceeded; exiting..." 46 | exit 1 47 | fi 48 | echo "Wait for the BACKEND $NAME being deleted." 49 | RETRIES_REMAINING=$(("$RETRIES_REMAINING" - 1)) 50 | sleep 10 51 | done 52 | echo "Service $NAME has been deleted." 53 | 54 | # delete the particular firewall rule (ie k8s-d0bf239d98a9d918-node-hc) 55 | gcloud compute firewall-rules delete "${NAME/-ig-/}-node-hc" --quiet >/dev/null 2>&1 56 | 57 | # wait for the firewall rules to be deleted (max time 5 minutes) 58 | RETRIES_REMAINING=30 59 | while [[ $(gcloud compute firewall-rules list --format=json | grep "${NAME/-ig-/}") != "" ]]; do 60 | if [[ $RETRIES_REMAINING -eq 0 ]]; then 61 | echo "Retry limit exceeded; exiting..." 62 | exit 1 63 | fi 64 | echo "Wait for the Firewall ${NAME/-ig-/} being deleted." 65 | RETRIES_REMAINING=$(("$RETRIES_REMAINING" - 1)) 66 | sleep 10 67 | done 68 | echo "Firewall ${NAME/-ig-/} has been deleted." 69 | exit 0 70 | fi 71 | done 72 | 73 | 74 | --------------------------------------------------------------------------------