├── .gitignore ├── README.md └── kubernetes ├── .env.example ├── Makefile ├── README.md ├── config └── ansible │ ├── Dockerfile │ ├── config.sh │ ├── create_and_upload_ansible_vars.yml │ ├── create_kubeconfig_for_service.yml │ ├── files │ └── ca-config.json │ ├── generate_certificates.yml │ ├── generate_single_certificate.yml │ ├── group_vars │ └── all │ │ └── vars.yml │ ├── post_deploy.sh │ ├── roles │ ├── common │ │ ├── files │ │ │ └── resolv.conf │ │ └── tasks │ │ │ └── main.yml │ ├── controller │ │ ├── defaults │ │ │ └── main.yml │ │ ├── files │ │ │ ├── clusterrole_kube_apiserver_to_kubelet.yaml │ │ │ └── clusterrolebinding_kube_apiserver_to_kubelet.yaml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ ├── encryption-config.yaml.tmpl │ │ │ ├── kube-apiserver.service.tmpl │ │ │ ├── kube-controller-manager.service.tmpl │ │ │ ├── kube-scheduler.service.tmpl │ │ │ ├── kube-scheduler.yaml.tmpl │ │ │ └── kubernetes.default.svc.cluster.local.tmpl │ ├── etcd │ │ ├── defaults │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── etcd.service.tmpl │ └── worker │ │ ├── defaults │ │ └── main.yml │ │ ├── files │ │ ├── containerd.service │ │ ├── containerd_config.toml │ │ ├── kube-proxy.service │ │ └── kubelet.service │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ ├── 10-bridge.conf.j2 │ │ ├── 99-loopback.conf.j2 │ │ ├── kube-proxy-config.yaml.j2 │ │ └── kubelet-config.yaml.j2 │ ├── setup.sh │ ├── site.yml │ ├── templates │ ├── admin-csr.json.tmpl │ ├── ansible_vars.yml.tmpl │ ├── ca-csr.json.tmpl │ ├── kube-controller-manager-csr.json.tmpl │ ├── kube-proxy-csr.json.tmpl │ ├── kube-scheduler-csr.json.tmpl │ ├── kubernetes-csr.json.tmpl │ ├── service-account-csr.json.tmpl │ └── worker-csr.json.tmpl │ └── test.sh ├── control_plane └── aws │ ├── data.tf │ ├── instances.tf │ ├── load_balancer.tf │ ├── networking.tf │ ├── security_groups.tf │ ├── tags │ ├── all.json │ ├── etcd.json │ ├── kubernetes_controller.json │ └── kubernetes_worker.json │ ├── templates │ └── user_data │ ├── terraform.tfvars.template │ └── variables.tf ├── include └── scripts │ ├── create_terraform_configuration.sh │ └── generate_tags.bash └── tools ├── Dockerfile ├── entrypoint.sh └── tests ├── 1_ensure_kubectl_present.bats └── 2_ensure_additional_binaries_are_present.bats /.gitignore: -------------------------------------------------------------------------------- 1 | # Terraform ignores. 2 | *.terraform 3 | *.tfstate 4 | *.tfplan 5 | *.tfstate.lock 6 | terraform.tfvars 7 | !terraform.tfvars.template 8 | provider.tf 9 | backend.tf 10 | 11 | # Environment variable ignores 12 | .env 13 | .env.* 14 | env.* 15 | !.env.example 16 | !env.tar.enc 17 | 18 | # Ansible ignores 19 | *.retry 20 | 21 | # Scratch directory 22 | scratch/ 23 | 24 | # Ignore binaries 25 | kubectl 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # What is this? 2 | 3 | This is the code powering the infrastructure for [carlosnunez.me](https://carlosnunez.me) 4 | -------------------------------------------------------------------------------- /kubernetes/.env.example: -------------------------------------------------------------------------------- 1 | # AWS primitives. 2 | AWS_REGION="change me" 3 | AWS_ACCESS_KEY_ID="change me" 4 | AWS_SECRET_ACCESS_KEY="change me" 5 | 6 | # The domain name to use for your Kubernetes cluster. 7 | # If you're using AWS, a Route 53 record will be created in the same zone 8 | # as DOMAIN_NAME for your control plane load balancer. 9 | DOMAIN_NAME="change me" 10 | 11 | # A unique environment name to use for storing deployment configurations 12 | # (Terraform states, certificates and SSH keys in S3, etc.) 13 | ENVIRONMENT_NAME="change me" 14 | 15 | # This deployment uses one cluster across multiple AZs. This specifies 16 | # the number of AZs to use for this cluster. 17 | # NOTE: This deployment will fail in awful ways if NUMBER_OF_AVAILABILITY_ZONES 18 | # exceeds the number of availability zones available to you in your region. 19 | # Interesting follow-up convo: https://github.com/kubernetes/kubernetes/issues/13056 20 | NUMBER_OF_AVAILABILITY_ZONES="change me" 21 | 22 | # The number of workers to deploy. This will deploy 23 | # NUMBER_OF_AVAILABILITY_ZONES*NUMBER_OF_AVAILABILITY_ZONES number of 24 | # instances. 25 | NUMBER_OF_WORKERS_PER_CLUSTER="change me" 26 | 27 | # The AWS S3 bucket to use for storing SSH keys. 28 | SSH_KEY_S3_BUCKET_NAME="change me" 29 | SSH_KEY_S3_KEY_PATH="change me" 30 | 31 | # The user name to use for SSHing into instances. 32 | SSH_USER_NAME="change me" 33 | 34 | # Information to use in the X.509 certificates created for your cluster. 35 | CA_KEY_S3_BUCKET_NAME="change me" 36 | CA_KEY_S3_KEY_PATH="change me" 37 | CA_CSR_CITY="change me" 38 | CA_CSR_STATE="change me" 39 | CA_CSR_COUNTRY_INITIALS="change me" 40 | CA_CSR_ORGANIZATION="change me" 41 | CA_CSR_COMMON_NAME="change me" 42 | 43 | # A name to use for your cluster. 44 | KUBERNETES_CLUSTER_NAME="change me" 45 | 46 | # The version of Kubernetes to use. 47 | KUBERNETES_VERSION="change me" 48 | 49 | # The CIDR to use for Pods. Only /16's are supported for this deployment. 50 | # Subnet CIDRs for your Pods will be calculated automatically. 51 | # The service IP defaults to 10.32.0.10. 52 | KUBERNETES_POD_CIDR="change me" 53 | 54 | # The public port to use for your Kubernetes load balancer. 55 | # Only 6443 was tested at this time of writing. 56 | KUBERNETES_PUBLIC_PORT="change me" 57 | 58 | # Because the configuration management used for this deployment 59 | # was designed to be agnostic, every node clones a copy of this repository 60 | # on first-boot and creates a systemd service to initialize this code. 61 | # (See how in the Makefile.) 62 | # This controls the branch that each node should use during this process. 63 | KUBERNETES_GIT_BRANCH="change me" 64 | 65 | # S3 buckets to use for storing certificates. 66 | # This deployment appends a unique token to each certificate that is then 67 | # copied to the user-data of each worker and controller. This prevents 68 | # nodes from getting stale certificates during re-deploys. 69 | # This is also not emptied out automatically, so you'll need to purge 70 | # this from time to time. 71 | KUBERNETES_CERTIFICATE_S3_BUCKET="change me" 72 | KUBERNETES_CERTIFICATE_S3_KEY="change me" 73 | 74 | # For those using the Ansible playbooks included in this repository, 75 | # these keys specify the S3 bucket to store auto-generated Ansible variables. 76 | ANSIBLE_VARS_S3_BUCKET="change me" 77 | ANSIBLE_VARS_S3_KEY="change me" 78 | 79 | # S3 buckets to use for storing Terraform state and other metadata. 80 | TERRAFORM_STATE_S3_BUCKET="change me" 81 | TERRAFORM_STATE_S3_KEY="change me" 82 | -------------------------------------------------------------------------------- /kubernetes/Makefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env make 2 | MAKEFLAGS += --always-make 3 | MAKEFLAGS += --silent 4 | SHELL := /usr/bin/env bash 5 | ENVIRONMENT_FILE ?= .env 6 | 7 | ifeq ("$(wildcard $(ENVIRONMENT_FILE))","") 8 | $(error "Please create a .env.") 9 | endif 10 | 11 | include $(ENVIRONMENT_FILE) 12 | export $(shell cat $(ENVIRONMENT_FILE)| grep "^\#" | cut -f1 -d '=') 13 | 14 | ifeq ($(AWS_ACCESS_KEY_ID),) 15 | $(error Please provide an access key to an AWS account) 16 | endif 17 | ifeq ($(AWS_SECRET_ACCESS_KEY),) 18 | $(error Please provide a secret key to an AWS account) 19 | endif 20 | ifeq ($(TERRAFORM_STATE_S3_BUCKET),) 21 | $(error Please provide the bucket into which the Terraform state will be stored) 22 | endif 23 | ifeq ($(TERRAFORM_STATE_S3_KEY),) 24 | $(error Please provide the S3 key for the TERRAFORM_STATE_S3_BUCKET) 25 | endif 26 | ifeq ($(KUBERNETES_POD_CIDR),) 27 | $(error Please define a CIDR to use for allocating Pods within your cluster.) 28 | endif 29 | 30 | PROJECT_PATH := $(shell echo "$$(git rev-parse --show-toplevel)")/kubernetes 31 | SCRIPTS_PATH := $(PROJECT_PATH)/include/scripts 32 | AWS_REGION ?= us-east-1 33 | BATS_DOCKER_IMAGE ?= graze/bats 34 | CERTIFICATE_DOWNLOAD_DIRECTORY ?= /tmp 35 | CONFIG_MGMT_IMAGE ?= williamyeh/ansible:alpine3 36 | CONFIG_MGMT_LINT_TOOL_COMMAND ?= sh -c 'find /work/ansible -type f -name "*.yml" | xargs -r ansible-lint --force-color' 37 | CONFIG_MGMT_LINT_TOOL_IMAGE ?= yokogawa/ansible-lint 38 | CONFIG_MGMT_PATH := $(PROJECT_PATH)/config 39 | CONFIG_MGMT_TOOL ?= ansible 40 | CONTROL_PLANE_SOURCE_PATH := $(PROJECT_PATH)/control_plane 41 | DOCKER_HUB_USERNAME ?= carlosonunez 42 | DRY_RUN ?= false 43 | ENVIRONMENT_NAME ?= develop 44 | KUBERNETES_GIT_BRANCH ?= master 45 | KUBERNETES_STARTUP_DELAY_MINUTES ?= 10 46 | KUBERNETES_TOOLS_DOCKER_IMAGE_TAG ?= k8s_tools 47 | KUBERNETES_TOOLS_IMAGE_NAME := $(DOCKER_HUB_USERNAME)/$(KUBERNETES_TOOLS_DOCKER_IMAGE_TAG) 48 | KUBERNETES_TOOLS_SOURCE_PATH := $(PROJECT_PATH)/tools 49 | NUMBER_OF_AVAILABILITY_ZONES ?= 3 50 | PRIVATE_KEY_PATH_VARIABLE := temporary_key_path 51 | SKIP_BUILDING_IMAGES ?= false 52 | SKIP_LOCAL_ENVIRONMENT_PREP ?= false 53 | SSH_DELAY_IN_SECONDS ?= 60 54 | TERRAFORM_DOCKER_IMAGE ?= hashicorp/terraform:0.11.9 55 | 56 | .PHONY: \ 57 | deploy_cluster \ 58 | deploy_cluster \ 59 | deploy_tools_image \ 60 | destroy_cluster \ 61 | generate_certificate_token \ 62 | get_certificate_token \ 63 | recycle_cluster \ 64 | recycle_nodes \ 65 | recycle_controllers \ 66 | recycle_workers \ 67 | recycle_etcd_cluster 68 | 69 | # TODO: Add certificate stuff back 70 | deploy_cluster: 71 | $(MAKE) generate_certificate_token && \ 72 | $(MAKE) build_config_mgmt_image && \ 73 | $(MAKE) prepare_local_environment && \ 74 | $(MAKE) configure_terraform && \ 75 | $(MAKE) configure_terraform_tags && \ 76 | TERRAFORM_ARGS=-force-copy $(MAKE) terraform_init && \ 77 | $(MAKE) terraform_validate && \ 78 | $(MAKE) deploy_terraform_plan && \ 79 | $(MAKE) disable_source_dest_checks && \ 80 | $(MAKE) perform_local_post_deployment_steps && \ 81 | $(MAKE) check_that_kubernetes_has_started && \ 82 | $(MAKE) build_tools_image && \ 83 | $(MAKE) create_kubectl_convenience_script && \ 84 | echo "Your Kubernetes cluster is ready at $$($(MAKE) get_kubernetes_api_public_address)." 85 | 86 | destroy_cluster: 87 | $(MAKE) configure_terraform && \ 88 | $(MAKE) configure_terraform_tags && \ 89 | $(MAKE) destroy_terraform_plan 90 | 91 | generate_certificate_token: 92 | token=$$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 8); \ 93 | if ! cat $(ENVIRONMENT_FILE) | grep -q CERT_TOKEN; \ 94 | then \ 95 | echo "CERT_TOKEN=$$token" >> $(ENVIRONMENT_FILE); \ 96 | else \ 97 | sed -i "s/CERT_TOKEN=.*/CERT_TOKEN=$$token/" $(ENVIRONMENT_FILE); \ 98 | fi 99 | 100 | get_certificate_token: 101 | cat $(ENVIRONMENT_FILE) | grep CERT_TOKEN | cut -f2 -d = 102 | 103 | recycle_cluster: 104 | $(MAKE) destroy_cluster && $(MAKE) deploy_cluster 105 | 106 | recycle_nodes: 107 | $(MAKE) configure_terraform && \ 108 | $(MAKE) configure_terraform_tags && \ 109 | targets=$$(TERRAFORM_ARGS=list make terraform_state | \ 110 | tr -d $$'\r' | \ 111 | grep 'aws_spot_instance' | \ 112 | sed 's/^/-target=/' | \ 113 | tr "\n" " "); \ 114 | TERRAFORM_ARGS="-auto-approve -force $$targets" $(MAKE) terraform_destroy && \ 115 | $(MAKE) deploy_cluster 116 | 117 | recycle_controllers: 118 | $(MAKE) configure_terraform && \ 119 | $(MAKE) configure_terraform_tags && \ 120 | targets=$$(TERRAFORM_ARGS=list make terraform_state | \ 121 | tr -d $$'\r' | \ 122 | grep -E 'aws_spot_instance.*kubernetes_control_plane.*' | \ 123 | sed 's/^/-target=/' | \ 124 | tr "\n" " "); \ 125 | TERRAFORM_ARGS="-auto-approve -force $$targets" $(MAKE) terraform_destroy && \ 126 | $(MAKE) deploy_cluster 127 | 128 | recycle_workers: 129 | $(MAKE) configure_terraform && \ 130 | $(MAKE) configure_terraform_tags && \ 131 | targets=$$(TERRAFORM_ARGS=list make terraform_state | \ 132 | tr -d $$'\r' | \ 133 | grep -E 'aws_spot_instance.*kubernetes_workers.*' | \ 134 | sed 's/^/-target=/' | \ 135 | tr "\n" " "); \ 136 | TERRAFORM_ARGS="-auto-approve -force $$targets" $(MAKE) terraform_destroy && \ 137 | $(MAKE) deploy_cluster 138 | 139 | recycle_etcd_cluster: 140 | $(MAKE) configure_terraform && \ 141 | $(MAKE) configure_terraform_tags && \ 142 | targets=$$(TERRAFORM_ARGS=list make terraform_state | \ 143 | tr -d $$'\r' | \ 144 | grep -E 'aws_spot_instance.*etcd.*' | \ 145 | sed 's/^/-target=/' | \ 146 | tr "\n" " "); \ 147 | TERRAFORM_ARGS="-auto-approve -force $$targets" $(MAKE) terraform_destroy && \ 148 | $(MAKE) deploy_cluster 149 | 150 | 151 | .PHONY: \ 152 | build_config_mgmt_image \ 153 | lint_configuration \ 154 | test_configuration \ 155 | build_config_mgmt_image \ 156 | perform_local_post_deployment_steps \ 157 | prepare_local_environment \ 158 | run_configuration_manually \ 159 | disable_source_dest_checks 160 | 161 | disable_source_dest_checks: 162 | aws ec2 describe-instances \ 163 | --filter "Name=instance-state-name,Values=running" \ 164 | --query "Reservations[*].Instances[*].InstanceId" \ 165 | --output text | \ 166 | xargs -I {} aws ec2 modify-instance-attribute \ 167 | --instance-id {} \ 168 | --no-source-dest-check 169 | 170 | build_config_mgmt_image: 171 | if [ "$(SKIP_BUILDING_IMAGES)" == "true" ]; \ 172 | then \ 173 | exit 0; \ 174 | fi; \ 175 | image_name=$(DOCKER_HUB_USERNAME)/$(CONFIG_MGMT_TOOL); \ 176 | context_path=$(CONFIG_MGMT_PATH)/$(CONFIG_MGMT_TOOL); \ 177 | >&2 echo "INFO: Building Ansible image: $$image_name"; \ 178 | docker build -q -t $$image_name \ 179 | -f $$context_path/Dockerfile \ 180 | $$context_path >/dev/null 181 | 182 | lint_configuration: 183 | docker run --tty \ 184 | --rm \ 185 | --volume $$PWD/config:/work \ 186 | "$(CONFIG_MGMT_LINT_TOOL_IMAGE)" $(CONFIG_MGMT_LINT_TOOL_COMMAND) 187 | 188 | run_configuration_manually: 189 | image_name=$(DOCKER_HUB_USERNAME)/$(CONFIG_MGMT_TOOL); \ 190 | docker run --tty \ 191 | --interactive \ 192 | --rm \ 193 | --env "CONFIG_MGMT_DOCKER_IMAGE=$$image_name" \ 194 | --env "CONFIG_MGMT_CODE_PATH=$(CONFIG_MGMT_PATH)/$(CONFIG_MGMT_TOOL)" \ 195 | --env "AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID)" \ 196 | --env "AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY)" \ 197 | --env "AWS_REGION=$(AWS_REGION)" \ 198 | --env-file $(ENVIRONMENT_FILE) \ 199 | --entrypoint sh \ 200 | --volume $$(which docker):/usr/local/bin/docker \ 201 | --volume /var/run/docker.sock:/var/run/docker.sock \ 202 | --volume $$PWD/config/$(CONFIG_MGMT_TOOL):/work \ 203 | --workdir /work \ 204 | "$$image_name" 205 | 206 | test_configuration: 207 | image_name=$(DOCKER_HUB_USERNAME)/$(CONFIG_MGMT_TOOL); \ 208 | docker run --tty \ 209 | --rm \ 210 | --env "CONFIG_MGMT_DOCKER_IMAGE=$$image_name" \ 211 | --env "CONFIG_MGMT_CODE_PATH=$(CONFIG_MGMT_PATH)/$(CONFIG_MGMT_TOOL)" \ 212 | --env "AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID)" \ 213 | --env "AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY)" \ 214 | --env "AWS_REGION=$(AWS_REGION)" \ 215 | --env-file $(ENVIRONMENT_FILE) \ 216 | --entrypoint sh \ 217 | --volume $$(which docker):/usr/local/bin/docker \ 218 | --volume /var/run/docker.sock:/var/run/docker.sock \ 219 | --volume $$PWD/config/$(CONFIG_MGMT_TOOL):/work \ 220 | --workdir /work \ 221 | "$$image_name" -c "/work/test.sh $(KUBERNETES_ROLE_TO_TEST)" 222 | 223 | perform_local_post_deployment_steps: 224 | image_name=$(DOCKER_HUB_USERNAME)/$(CONFIG_MGMT_TOOL); \ 225 | docker run --tty \ 226 | --rm \ 227 | --env "CONFIG_MGMT_DOCKER_IMAGE=$$image_name" \ 228 | --env "CONFIG_MGMT_CODE_PATH=$(CONFIG_MGMT_PATH)/$(CONFIG_MGMT_TOOL)" \ 229 | --env "AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID)" \ 230 | --env "AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY)" \ 231 | --env "AWS_REGION=$(AWS_REGION)" \ 232 | --env-file $(ENVIRONMENT_FILE) \ 233 | --entrypoint sh \ 234 | --volume $$(which docker):/usr/local/bin/docker \ 235 | --volume /var/run/docker.sock:/var/run/docker.sock \ 236 | --volume $$PWD/config/$(CONFIG_MGMT_TOOL):/work \ 237 | --workdir /work \ 238 | "$$image_name" -c "/work/post_deploy.sh" 239 | 240 | prepare_local_environment: 241 | if [ "$(SKIP_LOCAL_ENVIRONMENT_PREP)" == "true" ]; \ 242 | then \ 243 | exit 0; \ 244 | fi; \ 245 | image_name=$(DOCKER_HUB_USERNAME)/$(CONFIG_MGMT_TOOL); \ 246 | docker run --tty \ 247 | --rm \ 248 | --env "CONFIG_MGMT_DOCKER_IMAGE=$$image_name" \ 249 | --env "CONFIG_MGMT_CODE_PATH=$(CONFIG_MGMT_PATH)/$(CONFIG_MGMT_TOOL)" \ 250 | --env "AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID)" \ 251 | --env "AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY)" \ 252 | --env "AWS_REGION=$(AWS_REGION)" \ 253 | --env-file $(ENVIRONMENT_FILE) \ 254 | --entrypoint sh \ 255 | --volume $$(which docker):/usr/local/bin/docker \ 256 | --volume /var/run/docker.sock:/var/run/docker.sock \ 257 | --volume $$PWD/config/$(CONFIG_MGMT_TOOL):/work \ 258 | --workdir /work \ 259 | "$$image_name" -c "/work/setup.sh $(ADDITIONAL_SETUP_OPTIONS)" 260 | 261 | .PHONY: \ 262 | get_control_plane_addresses \ 263 | get_worker_addresses \ 264 | get_etcd_node_addresses \ 265 | get_kubernetes_api_public_address \ 266 | check_that_kubernetes_has_started 267 | 268 | get_kubernetes_api_public_address: 269 | TERRAFORM_ARGS="kubernetes_control_plane_dns_address" $(MAKE) terraform_output | \ 270 | tr -d $$'\r' 271 | 272 | get_worker_addresses: 273 | aws ec2 describe-instances --filter "Name=tag:kubernetes_role,Values=worker" \ 274 | "Name=instance-state-name,Values=running" \ 275 | --query "Reservations[*].Instances[*].PublicIpAddress[]" --output text | \ 276 | sed 's/[ \t]\+/ /g' | \ 277 | tr ' ' '\n' 278 | 279 | get_control_plane_addresses: 280 | aws ec2 describe-instances --filter "Name=tag:kubernetes_role,Values=controller" \ 281 | "Name=instance-state-name,Values=running" \ 282 | --query "Reservations[*].Instances[*].PublicIpAddress[]" --output text | \ 283 | sed 's/[ \t]\+/ /g' | \ 284 | tr ' ' '\n' 285 | 286 | get_etcd_node_addresses: 287 | aws ec2 describe-instances --filter "Name=tag:kubernetes_role,Values=etcd" \ 288 | "Name=instance-state-name,Values=running" \ 289 | --query "Reservations[*].Instances[*].PublicIpAddress[]" --output text | \ 290 | sed 's/[ \t]\+/ /g' | \ 291 | tr ' ' '\n' 292 | 293 | check_that_kubernetes_has_started: 294 | echo "Waiting for Kubernetes to start. \ 295 | This can take up to $(KUBERNETES_STARTUP_DELAY_MINUTES) minutes to complete."; \ 296 | max_seconds_to_wait=$$(($(KUBERNETES_STARTUP_DELAY_MINUTES)*60)); \ 297 | seconds_elapsed=0; \ 298 | while [ "$$seconds_elapsed" -lt "$$max_seconds_to_wait" ]; \ 299 | do \ 300 | kubernetes_address=$$($(MAKE) get_control_plane_addresses | sort -R | head -1); \ 301 | if ! nc -z "$$kubernetes_address" 22 &>/dev/null; \ 302 | then \ 303 | >&2 echo "WARN: Kubernetes hasn't started yet at: $$kubernetes_address"; \ 304 | sleep 30; \ 305 | seconds_elapsed=$$((seconds_elapsed+30)); \ 306 | else \ 307 | seconds_elapsed=0; \ 308 | echo -ne '\007'; \ 309 | >&2 echo "INFO: You can now SSH into the Kubernetes control plane. Waiting $$max_seconds_to_wait seconds for kube-apiserver"; \ 310 | kubernetes_api_address=$$($(MAKE) get_kubernetes_api_public_address); \ 311 | while [ "$$seconds_elapsed" -lt "$$max_seconds_to_wait" ]; \ 312 | do \ 313 | if curl -skL https://$$kubernetes_api_address:6443/version | grep -q "gitCommit"; \ 314 | then \ 315 | for i in $$(seq 1 3); \ 316 | do \ 317 | echo -ne '\007'; \ 318 | sleep 0.25; \ 319 | done; \ 320 | >&2 echo "INFO: Kubernetes is ready."; \ 321 | exit 0; \ 322 | else \ 323 | >&2 echo "WARN: kube-apiserver isn't available yet."; \ 324 | sleep 30; \ 325 | seconds_elapsed=$$((seconds_elapsed+30)); \ 326 | fi; \ 327 | done; \ 328 | fi; \ 329 | done; 330 | 331 | wait_for_ssh: 332 | >&2 echo "Waiting $(SSH_DELAY_IN_SECONDS) seconds for hosts to accept SSH connections."; \ 333 | sleep $(SSH_DELAY_IN_SECONDS) 334 | 335 | .PHONY: \ 336 | configure_terraform \ 337 | configure_terraform_tags \ 338 | get_private_key_path \ 339 | terraform_% \ 340 | deploy_terraform_plan \ 341 | destroy_terraform_plan \ 342 | wait_for_ssh 343 | 344 | configure_terraform: 345 | PRIVATE_KEY_PATH_VARIABLE=$(PRIVATE_KEY_PATH_VARIABLE) \ 346 | ENV_FILE=$(ENVIRONMENT_FILE) \ 347 | $(SCRIPTS_PATH)/create_terraform_configuration.sh 348 | 349 | configure_terraform_tags: 350 | TAGS_PATH=$(CONTROL_PLANE_SOURCE_PATH)/aws/tags \ 351 | TERRAFORM_TFVARS_PATH=$(CONTROL_PLANE_SOURCE_PATH)/aws/terraform.tfvars \ 352 | ENV_FILE=$(ENVIRONMENT_FILE) \ 353 | $(SCRIPTS_PATH)/generate_tags.bash 354 | 355 | get_private_key_path: 356 | cat $(CONTROL_PLANE_SOURCE_PATH)/aws/terraform.tfvars | \ 357 | grep -E "^$(PRIVATE_KEY_PATH_VARIABLE)" | \ 358 | cut -f2 -d "=" | \ 359 | tr -d '"' 360 | 361 | terraform_%: 362 | action=$$(echo $@ | sed 's/terraform_//') && \ 363 | docker run --tty \ 364 | --rm --volume "$(CONTROL_PLANE_SOURCE_PATH)/aws:/infra" \ 365 | --env-file .env \ 366 | --env AWS_ACCESS_KEY_ID \ 367 | --env AWS_SECRET_ACCESS_KEY \ 368 | --env AWS_REGION \ 369 | --workdir /infra \ 370 | $(TERRAFORM_DOCKER_IMAGE) $$action $(TERRAFORM_ARGS) 371 | 372 | deploy_terraform_plan: 373 | if [ "$(DRY_RUN)" == "true" ]; \ 374 | then \ 375 | $(MAKE) terraform_plan; \ 376 | else \ 377 | TERRAFORM_ARGS="-auto-approve" $(MAKE) terraform_apply; \ 378 | fi 379 | 380 | destroy_terraform_plan: 381 | if [ "$(DRY_RUN)" == "true" ]; \ 382 | then \ 383 | $(MAKE) terraform_destroy; \ 384 | else \ 385 | TERRAFORM_ARGS=-auto-approve $(MAKE) terraform_destroy; \ 386 | fi 387 | 388 | .PHONY: \ 389 | build_tools_image \ 390 | test_tools_image \ 391 | deploy_tools_image 392 | 393 | build_tools_image: 394 | if [ "$(SKIP_BUILDING_IMAGES)" == "true" ]; \ 395 | then \ 396 | exit 0; \ 397 | fi; \ 398 | >&2 echo "INFO: Building Kubernetes tools image: $(KUBERNETES_TOOLS_IMAGE_NAME)"; \ 399 | docker build -q -t $(KUBERNETES_TOOLS_IMAGE_NAME) \ 400 | -f $(KUBERNETES_TOOLS_SOURCE_PATH)/Dockerfile \ 401 | $(KUBERNETES_TOOLS_SOURCE_PATH) >/dev/null && \ 402 | $(MAKE) test_tools_image 403 | 404 | test_tools_image: 405 | docker run --tty --rm --volume $(KUBERNETES_TOOLS_SOURCE_PATH):/app \ 406 | --volume /var/run/docker.sock:/var/run/docker.sock \ 407 | --workdir /app \ 408 | --env DOCKER_IMAGE_UNDER_TEST=$(KUBERNETES_TOOLS_IMAGE_NAME) \ 409 | $(BATS_DOCKER_IMAGE) /app/tests 410 | 411 | .PHONY: \ 412 | ssh_into_kubernetes_worker \ 413 | ssh_into_kubernetes_controller \ 414 | ssh_into_etcd_node 415 | 416 | ssh_into_kubernetes_worker: 417 | $(MAKE) get_worker_addresses | \ 418 | sort -R | \ 419 | head -1 | \ 420 | xargs -I {} echo "ssh -i /tmp/kubernetes_cluster_ssh_key $(SSH_USER_NAME)@{}" 421 | 422 | ssh_into_kubernetes_controller: 423 | $(MAKE) get_control_plane_addresses | \ 424 | sort -R | \ 425 | head -1 | \ 426 | xargs -I {} echo "ssh -i /tmp/kubernetes_cluster_ssh_key $(SSH_USER_NAME)@{}" 427 | 428 | ssh_into_etcd_node: 429 | $(MAKE) get_etcd_node_addresses | \ 430 | sort -R | \ 431 | head -1 | \ 432 | xargs -I {} echo "ssh -i /tmp/kubernetes_cluster_ssh_key $(SSH_USER_NAME)@{}" 433 | 434 | .PHONY: \ 435 | get_kubernetes_certificates \ 436 | create_kubectl_convenience_script 437 | 438 | get_kubernetes_certificates: 439 | certificate_download_directory=$(CERTIFICATE_DOWNLOAD_DIRECTORY); \ 440 | cert_token=$$($(MAKE) get_certificate_token); \ 441 | for cert_name in ca admin admin-key; \ 442 | do \ 443 | rm $(CERTIFICATE_DOWNLOAD_DIRECTORY)/$$cert_name.pem 2>/dev/null; \ 444 | aws s3 cp \ 445 | s3://$(KUBERNETES_CERTIFICATE_S3_BUCKET)/$(KUBERNETES_CERTIFICATE_S3_KEY)/$(ENVIRONMENT_NAME)/$${cert_token}-$${cert_name}.pem \ 446 | "$${certificate_download_directory}/$${cert_name}.pem"; \ 447 | done 448 | 449 | create_kubectl_convenience_script: 450 | $(MAKE) get_kubernetes_certificates && \ 451 | kubectl_script_location=$$PWD; \ 452 | rm -f $$kubectl_script_location/kubectl && \ 453 | echo "docker run -it --env USER_CERT_PRIVATE_KEY=/certs/admin-key.pem \ 454 | --env KUBERNETES_CLUSTER_ADDRESS=$$($(MAKE) get_kubernetes_api_public_address) \ 455 | --env KUBERNETES_CLUSTER_NAME=$(KUBERNETES_CLUSTER_NAME) \ 456 | --volume $(CERTIFICATE_DOWNLOAD_DIRECTORY):/certs \ 457 | $(KUBERNETES_TOOLS_IMAGE_NAME) kubectl \$$@" > "$$kubectl_script_location/kubectl" && \ 458 | chmod +x $$kubectl_script_location/kubectl && \ 459 | >&2 echo "INFO: kubectl is located at: $$kubectl_script_location/kubectl." 460 | -------------------------------------------------------------------------------- /kubernetes/README.md: -------------------------------------------------------------------------------- 1 | # What is this? 2 | 3 | This project will help you deploy a basic Kubernetes cluster per [Kubernetes the Hard Way](https://github.com/kelseyhightower/kubernetes-the-hard-way) 4 | on AWS using Spot Instances. Using `t2.medium` instances, this toy cluster will cost you less than $1/day. 5 | 6 | A future commit will enable examples for other cloud providers aside from GCP. 7 | 8 | # When Should I Use This? 9 | 10 | ✓ You're trying to pass the [Certified Kubernetes Administrator](https://www.cncf.io/certification/cka/) exam. 11 | 12 | ✓ You're interested in seeing how absolutely involved a Kubernetes setup is so you can appreciate [kops](https://github.com/kubernetes/kops) or [kubespray](https://github.com/kubernetes-incubator/kubespray) more, 13 | 14 | ✓ You're trying to deploy Kubernetes onto a platform that's not supported by other installers and are looking for a starting point. 15 | 16 | ✓ You hate yourself. 17 | 18 | ✓ You've won the cryptocurrency lottery, are now retired and find yourself with many, many, *many* hours to burn. 19 | 20 | # When Shouldn't I Use This? 21 | 22 | **Any other time.** 23 | 24 | # Really? 25 | 26 | Yes. Use kops, kubespray, or a managed Kubernetes offering. 27 | 28 | # Why? 29 | 30 | - This codebase does not provision triggers to account for random nodes dying due to getting outbid on the Spot market. 31 | Deploying this will likely cause split-brains, randomly-dying `Pod`s, CNI black holes and other misadventures. 32 | - kube-dns also seems to be broken. Records will resolve for `Pod`s that live on the same node as the kube-dns `Pod`. 33 | This happens because the bridge network created by CNI is unable to find a route from workers that are not 34 | hosting kube-dns to workers that are, and kube-dns is not a `DaemonSet`. 35 | - None of this is tested at all, unfortunately, outside of a few verifications made by Ansible playbooks. 36 | 37 | # Still interested in using this? 38 | 39 | Cool! Here are some instructions to help you out. 40 | 41 | ## Deploying 42 | 43 | **NOTE**: If you are using a Mac, ensure that you are using the GNU version of sed by installing 44 | `brew install gnu-sed` on your Mac and aliasing `sed` to `$(which gsed)`. 45 | 46 | 1. Copy the `.env.example`: `cat .env.example | grep -Ev '^#' > .env` 47 | 2. Fill in the environment variables provided with your desired values. 48 | 3. Run `make deploy_cluster`. You will hear one bell when SSH access is available and three bells when Kubernetes is accessible. 49 | **NOTE**: This takes between 5-10 minutes to provision. 50 | 4. Confirm that your cluster is available: `kubectl get cs` 51 | 52 | ## Destroying 53 | 54 | Run `make destroy_cluster` to wipe your cluster. You'll hear one bell when this is done. 55 | 56 | **NOTE**: This takes 2-5 minutes to complete. 57 | 58 | ## Troubleshooting 59 | 60 | There are several Make targets that can help you in your (unfortunately) inevitable troubleshooting efforts. 61 | 62 | - `get_control_plane_addresses`: Retrieves IP addresses for all Kubernetes controllers 63 | - `get_worker_addresses`: Same as above, but for workers. 64 | - `get_etcd_node_addresses`: Same as above, but for your `etcd` cluster. 65 | - `ssh_into_kubernetes_controller`: `echo`s a SSH command to run to SSH into any Kubernetes controller 66 | - `ssh_into_kubernetes_worker`: Same, but for workers. 67 | - `ssh_into_etcd_node`: Same, but for `etcd` nodes. 68 | - `recycle_nodes`: Deletes and recreates Kubernetes nodes while keeping all other infrastructure intact. 69 | - `recycle_cluster`: Same as `destroy_cluster` and `deploy_cluster`. 70 | - `recycle_controllers`: Same as `recycle_nodes`, but for controllers. 71 | - `recycle_workers`: Same as `recycle_nodes`, but for workers. 72 | - `recycle_etcd_nodes`: Same as `recycle_nodes`, but for etcd nodes. 73 | - `run_configuration_manually`: Drops into the container for your desired configuration management tool to run code manually. 74 | 75 | # Maintenance and Support 76 | 77 | I'll gladly accept PRs to fix things that I've missed. However, please do not expect any sort of timely support. 78 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM williamyeh/ansible:ubuntu16.04 2 | MAINTAINER Carlos Nunez 3 | 4 | RUN apt update && \ 5 | apt -y install dbus && \ 6 | mkdir -p /run/dbus && \ 7 | pip install docker-py boto3 botocore 8 | 9 | CMD ["/usr/sbin/init"] 10 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if ! { 4 | sudo apt -yq update && 5 | sudo apt -yq install software-properties-common && 6 | sudo apt-add-repository ppa:ansible/ansible && 7 | sudo apt -yq update && 8 | sudo apt -yq install ansible; 9 | } 10 | then 11 | >&2 echo "ERROR: Failed to install Ansible." 12 | return 1 13 | fi 14 | 15 | pushd "$( dirname "${BASH_SOURCE[0]}" )" 16 | ansible-playbook \ 17 | -e @/etc/ansible_vars.yml \ 18 | -s site.yml 19 | popd 20 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/create_and_upload_ansible_vars.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: localhost 4 | connection: local 5 | tasks: 6 | - name: Delete the vars file in S3 if we're regenerating 7 | aws_s3: 8 | mode: delobj 9 | bucket: "{{ ansible_env.ANSIBLE_VARS_S3_BUCKET }}" 10 | object: "{{ ansible_env.ANSIBLE_VARS_S3_KEY }}/{{ ansible_env.ENVIRONMENT_NAME }}/ansible_vars.yml" 11 | 12 | - name: Parse Ansible vars template 13 | template: 14 | src: templates/ansible_vars.yml.tmpl 15 | dest: /tmp/ansible_vars.yml 16 | 17 | - name: Upload template to s3 18 | aws_s3: 19 | bucket: "{{ ansible_env.ANSIBLE_VARS_S3_BUCKET }}" 20 | object: "{{ ansible_env.ANSIBLE_VARS_S3_KEY }}/{{ ansible_env.ENVIRONMENT_NAME }}/ansible_vars.yml" 21 | src: /tmp/ansible_vars.yml 22 | mode: put 23 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/create_kubeconfig_for_service.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Show us the kubeconfig we're creating 3 | debug: 4 | msg: "Creating kubeconfig: {{ kubeconfig_name }}" 5 | 6 | - name: Ensure certificate authority is present. 7 | stat: 8 | path: "{{ certificate_authority_certificate_location }}/ca.pem" 9 | failed_when: False 10 | 11 | - name: Create kubeconfig 12 | shell: "kubectl config {{ item }} --kubeconfig={{ kubeconfig_location }}/{{ kubeconfig_name }}.kubeconfig" 13 | register: kubectl_result 14 | failed_when: kubectl_result.rc != 0 15 | with_items: 16 | - "set-cluster {{ cluster_name }} --certificate-authority={{ certificate_authority_certificate_location }}/ca.pem --embed-certs=true --server={{ kubernetes_control_plane_server }}" 17 | - "set-credentials {{ user }} --client-certificate={{ client_certificate }} --client-key={{ client_key }} --embed-certs=true" 18 | - "set-context default --cluster={{ cluster_name }} --user={{ user }}" 19 | - "use-context default" 20 | 21 | - name: Verify that the kubeconfig was created 22 | stat: 23 | path: "{{ kubeconfig_location }}/{{ kubeconfig_name }}.kubeconfig" 24 | failed_when: False 25 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/files/ca-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "signing": { 3 | "default": { 4 | "expiry": "8760h" 5 | }, 6 | "profiles": { 7 | "kubernetes": { 8 | "usages": ["signing", "key encipherment", "server auth", "client auth"], 9 | "expiry": "8760h" 10 | } 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/generate_certificates.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - hosts: localhost 3 | connection: local 4 | roles: 5 | - common 6 | 7 | - hosts: localhost 8 | connection: local 9 | tasks: 10 | - name: Ensure that we have all of the env vars that we need 11 | fail: 12 | msg: "{{ item }} is not defined. Please define it in your .env." 13 | when: lookup('env', item) == "" 14 | with_items: 15 | - AWS_ACCESS_KEY_ID 16 | - AWS_SECRET_ACCESS_KEY 17 | - KUBERNETES_CERTIFICATE_S3_BUCKET 18 | - KUBERNETES_CERTIFICATE_S3_KEY 19 | - ENVIRONMENT_NAME 20 | 21 | - name: Expose the AWS S3 bucket name to store our certs in 22 | set_fact: 23 | environment_name: "{{ lookup('env', 'ENVIRONMENT_NAME') }}" 24 | s3_bucket: "{{ lookup('env', 'KUBERNETES_CERTIFICATE_S3_BUCKET') }}" 25 | s3_key: "{{ lookup('env', 'KUBERNETES_CERTIFICATE_S3_KEY') }}" 26 | 27 | - name: Provision our certificates. 28 | include: generate_single_certificate.yml 29 | vars: 30 | certificate_s3_bucket: "{{ s3_bucket }}" 31 | certificate_s3_key: "{{ s3_key }}" 32 | loop_control: 33 | loop_var: certificate_csr_name 34 | loop: 35 | - ca 36 | - admin 37 | - kube-controller-manager 38 | - kube-proxy 39 | - kube-scheduler 40 | - service-account 41 | - kubernetes 42 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/generate_single_certificate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Retrieve our certificate token 3 | set_fact: 4 | certificate_token: "{{ lookup('env', 'CERT_TOKEN') }}" 5 | failed_when: certificate_token == "" 6 | 7 | - name: Set the name to use for creating the cert. 8 | when: certificate_csr_name != "worker" 9 | set_fact: 10 | certificate_file_name: "{{ certificate_csr_name }}" 11 | 12 | - name: Set the name to use for our cert as the host's hostname. 13 | when: certificate_csr_name == "worker" 14 | set_fact: 15 | certificate_file_name: "{{ system_hostname }}" 16 | 17 | - name: Delete existing certificate 18 | aws_s3: 19 | bucket: "{{ certificate_s3_bucket }}" 20 | object: "{{ certificate_s3_key }}/{{ environment_name }}/{{ item }}" 21 | mode: delobj 22 | with_items: 23 | - "{{ certificate_file_name }}.pem" 24 | - "{{ certificate_file_name }}-key.pem" 25 | 26 | 27 | - name: Get all running instances within our cluster. 28 | when: certificate_csr_name == "kubernetes" 29 | ec2_instance_facts: 30 | filters: 31 | instance-state-name: running 32 | register: found_instances 33 | 34 | - name: Create a list of IP addresses 35 | when: certificate_csr_name == "kubernetes" 36 | set_fact: 37 | cluster_ip_addresses_string: "{% for instance in found_instances.instances %}\"{{instance.private_ip_address}}\",{% endfor %}" 38 | 39 | - name: Remove last comma in IP address string. 40 | when: certificate_csr_name == "kubernetes" 41 | set_fact: 42 | cluster_ip_addresses_string: "{{ cluster_ip_addresses_string[:-1] }}" 43 | 44 | - name: Generate our cfssl config 45 | copy: 46 | src: "{{ ansible_home_directory }}/files/ca-config.json" 47 | dest: "{{ local_certificate_storage_path }}/ca-config.json" 48 | 49 | - name: Generate signing request 50 | template: 51 | src: "{{ ansible_home_directory }}/templates/{{ certificate_csr_name }}-csr.json.tmpl" 52 | dest: "{{ local_certificate_storage_path }}/{{ certificate_csr_name }}-csr.json" 53 | 54 | - name: Provision the certificate authority. 55 | when: certificate_csr_name == "ca" 56 | shell: cfssl gencert -initca {{ certificate_csr_name }}-csr.json | cfssljson -bare {{ certificate_file_name }} 57 | args: 58 | chdir: "{{ local_certificate_storage_path }}" 59 | register: ca_result 60 | failed_when: ca_result.rc != 0 61 | 62 | - name: Provision the certificate. 63 | when: certificate_csr_name != "ca" 64 | shell: cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes {{ certificate_csr_name }}-csr.json | cfssljson -bare {{ certificate_file_name }} 65 | args: 66 | chdir: "{{ local_certificate_storage_path }}" 67 | register: cert_provision_result 68 | failed_when: cert_provision_result.rc != 0 69 | 70 | - name: Check that the certificate and private key were generated. 71 | stat: 72 | path: "{{ local_certificate_storage_path }}/{{ item }}" 73 | with_items: 74 | - "{{ certificate_file_name }}.pem" 75 | - "{{ certificate_file_name }}-key.pem" 76 | register: found_certificate_components 77 | failed_when: not found_certificate_components.stat.exists 78 | 79 | - name: Upload certificate into AWS S3 80 | aws_s3: 81 | bucket: "{{ certificate_s3_bucket }}" 82 | object: "{{ certificate_s3_key }}/{{ environment_name }}/{{ certificate_token }}-{{ item }}" 83 | src: "{{ local_certificate_storage_path }}/{{ item }}" 84 | mode: put 85 | with_items: 86 | - "{{ certificate_file_name }}.pem" 87 | - "{{ certificate_file_name }}-key.pem" 88 | 89 | - name: Copy certificate and private key into home directory 90 | when: copy_cert_to_home_directory is defined and copy_cert_to_home_directory == True and (certificate_csr_name == "ca" or not certificates_already_present_in_s3) 91 | copy: 92 | src: "{{ local_certificate_storage_path }}/{{ item }}" 93 | dest: "{{ ansible_env.HOME }}" 94 | with_items: 95 | - "{{ certificate_file_name }}.pem" 96 | - "{{ certificate_file_name }}-key.pem" 97 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/group_vars/all/vars.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ansible_home_directory: "{{ ansible_env.PWD }}" 4 | kubernetes_version: 1.10.2 5 | local_certificate_storage_path: /tmp 6 | kubernetes_ca_state_code: TX 7 | kubernetes_ca_ou_code: CA 8 | kubernetes_ca_common_name: Kubernetes 9 | kubernetes_ca_country_code: US 10 | kubernetes_ca_location_code: Dallas 11 | cfssl_docker_image: cfssl/cfssl 12 | cfssl_url: https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 13 | cfssljson_url: https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 14 | kubectl_url: https://storage.googleapis.com/kubernetes-release/release/v{{ kubernetes_version }}/bin/linux/amd64/kubectl 15 | public_ip_address_api_url: https://api.ipify.org 16 | use_systemd: True 17 | async_value_to_leave_background_process_in_background_forever_sort_of: 100000000000000000000000000000000000000000000000 18 | skip_kubectl: False 19 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/post_deploy.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ansible-playbook -e "skip_kubectl=True" generate_certificates.yml || exit 1 4 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/common/files/resolv.conf: -------------------------------------------------------------------------------- 1 | # CNI-compatible /etc/resolv.conf. 2 | # Managed by Ansible. 3 | search ec2.internal 4 | nameserver 169.254.169.253 5 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Are we using systemd? {{ use_systemd }}" 4 | 5 | - name: Expose the certificate token 6 | set_fact: 7 | certificate_token: "{{ lookup('env', 'CERT_TOKEN') }}" 8 | failed_when: certificate_token == "" 9 | 10 | - name: Gather EC2 facts. 11 | action: ec2_facts 12 | when: lookup('env', 'INSTANCE_ROLE') != "" 13 | 14 | - name: Tag our instance with its role 15 | ec2_tag: 16 | resource: "{{ ansible_ec2_instance_id }}" 17 | state: present 18 | region: "{{ ansible_ec2_placement_region }}" 19 | tags: 20 | kubernetes_role: "{{ lookup('env', 'INSTANCE_ROLE') }}" 21 | when: lookup('env', 'INSTANCE_ROLE') != "" 22 | 23 | - name: Fetch Ansible S3 bucket and key from environment if defined. 24 | ignore_errors: true 25 | set_fact: 26 | ansible_vars_s3_bucket: "{{ lookup('env', 'ANSIBLE_VARS_S3_BUCKET') }}" 27 | ansible_vars_s3_key: "{{ lookup('env', 'ANSIBLE_VARS_S3_KEY') }}" 28 | environment_name: "{{ lookup('env', 'ENVIRONMENT_NAME') }}" 29 | when: > 30 | lookup('env', 'ANSIBLE_VARS_S3_BUCKET') != "" and 31 | lookup('env', 'ANSIBLE_VARS_S3_KEY') != "" and 32 | lookup('env', 'ENVIRONMENT_NAME') != "" 33 | 34 | - name: Fetch Ansible vars from s3 35 | aws_s3: 36 | bucket: "{{ ansible_vars_s3_bucket }}" 37 | object: "{{ ansible_vars_s3_key }}/{{ environment_name }}/ansible_vars.yml" 38 | dest: /etc/ansible/ansible_vars.yml 39 | mode: get 40 | 41 | - name: Protect the vars file 42 | file: 43 | mode: 0600 44 | path: /etc/ansible/ansible_vars.yml 45 | 46 | - name: Load vars 47 | include_vars: /etc/ansible/ansible_vars.yml 48 | 49 | - name: Get our public IP address. 50 | block: 51 | - name: Get our public IP address 52 | uri: 53 | url: "{{ public_ip_address_api_url }}" 54 | timeout: 5 55 | return_content: yes 56 | method: GET 57 | register: public_ip_address_response 58 | failed_when: False 59 | - name: Expose our public IP address as an Ansible fact. 60 | set_fact: 61 | external_ip: "{{ public_ip_address_response.content }}" 62 | 63 | - name: Download cfssl and cfssljson 64 | get_url: 65 | url: "{{ item }}" 66 | dest: "/usr/local/bin/{{ item.split('/')[-1].split('_')[0] }}" 67 | with_items: 68 | - "{{ cfssl_url }}" 69 | - "{{ cfssljson_url }}" 70 | 71 | - name: Mark cfssl and cfssljson as executable 72 | file: 73 | path: "/usr/local/bin/{{ item }}" 74 | mode: 0755 75 | with_items: 76 | - cfssl 77 | - cfssljson 78 | 79 | - name: Download kubectl 80 | get_url: 81 | url: "{{ kubectl_url }}" 82 | dest: /usr/local/bin/kubectl 83 | mode: 0755 84 | when: not (skip_kubectl | bool) 85 | 86 | - name: Verify kubectl 87 | shell: "kubectl version --client" 88 | register: kubectl_output 89 | failed_when: kubectl_output.rc != 0 90 | when: not (skip_kubectl | bool) 91 | tags: 92 | - skip_ansible_lint 93 | 94 | - name: Enable br_netfilter if we're running on a Kubernetes node 95 | modprobe: 96 | name: br_netfilter 97 | state: present 98 | when: lookup('env', 'INSTANCE_ROLE') == "controller" or lookup('env', 'INSTANCE_ROLE') == "worker" 99 | 100 | - name: Disable systemd-resolved if we're running on an instance 101 | systemd: 102 | name: systemd-resolved 103 | state: stopped 104 | enabled: no 105 | when: lookup('env', 'INSTANCE_ROLE') == "controller" or lookup('env', 'INSTANCE_ROLE') == "worker" 106 | 107 | - name: Use a generic /etc/resolv.conf if we're running on an instance 108 | copy: 109 | src: files/resolv.conf 110 | dest: /etc/resolv.conf 111 | when: lookup('env', 'INSTANCE_ROLE') == "controller" or lookup('env', 'INSTANCE_ROLE') == "worker" 112 | 113 | - name: Enable IPv4 and IPv6 packet forwarding (for CNI to work) 114 | sysctl: 115 | name: "{{ item }}" 116 | value: 1 117 | sysctl_set: yes 118 | state: present 119 | reload: yes 120 | when: lookup('env', 'INSTANCE_ROLE') == "controller" or lookup('env', 'INSTANCE_ROLE') == "worker" 121 | with_items: 122 | - net.ipv4.ip_forward 123 | - net.ipv6.conf.all.forwarding 124 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/defaults/main.yml: -------------------------------------------------------------------------------- 1 | kube_apiserver_url: "https://storage.googleapis.com/kubernetes-release/release/v{{ kubernetes_version }}/bin/linux/amd64/kube-apiserver" 2 | kube_controller_manager_url: "https://storage.googleapis.com/kubernetes-release/release/v{{ kubernetes_version }}/bin/linux/amd64/kube-controller-manager" 3 | kube_scheduler_url: "https://storage.googleapis.com/kubernetes-release/release/v{{ kubernetes_version }}/bin/linux/amd64/kube-scheduler" 4 | kubernetes_cluster_ip_range: "10.32.0.0/24" 5 | kubernetes_cluster_cidr: "10.200.0.0/16" 6 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/files/clusterrole_kube_apiserver_to_kubelet.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRole 3 | metadata: 4 | annotations: 5 | rbac.authorization.kubernetes.io/autoupdate: "true" 6 | labels: 7 | kubernetes.io/bootstrapping: rbac-defaults 8 | name: system:kube-apiserver-to-kubelet 9 | rules: 10 | - apiGroups: 11 | - "" 12 | resources: 13 | - nodes/proxy 14 | - nodes/stats 15 | - nodes/log 16 | - nodes/spec 17 | - nodes/metrics 18 | verbs: 19 | - "*" 20 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/files/clusterrolebinding_kube_apiserver_to_kubelet.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1beta1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: system:kube-apiserver 5 | namespace: "" 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: system:kube-apiserver-to-kubelet 10 | subjects: 11 | - apiGroup: rbac.authorization.k8s.io 12 | kind: User 13 | name: kubernetes 14 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Start the playbook 2 | debug: 3 | msg: "Waiting for {{ expected_controllers }} to start." 4 | 5 | - name: Discover other controllers in this control plane. 6 | ec2_instance_facts: 7 | filters: 8 | instance-state-name: running 9 | "tag:kubernetes_role": controller 10 | register: found_controllers 11 | until: ( found_controllers.instances | length ) == ( expected_controllers | int ) 12 | retries: 20 13 | delay: 5 14 | 15 | - name: Create required directories 16 | file: 17 | state: directory 18 | path: "{{ item }}" 19 | with_items: 20 | - /etc/kubernetes/config 21 | - /var/lib/kubernetes 22 | 23 | - name: Fetch relevant certificates. 24 | aws_s3: 25 | mode: get 26 | bucket: "{{ certificate_s3_bucket }}" 27 | object: "{{ certificate_s3_key }}/{{ environment_name }}/{{ certificate_token }}-{{ item }}" 28 | dest: "/var/lib/kubernetes/{{ item }}" 29 | retries: 5 30 | delay: 5 31 | with_items: 32 | - ca.pem 33 | - ca-key.pem 34 | - kubernetes-key.pem 35 | - kubernetes.pem 36 | - kube-controller-manager.pem 37 | - kube-controller-manager-key.pem 38 | - kube-scheduler.pem 39 | - kube-scheduler-key.pem 40 | - service-account-key.pem 41 | - service-account.pem 42 | - admin-key.pem 43 | - admin.pem 44 | 45 | - name: Generate required Kubeconfigs 46 | include_tasks: "{{ ansible_home_directory }}/create_kubeconfig_for_service.yml" 47 | vars: 48 | kubeconfig_location: /var/lib/kubernetes 49 | kubeconfig_name: "{{ kubernetes_component }}" 50 | cluster_name: "{{ kubernetes_cluster_name }}" 51 | user: "system:{{ kubernetes_component }}" 52 | kubernetes_control_plane_server: "https://{{ kubernetes_public_address }}:{{ kubernetes_public_port }}" 53 | client_certificate: "/var/lib/kubernetes/{{ kubernetes_component }}.pem" 54 | client_key: "/var/lib/kubernetes/{{ kubernetes_component }}-key.pem" 55 | certificate_authority_certificate_location: /var/lib/kubernetes 56 | loop_control: 57 | loop_var: kubernetes_component 58 | with_items: 59 | - kube-controller-manager 60 | - kube-scheduler 61 | - admin 62 | 63 | - name: Copy admin.kubeconfig to the home directory 64 | copy: 65 | src: /var/lib/kubernetes/admin.kubeconfig 66 | dest: "{{ ansible_env.HOME }}/admin.kubeconfig" 67 | 68 | - name: Create an encryption key 69 | shell: head -c 32 /dev/urandom | base64 70 | register: command_result 71 | failed_when: command_result.rc != 0 72 | 73 | - name: Set that key as a fact 74 | set_fact: 75 | encryption_key: "{{ command_result.stdout }}" 76 | 77 | - name: Generate encryption config 78 | template: 79 | src: templates/encryption-config.yaml.tmpl 80 | dest: /var/lib/kubernetes/encryption-config.yaml 81 | 82 | - name: Download control plane binaries 83 | get_url: 84 | url: "{{ item }}" 85 | dest: /usr/local/bin 86 | mode: 0755 87 | with_items: 88 | - "{{ kube_apiserver_url }}" 89 | - "{{ kube_controller_manager_url }}" 90 | - "{{ kube_scheduler_url }}" 91 | 92 | - name: Verify that binaries are present 93 | stat: 94 | path: "/usr/local/bin/{{ item }}" 95 | register: kube_binary 96 | failed_when: not kube_binary.stat.exists 97 | with_items: 98 | - kube-apiserver 99 | - kube-controller-manager 100 | - kube-scheduler 101 | 102 | - name: Discover other etcd clusters in this cluster 103 | ec2_instance_facts: 104 | filters: 105 | instance-state-name: running 106 | "tag:kubernetes_role": etcd 107 | register: found_instances 108 | 109 | - name: Save number of controllers 110 | set_fact: 111 | kubernetes_controller_count: "{{ found_controllers.instances | length }}" 112 | 113 | - name: Generate etcd initial cluster list 114 | set_fact: 115 | etcd_http_servers: "{% for instance in found_instances.instances %}https://{{instance.private_ip_address}}:2379,{% endfor %}" 116 | 117 | - name: Remove last comma from string 118 | set_fact: 119 | etcd_http_servers: "{{ etcd_http_servers[:-1] }}" 120 | 121 | - name: Create systemd services 122 | template: 123 | src: "templates/{{ item }}.service.tmpl" 124 | dest: "/etc/systemd/system/{{ item }}.service" 125 | with_items: 126 | - kube-apiserver 127 | - kube-controller-manager 128 | - kube-scheduler 129 | 130 | - name: Create kube-scheduler configuration YAML 131 | template: 132 | src: templates/kube-scheduler.yaml.tmpl 133 | dest: /etc/kubernetes/config/kube-scheduler.yaml 134 | 135 | - name: Start systemd services 136 | systemd: 137 | name: "{{ item }}" 138 | state: started 139 | daemon_reload: yes 140 | with_items: 141 | - kube-apiserver 142 | - kube-controller-manager 143 | - kube-scheduler 144 | 145 | - name: Enable health checks 146 | block: 147 | - name: Install nginx 148 | package: 149 | name: nginx 150 | 151 | - name: Create health check site 152 | template: 153 | src: templates/kubernetes.default.svc.cluster.local.tmpl 154 | dest: /etc/nginx/sites-available/kubernetes.default.svc.cluster.local 155 | 156 | - name: Enable the site 157 | file: 158 | src: /etc/nginx/sites-available/kubernetes.default.svc.cluster.local 159 | dest: /etc/nginx/sites-enabled/kubernetes.default.svc.cluster.local 160 | state: link 161 | 162 | - name: Start nginx 163 | systemd: 164 | name: nginx 165 | state: restarted 166 | 167 | - name: Wait 60 seconds for kube-apiserver to become available 168 | shell: "kubectl get componentstatuses --kubeconfig /var/lib/kubernetes/admin.kubeconfig" 169 | register: kubectl_result 170 | until: kubectl_result == 0 171 | retries: 60 172 | delay: 1 173 | failed_when: kubectl_result.rc != 0 174 | 175 | - name: Copy RBAC manifests 176 | copy: 177 | src: "files/{{ item }}.yaml" 178 | dest: "/var/lib/kubernetes/{{ item }}.yaml" 179 | with_items: 180 | - clusterrole_kube_apiserver_to_kubelet 181 | - clusterrolebinding_kube_apiserver_to_kubelet 182 | 183 | - name: Apply RBAC manifests 184 | shell: "kubectl apply --kubeconfig /var/lib/kubernetes/admin.kubeconfig -f /var/lib/kubernetes/{{ item }}.yaml" 185 | register: kubectl_result 186 | with_items: 187 | - clusterrole_kube_apiserver_to_kubelet 188 | - clusterrolebinding_kube_apiserver_to_kubelet 189 | failed_when: kubectl_result.rc != 0 190 | 191 | - name: Verify /healthz 192 | uri: 193 | url: http://127.0.0.1/healthz 194 | headers: 195 | Host: kubernetes.default.svc.cluster.local 196 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/templates/encryption-config.yaml.tmpl: -------------------------------------------------------------------------------- 1 | kind: EncryptionConfig 2 | apiVersion: v1 3 | resources: 4 | - resources: 5 | - secrets 6 | providers: 7 | - aescbc: 8 | keys: 9 | - name: key1 10 | secret: {{ encryption_key }} 11 | - identity: {} 12 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/templates/kube-apiserver.service.tmpl: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes API Server 3 | Documentation=https://github.com/kubernetes/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-apiserver \ 7 | --advertise-address={{ ansible_default_ipv4.address }} \ 8 | --allow-privileged=true \ 9 | --apiserver-count={{ kubernetes_controller_count }} \ 10 | --audit-log-maxage=30 \ 11 | --audit-log-maxbackup=3 \ 12 | --audit-log-maxsize=100 \ 13 | --audit-log-path=/var/log/audit.log \ 14 | --authorization-mode=Node,RBAC \ 15 | --bind-address=0.0.0.0 \ 16 | --client-ca-file=/var/lib/kubernetes/ca.pem \ 17 | --enable-admission-plugins=Initializers,NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ 18 | --enable-swagger-ui=true \ 19 | --etcd-cafile=/var/lib/kubernetes/ca.pem \ 20 | --etcd-certfile=/var/lib/kubernetes/kubernetes.pem \ 21 | --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem \ 22 | --etcd-servers={{ etcd_http_servers }} \ 23 | --event-ttl=1h \ 24 | --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem \ 25 | --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem \ 26 | --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem \ 27 | --kubelet-https=true \ 28 | --runtime-config=api/all \ 29 | --service-account-key-file=/var/lib/kubernetes/service-account.pem \ 30 | --service-cluster-ip-range={{ kubernetes_cluster_ip_range }}\ 31 | --service-node-port-range=30000-32767 \ 32 | --tls-cert-file=/var/lib/kubernetes/kubernetes.pem \ 33 | --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem \ 34 | --v=2 35 | Restart=on-failure 36 | RestartSec=5 37 | 38 | [Install] 39 | WantedBy=multi-user.target 40 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/templates/kube-controller-manager.service.tmpl: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Controller Manager 3 | Documentation=https://github.com/kubernetes/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-controller-manager \ 7 | --address=0.0.0.0 \ 8 | --cluster-cidr={{ kubernetes_cluster_cidr }} \ 9 | --cluster-name={{ kubernetes_cluster_name }} \ 10 | --cluster-signing-cert-file=/var/lib/kubernetes/ca.pem \ 11 | --cluster-signing-key-file=/var/lib/kubernetes/ca-key.pem \ 12 | --kubeconfig=/var/lib/kubernetes/kube-controller-manager.kubeconfig \ 13 | --leader-elect=true \ 14 | --root-ca-file=/var/lib/kubernetes/ca.pem \ 15 | --service-account-private-key-file=/var/lib/kubernetes/service-account-key.pem \ 16 | --service-cluster-ip-range={{ kubernetes_cluster_ip_range }} \ 17 | --use-service-account-credentials=true \ 18 | --v=2 19 | Restart=on-failure 20 | RestartSec=5 21 | 22 | [Install] 23 | WantedBy=multi-user.target 24 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/templates/kube-scheduler.service.tmpl: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Scheduler 3 | Documentation=https://github.com/kubernetes/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-scheduler \ 7 | --config=/etc/kubernetes/config/kube-scheduler.yaml \ 8 | --v=2 9 | Restart=on-failure 10 | RestartSec=5 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/templates/kube-scheduler.yaml.tmpl: -------------------------------------------------------------------------------- 1 | apiVersion: componentconfig/v1alpha1 2 | kind: KubeSchedulerConfiguration 3 | clientConnection: 4 | kubeconfig: "/var/lib/kubernetes/kube-scheduler.kubeconfig" 5 | leaderElection: 6 | leaderElect: true 7 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/controller/templates/kubernetes.default.svc.cluster.local.tmpl: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name kubernetes.default.svc.cluster.local; 4 | 5 | location /healthz { 6 | proxy_pass https://127.0.0.1:6443/healthz; 7 | proxy_ssl_trusted_certificate /var/lib/kubernetes/ca.pem; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/etcd/defaults/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | etcd_version: "3.3.5" 4 | etcd_url: "https://github.com/coreos/etcd/releases/download/v{{ etcd_version }}/etcd-v{{ etcd_version }}-linux-amd64.tar.gz" 5 | etcd_cluster_token: etcd-cluster-0 6 | etcd_confirm_command: >- 7 | ETCDCTL_API=3 /usr/local/bin/etcdctl member list 8 | --endpoints=https://127.0.0.1:2379 9 | --cacert=/etc/etcd/ca.pem 10 | --cert=/etc/etcd/kubernetes.pem 11 | --key=/etc/etcd/kubernetes-key.pem 12 | etcd_start_command: >- 13 | /usr/local/bin/etcd \ 14 | --name {{ system_hostname }} \ 15 | --cert-file=/etc/etcd/kubernetes.pem \ 16 | --key-file=/etc/etcd/kubernetes-key.pem \ 17 | --peer-cert-file=/etc/etcd/kubernetes.pem \ 18 | --peer-key-file=/etc/etcd/kubernetes-key.pem \ 19 | --trusted-ca-file=/etc/etcd/ca.pem \ 20 | --peer-trusted-ca-file=/etc/etcd/ca.pem \ 21 | --peer-client-cert-auth \ 22 | --client-cert-auth \ 23 | --initial-advertise-peer-urls https://{{ ansible_default_ipv4.address }}:2380 \ 24 | --listen-peer-urls https://{{ ansible_default_ipv4.address }}:2380 \ 25 | --listen-client-urls https://{{ ansible_default_ipv4.address }}:2379,https://127.0.0.1:2379 \ 26 | --advertise-client-urls https://{{ ansible_default_ipv4.address }}:2379 \ 27 | --initial-cluster-token "{{ etcd_cluster_token }}" \ 28 | --initial-cluster "{{ etcd_initial_cluster_string }}" \ 29 | --initial-cluster-state new \ 30 | --data-dir=/var/lib/etcd 31 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/etcd/tasks/main.yml: -------------------------------------------------------------------------------- 1 | - name: Start the playbook 2 | debug: 3 | msg: "Waiting for {{ expected_etcd_servers }} to start." 4 | 5 | - name: Discover other etcd clusters in this cluster 6 | ec2_instance_facts: 7 | filters: 8 | instance-state-name: running 9 | "tag:kubernetes_role": etcd 10 | register: found_instances 11 | until: ( found_instances.instances | length ) == ( expected_etcd_servers | int ) 12 | retries: 20 13 | delay: 5 14 | 15 | - name: Create some folders that we'll need for etcd 16 | file: 17 | state: directory 18 | path: "{{ item }}" 19 | with_items: 20 | - /etc/etcd 21 | - /var/lib/etcd 22 | - /tmp/etcd 23 | 24 | - name: Fetch relevant certificates. 25 | aws_s3: 26 | mode: get 27 | bucket: "{{ certificate_s3_bucket }}" 28 | object: "{{ certificate_s3_key }}/{{ environment_name }}/{{ certificate_token }}-{{ item }}" 29 | dest: "/etc/etcd/{{ item }}" 30 | with_items: 31 | - ca.pem 32 | - kubernetes-key.pem 33 | - kubernetes.pem 34 | retries: 5 35 | delay: 5 36 | 37 | - name: Generate etcd initial cluster list 38 | set_fact: 39 | etcd_initial_cluster_string: "{% for instance in found_instances.instances %}ip-{{instance.private_ip_address.replace('.','-')}}=https://{{instance.private_ip_address}}:2380,{% endfor %}" 40 | 41 | - name: Remove last comma from string 42 | set_fact: 43 | etcd_initial_cluster_string: "{{ etcd_initial_cluster_string[:-1] }}" 44 | 45 | - name: Download and unarchive the etcd tarball. 46 | unarchive: 47 | src: "{{ etcd_url }}" 48 | dest: /tmp/etcd 49 | remote_src: yes 50 | 51 | - name: Move etcd binaries into /usr/local/bin. 52 | shell: "cp -v /tmp/etcd/etcd-v{{ etcd_version }}-linux-amd64/etcd* /usr/local/bin/" 53 | register: command_result 54 | 55 | - debug: 56 | msg: "Command result: {{ command_result.stdout }}" 57 | 58 | - name: Ensure binaries we want are present 59 | stat: 60 | path: "/usr/local/bin/{{ item }}" 61 | register: etcd_binary 62 | failed_when: not etcd_binary.stat.exists 63 | with_items: 64 | - etcd 65 | - etcdctl 66 | 67 | - name: Create systemd service. 68 | template: 69 | src: templates/etcd.service.tmpl 70 | dest: /etc/systemd/system/etcd.service 71 | when: (use_systemd | bool) 72 | 73 | - name: Ensure that the etcd systemd service is up. 74 | systemd: 75 | name: etcd 76 | daemon_reload: yes 77 | state: started 78 | when: (use_systemd | bool) 79 | 80 | - name: Start etcd in the background. 81 | shell: "nohup {{ etcd_start_command }} &" 82 | poll: 0 83 | async: "{{ async_value_to_leave_background_process_in_background_forever_sort_of }}" 84 | when: not (use_systemd | bool) 85 | 86 | - name: Wait for all members of this etcd cluster to discover themselves. 87 | pause: 88 | seconds: 15 89 | 90 | - name: Ensure that the etcd cluster has started. 91 | shell: "{{ etcd_confirm_command }}" 92 | register: etcdctl_result 93 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/etcd/templates/etcd.service.tmpl: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=etcd 3 | Documentation=https://github.com/coreos 4 | 5 | [Service] 6 | ExecStart={{ etcd_start_command }} 7 | Restart=on-failure 8 | RestartSec=5 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/defaults/main.yml: -------------------------------------------------------------------------------- 1 | crictl_url: "https://github.com/kubernetes-incubator/cri-tools/releases/download/v1.0.0-beta.0/crictl-v1.0.0-beta.0-linux-amd64.tar.gz" 2 | runsc_url: "https://storage.googleapis.com/kubernetes-the-hard-way/runsc" 3 | runc_url: "https://github.com/opencontainers/runc/releases/download/v1.0.0-rc5/runc.amd64" 4 | cni_plugins_url: "https://github.com/containernetworking/plugins/releases/download/v0.6.0/cni-plugins-amd64-v0.6.0.tgz" 5 | containerd_url: "https://github.com/containerd/containerd/releases/download/v1.1.0/containerd-1.1.0.linux-amd64.tar.gz" 6 | kube_proxy_url: "https://storage.googleapis.com/kubernetes-release/release/v{{ kubernetes_version }}/bin/linux/amd64/kube-proxy" 7 | kubelet_url: "https://storage.googleapis.com/kubernetes-release/release/v{{ kubernetes_version }}/bin/linux/amd64/kubelet" 8 | kubernetes_control_plane_wait_delay_seconds: 3600 9 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/files/containerd.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=containerd container runtime 3 | Documentation=https://containerd.io 4 | After=network.target 5 | 6 | [Service] 7 | ExecStartPre=/sbin/modprobe overlay 8 | ExecStart=/bin/containerd 9 | Restart=always 10 | RestartSec=5 11 | Delegate=yes 12 | KillMode=process 13 | OOMScoreAdjust=-999 14 | LimitNOFILE=1048576 15 | LimitNPROC=infinity 16 | LimitCORE=infinity 17 | 18 | [Install] 19 | WantedBy=multi-user.target 20 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/files/containerd_config.toml: -------------------------------------------------------------------------------- 1 | [plugins] 2 | [plugins.cri.containerd] 3 | snapshotter = "overlayfs" 4 | [plugins.cri.containerd.default_runtime] 5 | runtime_type = "io.containerd.runtime.v1.linux" 6 | runtime_engine = "/usr/local/bin/runc" 7 | runtime_root = "" 8 | [plugins.cri.containerd.untrusted_workload_runtime] 9 | runtime_type = "io.containerd.runtime.v1.linux" 10 | runtime_engine = "/usr/local/bin/runsc" 11 | runtime_root = "/run/containerd/runsc" 12 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/files/kube-proxy.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Kube Proxy 3 | Documentation=https://github.com/kubernetes/kubernetes 4 | 5 | [Service] 6 | ExecStart=/usr/local/bin/kube-proxy \ 7 | --config=/var/lib/kube-proxy/kube-proxy-config.yaml \ 8 | --masquerade-all=true 9 | Restart=on-failure 10 | RestartSec=5 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/files/kubelet.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Kubernetes Kubelet 3 | Documentation=https://github.com/kubernetes/kubernetes 4 | After=containerd.service 5 | Requires=containerd.service 6 | 7 | [Service] 8 | ExecStart=/usr/local/bin/kubelet \ 9 | --config=/var/lib/kubelet/kubelet-config.yaml \ 10 | --container-runtime=remote \ 11 | --container-runtime-endpoint=unix:///var/run/containerd/containerd.sock \ 12 | --image-pull-progress-deadline=2m \ 13 | --kubeconfig=/var/lib/kubelet/kubeconfig \ 14 | --network-plugin=cni \ 15 | --register-node=true \ 16 | --v=2 17 | Restart=on-failure 18 | RestartSec=5 19 | 20 | [Install] 21 | WantedBy=multi-user.target 22 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Start the playbook 3 | debug: 4 | msg: "Waiting for {{ expected_workers }} to start." 5 | 6 | - name: Discover other workers in this control plane. 7 | ec2_instance_facts: 8 | filters: 9 | instance-state-name: running 10 | "tag:kubernetes_role": worker 11 | register: found_workers 12 | until: ( found_workers.instances | length ) == ( expected_workers | int ) 13 | retries: 20 14 | delay: 5 15 | 16 | - name: Get route tables containing this instance. 17 | ec2_vpc_route_table_facts: 18 | region: "{{ ansible_ec2_placement_region }}" 19 | filters: 20 | route.instance-id: "{{ ansible_ec2_instance_id }}" 21 | register: route_tables_found 22 | 23 | - name: Find Pod CIDR subnets associated with this instance 24 | set_fact: 25 | pod_cidr_subnet_for_this_worker: > 26 | {% for route_table in route_tables_found.route_tables %} 27 | {% for route in route_table.routes %} 28 | {% if route.instance_id == ansible_ec2_instance_id %} 29 | {{ route.destination_cidr_block }} 30 | {% endif %} 31 | {% endfor %} 32 | {% endfor %} 33 | 34 | - name: Fix up Pod CIDR 35 | set_fact: 36 | pod_cidr_subnet_for_this_worker: "{{ pod_cidr_subnet_for_this_worker | replace(' ','') | replace ('\n', '') }}" 37 | 38 | - name: Get our VPC 39 | uri: 40 | url: "http://169.254.169.254/latest/meta-data/network/interfaces/macs/{{ ansible_default_ipv4.macaddress }}/vpc-id" 41 | return_content: yes 42 | register: vpc_id_metadata 43 | 44 | - name: Extract VPC from metadata 45 | set_fact: 46 | vpc_id: "{{ vpc_id_metadata.content }}" 47 | 48 | - name: Create required directories 49 | file: 50 | state: directory 51 | path: "{{ item }}" 52 | with_items: 53 | - /var/lib/kubernetes 54 | - /etc/containerd 55 | - /etc/cni/net.d 56 | - /opt/cni/bin 57 | - /var/lib/kubelet 58 | - /var/lib/kube-proxy 59 | - /var/run/kubernetes 60 | 61 | - name: Fetch relevant certificates and store in the certificate volume. 62 | aws_s3: 63 | mode: get 64 | bucket: "{{ certificate_s3_bucket }}" 65 | object: "{{ certificate_s3_key }}/{{ environment_name }}/{{ certificate_token }}-{{ item }}" 66 | dest: "{{ local_certificate_storage_path }}/{{ item }}" 67 | with_items: 68 | - ca.pem 69 | - ca-key.pem 70 | - kube-proxy.pem 71 | - kube-proxy-key.pem 72 | 73 | - name: Copy CA certs into right place. 74 | copy: 75 | src: "{{ local_certificate_storage_path }}/{{ item }}" 76 | dest: "/var/lib/kubernetes/{{ item }}" 77 | with_items: 78 | - ca.pem 79 | - ca-key.pem 80 | - kube-proxy.pem 81 | - kube-proxy-key.pem 82 | 83 | - name: Generate worker certificate 84 | include: "{{ ansible_home_directory }}/generate_single_certificate.yml" 85 | vars: 86 | skip_s3_upload: True 87 | loop_control: 88 | loop_var: certificate_csr_name 89 | loop: 90 | - worker 91 | 92 | - name: Copy instance certs into right place. 93 | copy: 94 | src: "{{ local_certificate_storage_path }}/{{ item }}" 95 | dest: "/var/lib/kubelet/{{ item }}" 96 | with_items: 97 | - "{{ system_hostname }}.pem" 98 | - "{{ system_hostname }}-key.pem" 99 | 100 | - name: Create the kubeconfig for this instance. 101 | include_tasks: "{{ ansible_home_directory }}/create_kubeconfig_for_service.yml" 102 | vars: 103 | kubeconfig_location: "/var/lib/kubelet" 104 | kubeconfig_name: "{{ system_hostname }}" 105 | cluster_name: "{{ kubernetes_cluster_name }}" 106 | user: "system:node:{{ system_hostname }}" 107 | kubernetes_control_plane_server: "https://{{ kubernetes_public_address }}:{{ kubernetes_public_port }}" 108 | client_certificate: "{{ local_certificate_storage_path }}/{{ system_hostname }}.pem" 109 | client_key: "{{ local_certificate_storage_path }}/{{ system_hostname }}-key.pem" 110 | certificate_authority_certificate_location: /var/lib/kubernetes 111 | 112 | - name: Move to correct location 113 | copy: 114 | src: "/var/lib/kubelet/{{ system_hostname }}.kubeconfig" 115 | dest: "/var/lib/kubelet/kubeconfig" 116 | 117 | - name: Create the kubeconfig for kube-proxy 118 | include_tasks: "{{ ansible_home_directory }}/create_kubeconfig_for_service.yml" 119 | vars: 120 | kubeconfig_location: "/var/lib/kube-proxy" 121 | kubeconfig_name: "kube-proxy" 122 | cluster_name: "{{ kubernetes_cluster_name }}" 123 | user: "system:kube-proxy" 124 | kubernetes_control_plane_server: "https://{{ kubernetes_public_address }}:{{ kubernetes_public_port }}" 125 | client_certificate: "{{ local_certificate_storage_path }}/kube-proxy.pem" 126 | client_key: "{{ local_certificate_storage_path }}/kube-proxy-key.pem" 127 | certificate_authority_certificate_location: /var/lib/kubernetes 128 | 129 | - name: Move to correct location 130 | copy: 131 | src: "/var/lib/kube-proxy/kube-proxy.kubeconfig" 132 | dest: "/var/lib/kube-proxy/kubeconfig" 133 | 134 | - name: Install dependencies 135 | package: 136 | name: "{{ item }}" 137 | state: present 138 | with_items: 139 | - conntrack 140 | - socat 141 | - ipset 142 | 143 | - name: Download kube binaries 144 | get_url: 145 | url: "{{ item }}" 146 | mode: 0755 147 | dest: "/tmp/{{ item.split('/')[-1] }}" 148 | with_items: 149 | - "{{ runsc_url }}" 150 | - "{{ runc_url }}" 151 | - "{{ kube_proxy_url }}" 152 | - "{{ kubelet_url }}" 153 | 154 | - name: Install kube binaries 155 | copy: 156 | src: "/tmp/{{ item }}" 157 | dest: "/usr/local/bin/{{ item }}" 158 | mode: 0755 159 | with_items: 160 | - kube-proxy 161 | - kubelet 162 | - runsc 163 | 164 | - name: Install runc 165 | copy: 166 | src: /tmp/runc.amd64 167 | dest: /usr/local/bin/runc 168 | mode: 0755 169 | 170 | - name: Install runtime and CNI components 171 | unarchive: 172 | src: "{{ item.split(';')[0] }}" 173 | dest: "{{ item.split(';')[1] }}" 174 | remote_src: yes 175 | with_items: 176 | - "{{ crictl_url }};/usr/local/bin" 177 | - "{{ cni_plugins_url }};/opt/cni/bin" 178 | - "{{ containerd_url }};/" 179 | 180 | - name: Provision CNI configurations 181 | template: 182 | src: "templates/{{ item }}.conf.j2" 183 | dest: "/etc/cni/net.d/{{ item }}.conf" 184 | with_items: 185 | - 10-bridge 186 | - 99-loopback 187 | 188 | - name: Create containerd configuration 189 | copy: 190 | src: files/containerd_config.toml 191 | dest: /etc/containerd/config.toml 192 | 193 | - name: Create configuration manifests. 194 | template: 195 | src: "templates/{{ item }}-config.yaml.j2" 196 | dest: "/var/lib/{{ item }}/{{ item }}-config.yaml" 197 | with_items: 198 | - kubelet 199 | - kube-proxy 200 | 201 | - name: Create systemd services 202 | copy: 203 | src: "files/{{ item }}.service" 204 | dest: "/etc/systemd/system/{{ item }}.service" 205 | with_items: 206 | - kubelet 207 | - kube-proxy 208 | - containerd 209 | 210 | # Even though we have the CA cert and key on our system 211 | # Ansible doesn't support using it with the 'uri' module. 212 | # I'd rather try this insecurely than worry about adding it to the 213 | # list of system certs. 214 | - name: Wait for the Kubernetes control plane to start up 215 | uri: 216 | url: "https://{{ kubernetes_public_address }}:{{ kubernetes_public_port }}/version" 217 | validate_certs: no 218 | register: kube_api_response 219 | until: kube_api_response.status == 200 220 | retries: "{{ kubernetes_control_plane_wait_delay_seconds }}" 221 | delay: 1 222 | 223 | - name: Start systemd services 224 | systemd: 225 | name: "{{ item }}" 226 | daemon_reload: yes 227 | state: restarted 228 | with_items: 229 | - kubelet 230 | - kube-proxy 231 | - containerd 232 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/templates/10-bridge.conf.j2: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "name": "bridge", 4 | "type": "bridge", 5 | "bridge": "cnio0", 6 | "isGateway": true, 7 | "ipMasq": true, 8 | "ipam": { 9 | "type": "host-local", 10 | "ranges": [ 11 | [{"subnet": "{{ pod_cidr_subnet_for_this_worker }}"}] 12 | ], 13 | "routes": [{"dst": "0.0.0.0/0"}] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/templates/99-loopback.conf.j2: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.3.1", 3 | "type": "loopback" 4 | } 5 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/templates/kube-proxy-config.yaml.j2: -------------------------------------------------------------------------------- 1 | kind: KubeProxyConfiguration 2 | apiVersion: kubeproxy.config.k8s.io/v1alpha1 3 | clientConnection: 4 | kubeconfig: "/var/lib/kube-proxy/kubeconfig" 5 | mode: "iptables" 6 | clusterCIDR: "{{ kubernetes_pod_cidr_block }}" 7 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/roles/worker/templates/kubelet-config.yaml.j2: -------------------------------------------------------------------------------- 1 | kind: KubeletConfiguration 2 | apiVersion: kubelet.config.k8s.io/v1beta1 3 | authentication: 4 | anonymous: 5 | enabled: false 6 | webhook: 7 | enabled: true 8 | x509: 9 | clientCAFile: "/var/lib/kubernetes/ca.pem" 10 | authorization: 11 | mode: Webhook 12 | clusterDomain: "cluster.local" 13 | clusterDNS: 14 | - "10.32.0.10" 15 | podCIDR: "{{ pod_cidr_subnet_for_this_worker }}" 16 | runtimeRequestTimeout: "15m" 17 | tlsCertFile: "/var/lib/kubelet/{{ system_hostname }}.pem" 18 | tlsPrivateKeyFile: "/var/lib/kubelet/{{ system_hostname }}-key.pem" 19 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | for playbook in create_and_upload_ansible_vars 4 | do 5 | if [ "$1" == "--regenerate" ] 6 | then 7 | >&2 echo "INFO: Ansible vars will be regenerated." 8 | ansible-playbook -e "regenerate_vars=True" -e "skip_kubectl=True" "${playbook}.yml" || exit 1 9 | else 10 | ansible-playbook "${playbook}.yml" || exit 1 11 | fi 12 | done 13 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - hosts: localhost 4 | connection: local 5 | gather_facts: false 6 | tasks: 7 | - name: Install Python (when ssh becomes available) 8 | package: 9 | name: python 10 | become: true 11 | retries: 10 12 | delay: 20 13 | 14 | - name: Get pip installer. 15 | get_url: 16 | url: https://bootstrap.pypa.io/get-pip.py 17 | dest: /tmp/get-pip.py 18 | mode: 0755 19 | 20 | - name: Install pip. 21 | shell: python /tmp/get-pip.py 22 | tags: 23 | - skip_ansible_lint 24 | 25 | - name: Install required packages 26 | pip: 27 | name: "{{ item }}" 28 | with_items: 29 | - botocore 30 | - docker-py 31 | - boto3 32 | - boto 33 | 34 | - name: Tell us the kind of node that we are provisioning 35 | debug: 36 | msg: "Ansible is provisioning a new: {{ lookup('env', 'INSTANCE_ROLE') }}" 37 | 38 | - hosts: localhost 39 | connection: local 40 | roles: 41 | - common 42 | - role: "{{ lookup('env', 'INSTANCE_ROLE') }}" 43 | 44 | - hosts: localhost 45 | connection: local 46 | tasks: 47 | - name: Delete sensitive info. 48 | file: 49 | state: absent 50 | path: "{{ local_certificate_storage_path }}/*.{{ item }}" 51 | with_items: 52 | - pem 53 | - kubeconfig 54 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/admin-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "admin", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "{{ kubernetes_ca_country_code }}", 10 | "L": "{{ kubernetes_ca_location_code }}", 11 | "O": "system:masters", 12 | "OU": "{{ kubernetes_ca_ou_code }}", 13 | "ST": "{{ kubernetes_ca_state_code }}" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/ansible_vars.yml.tmpl: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | certificate_s3_bucket: "{{ ansible_env.KUBERNETES_CERTIFICATE_S3_BUCKET }}" 4 | certificate_s3_key: "{{ ansible_env.KUBERNETES_CERTIFICATE_S3_KEY }}" 5 | kubernetes_public_address: "kubernetes.{{ ansible_env.DOMAIN_NAME }}" 6 | kubernetes_public_port: "{{ ansible_env.KUBERNETES_PUBLIC_PORT }}" 7 | kubernetes_cluster_name: "{{ ansible_env.KUBERNETES_CLUSTER_NAME }}" 8 | domain_name: "{{ ansible_env.DOMAIN_NAME }}" 9 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/ca-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "{{ kubernetes_ca_common_name }}", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "{{ kubernetes_ca_country_code }}", 10 | "L": "{{ kubernetes_ca_location_code }}", 11 | "O": "{{ kubernetes_ca_common_name }}", 12 | "OU": "{{ kubernetes_ca_ou_code }}", 13 | "ST": "{{ kubernetes_ca_state_code }}" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/kube-controller-manager-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-controller-manager", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "{{ kubernetes_ca_country_code }}", 10 | "L": "{{ kubernetes_ca_location_code }}", 11 | "O": "system:kube-controller-manager", 12 | "OU": "{{ kubernetes_ca_ou_code }}", 13 | "ST": "{{ kubernetes_ca_state_code }}" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/kube-proxy-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-proxy", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "{{ kubernetes_ca_country_code }}", 10 | "L": "{{ kubernetes_ca_location_code }}", 11 | "O": "system:node-proxier", 12 | "OU": "{{ kubernetes_ca_ou_code }}", 13 | "ST": "{{ kubernetes_ca_state_code }}" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/kube-scheduler-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "system:kube-scheduler", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "{{ kubernetes_ca_country_code }}", 10 | "L": "{{ kubernetes_ca_location_code }}", 11 | "O": "system:kube-scheduler", 12 | "OU": "{{ kubernetes_ca_ou_code }}", 13 | "ST": "{{ kubernetes_ca_state_code }}" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/kubernetes-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "hosts": [ "10.32.0.1", {{ cluster_ip_addresses_string }}, "{{ kubernetes_public_address }}", "127.0.0.1", "kubernetes.default" ], 3 | "CN": "kubernetes", 4 | "key": { 5 | "algo": "rsa", 6 | "size": 2048 7 | }, 8 | "names": [ 9 | { 10 | "C": "{{ kubernetes_ca_country_code }}", 11 | "L": "{{ kubernetes_ca_location_code }}", 12 | "O": "Kubernetes", 13 | "OU": "Kubernetes cluster", 14 | "ST": "{{ kubernetes_ca_state_code }}" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/service-account-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "CN": "service-accounts", 3 | "key": { 4 | "algo": "rsa", 5 | "size": 2048 6 | }, 7 | "names": [ 8 | { 9 | "C": "{{ kubernetes_ca_country_code }}", 10 | "L": "{{ kubernetes_ca_location_code }}", 11 | "O": "Kubernetes", 12 | "OU": "{{ kubernetes_ca_ou_code }}", 13 | "ST": "{{ kubernetes_ca_state_code }}" 14 | } 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/templates/worker-csr.json.tmpl: -------------------------------------------------------------------------------- 1 | { 2 | "hosts": [ 3 | "{{ external_ip }}", 4 | "{{ ansible_default_ipv4.address }}", 5 | "{{ system_hostname }}" 6 | ], 7 | "CN": "system:node:{{ system_hostname }}", 8 | "key": { 9 | "algo": "rsa", 10 | "size": 2048 11 | }, 12 | "names": [ 13 | { 14 | "C": "{{ kubernetes_ca_country_code }}", 15 | "L": "{{ kubernetes_ca_location_code }}", 16 | "O": "system:nodes", 17 | "OU": "{{ kubernetes_ca_ou_code }}", 18 | "ST": "{{ kubernetes_ca_state_code }}" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /kubernetes/config/ansible/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID?Please provide an AWS access key ID.}" 3 | AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY?Please provide an AWS secret access key.}" 4 | AWS_REGION="${AWS_REGION?Please provide an AWS region.}" 5 | CONFIG_MGMT_CODE_PATH="${CONFIG_MGMT_CODE_PATH?Please provide the location containing our Ansible playbooks.}" 6 | ANSIBLE_VARS_S3_BUCKET="${ANSIBLE_VARS_S3_BUCKET?Please provide the bucket containing our Ansible variables.}" 7 | ANSIBLE_VARS_S3_KEY="${ANSIBLE_VARS_S3_KEY?Please provide the key for ANSIBLE_VARS_S3_BUCKET.}" 8 | ENVIRONMENT_NAME="${ENVIRONMENT_NAME?Please provide an environment name.}" 9 | 10 | run_test() { 11 | kubernetes_role="$1" 12 | this_image_name=$(docker ps | \ 13 | grep $(hostname) | \ 14 | awk '{print $2}' 15 | ) 16 | >&2 echo "INFO: Testing Ansible role: $kubernetes_role" 17 | if ! INSTANCE_ROLE=$kubernetes_role ansible-playbook -e "use_systemd=False" -s site.yml 18 | then 19 | >&2 echo "ERROR: Test failed for role: $kubernetes_role" 20 | return 1 21 | fi 22 | } 23 | 24 | NUMBER_OF_AVAILABILITY_ZONES=1 ansible-playbook -e "regenerate_vars=True" create_and_upload_ansible_vars.yml 25 | if [ ! -z "$1" ] 26 | then 27 | run_test "$1" 28 | else 29 | >&2 echo "INFO: Testing configuration against all Kubernetes roles." 30 | this_directory=$(dirname "$0") 31 | for role_directory in $this_directory/roles/* 32 | do 33 | role_name=$(basename $role_directory) 34 | run_test "$role_name" || exit 1 35 | done 36 | fi 37 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/data.tf: -------------------------------------------------------------------------------- 1 | data "aws_availability_zones" "available_to_this_account" {} 2 | 3 | data "aws_route53_zone" "route53_zone_for_domain" { 4 | name = "${var.domain_name}" 5 | } 6 | 7 | data "template_file" "kubernetes_controller_user_data" { 8 | template = "${file("${path.module}/templates/user_data")}" 9 | vars { 10 | aws_region = "${ var.aws_region }" 11 | aws_access_key_id = "${ var.aws_access_key_id }" 12 | aws_secret_access_key = "${ var.aws_secret_access_key }" 13 | certificate_token = "${ var.certificate_token }" 14 | domain_name = "${ var.domain_name }" 15 | kubernetes_configuration_github_repository = "${var.kubernetes_configuration_github_repository}" 16 | kubernetes_configuration_github_branch = "${var.kubernetes_configuration_github_branch}" 17 | kubernetes_configuration_management_tool = "${var.kubernetes_configuration_management_code_directory}" 18 | kubernetes_pod_cidr = "${ var.kubernetes_pod_cidr_block }" 19 | ansible_vars_s3_bucket = "${var.ansible_vars_s3_bucket}" 20 | ansible_vars_s3_key = "${var.ansible_vars_s3_key}" 21 | environment_name = "${var.environment_name}" 22 | expected_etcd_servers = "${var.number_of_masters_per_control_plane}" 23 | expected_controllers = "${var.number_of_masters_per_control_plane}" 24 | expected_workers = "${var.number_of_workers_per_cluster}" 25 | route_table_id = "${aws_route_table.kubernetes_clusters.id}" 26 | role = "controller" 27 | } 28 | } 29 | data "template_file" "kubernetes_worker_user_data" { 30 | template = "${file("${path.module}/templates/user_data")}" 31 | vars { 32 | certificate_token = "${ var.certificate_token }" 33 | aws_region = "${ var.aws_region }" 34 | aws_access_key_id = "${ var.aws_access_key_id }" 35 | aws_secret_access_key = "${ var.aws_secret_access_key }" 36 | domain_name = "${ var.domain_name }" 37 | kubernetes_configuration_github_repository = "${var.kubernetes_configuration_github_repository}" 38 | kubernetes_configuration_github_branch = "${var.kubernetes_configuration_github_branch}" 39 | kubernetes_configuration_management_tool = "${var.kubernetes_configuration_management_code_directory}" 40 | kubernetes_pod_cidr = "${ var.kubernetes_pod_cidr_block }" 41 | ansible_vars_s3_bucket = "${var.ansible_vars_s3_bucket}" 42 | ansible_vars_s3_key = "${var.ansible_vars_s3_key}" 43 | environment_name = "${var.environment_name}" 44 | expected_etcd_servers = "${var.number_of_masters_per_control_plane}" 45 | expected_controllers = "${var.number_of_masters_per_control_plane}" 46 | expected_workers = "${var.number_of_workers_per_cluster}" 47 | route_table_id = "${aws_route_table.kubernetes_clusters.id}" 48 | role = "worker" 49 | } 50 | } 51 | data "template_file" "etcd_node_user_data" { 52 | template = "${file("${path.module}/templates/user_data")}" 53 | vars { 54 | certificate_token = "${ var.certificate_token }" 55 | aws_region = "${ var.aws_region }" 56 | aws_access_key_id = "${ var.aws_access_key_id }" 57 | aws_secret_access_key = "${ var.aws_secret_access_key }" 58 | domain_name = "${ var.domain_name }" 59 | kubernetes_configuration_github_repository = "${var.kubernetes_configuration_github_repository}" 60 | kubernetes_configuration_github_branch = "${var.kubernetes_configuration_github_branch}" 61 | kubernetes_configuration_management_tool = "${var.kubernetes_configuration_management_code_directory}" 62 | kubernetes_pod_cidr = "${ var.kubernetes_pod_cidr_block }" 63 | ansible_vars_s3_bucket = "${var.ansible_vars_s3_bucket}" 64 | ansible_vars_s3_key = "${var.ansible_vars_s3_key}" 65 | environment_name = "${var.environment_name}" 66 | expected_etcd_servers = "${var.number_of_masters_per_control_plane}" 67 | expected_controllers = "${var.number_of_masters_per_control_plane}" 68 | expected_workers = "${var.number_of_workers_per_cluster}" 69 | route_table_id = "${aws_route_table.kubernetes_clusters.id}" 70 | role = "etcd" 71 | } 72 | } 73 | 74 | locals { 75 | kubernetes_public_port = "${var.kubernetes_public_port}" 76 | kubernetes_internal_port = "${var.kubernetes_internal_port}" 77 | subnet_cidr_blocks = [ 78 | "${replace(var.cidr_block_for_kubernetes_clusters, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.1.0/24")}", 79 | "${replace(var.cidr_block_for_kubernetes_clusters, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.2.0/24")}", 80 | "${replace(var.cidr_block_for_kubernetes_clusters, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.3.0/24")}" 81 | ] 82 | worker_subnet_cidr_blocks = [ 83 | "${replace(var.cidr_block_for_kubernetes_clusters, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.4.0/24")}", 84 | "${replace(var.cidr_block_for_kubernetes_clusters, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.5.0/24")}", 85 | "${replace(var.cidr_block_for_kubernetes_clusters, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.6.0/24")}" 86 | ] 87 | } 88 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/instances.tf: -------------------------------------------------------------------------------- 1 | resource "aws_key_pair" "all_instances" { 2 | key_name = "kubernetes_cluster_${var.environment_name}" 3 | public_key = "${var.kubernetes_cluster_public_key}" 4 | } 5 | 6 | resource "aws_spot_instance_request" "etcd_cluster" { 7 | count = "${var.number_of_masters_per_control_plane}" 8 | availability_zone = "${element(data.aws_availability_zones.available_to_this_account.names,count.index)}" 9 | subnet_id = "${element(aws_subnet.kubernetes_control_plane.*.id, count.index)}" 10 | ami = "${var.etcd_node_ami}" 11 | instance_type = "${var.etcd_node_instance_type}" 12 | key_name = "${aws_key_pair.all_instances.key_name}" 13 | security_groups = [ "${aws_security_group.kubernetes_clusters.id}" ] 14 | associate_public_ip_address = true 15 | user_data = "${data.template_file.etcd_node_user_data.rendered}" 16 | root_block_device { 17 | volume_type = "gp2" 18 | volume_size = 32 19 | delete_on_termination = true 20 | } 21 | spot_price = "${var.etcd_nodes_spot_price}" 22 | wait_for_fulfillment = true 23 | tags = "${var.etcd_tags}" 24 | } 25 | 26 | resource "aws_spot_instance_request" "kubernetes_control_plane" { 27 | count = "${var.number_of_masters_per_control_plane}" 28 | availability_zone = "${element(data.aws_availability_zones.available_to_this_account.names,count.index)}" 29 | subnet_id = "${element(aws_subnet.kubernetes_control_plane.*.id, count.index)}" 30 | ami = "${var.kubernetes_node_ami}" 31 | instance_type = "${var.kubernetes_node_instance_type}" 32 | key_name = "${aws_key_pair.all_instances.key_name}" 33 | security_groups = [ "${aws_security_group.kubernetes_clusters.id}" ] 34 | associate_public_ip_address = true 35 | user_data = "${data.template_file.kubernetes_controller_user_data.rendered}" 36 | root_block_device { 37 | volume_type = "gp2" 38 | volume_size = 32 39 | delete_on_termination = true 40 | } 41 | spot_price = "${var.kubernetes_nodes_spot_price}" 42 | wait_for_fulfillment = true 43 | tags = "${var.kubernetes_controller_tags}" 44 | source_dest_check = false 45 | } 46 | 47 | resource "aws_spot_instance_request" "kubernetes_workers" { 48 | count = "${var.number_of_workers_per_cluster}" 49 | availability_zone = "${element(data.aws_availability_zones.available_to_this_account.names,count.index)}" 50 | subnet_id = "${element(aws_subnet.kubernetes_workers.*.id, count.index)}" 51 | ami = "${var.kubernetes_node_ami}" 52 | instance_type = "${var.kubernetes_node_instance_type}" 53 | key_name = "${aws_key_pair.all_instances.key_name}" 54 | security_groups = [ "${aws_security_group.kubernetes_clusters.id}" ] 55 | associate_public_ip_address = true 56 | user_data = "${data.template_file.kubernetes_worker_user_data.rendered}" 57 | root_block_device { 58 | volume_type = "gp2" 59 | volume_size = 32 60 | delete_on_termination = true 61 | } 62 | spot_price = "${var.kubernetes_nodes_spot_price}" 63 | wait_for_fulfillment = true 64 | tags = "${var.kubernetes_worker_tags}" 65 | source_dest_check = false 66 | } 67 | 68 | resource "null_resource" "wait_for_worker_instances_to_be_fulfilled" { 69 | provisioner "local-exec" { 70 | command = "sleep ${var.seconds_to_wait_for_worker_fulfillment}" 71 | } 72 | triggers { 73 | instances_to_wait_on = "${join(",", aws_spot_instance_request.kubernetes_workers.*.id)}" 74 | } 75 | } 76 | 77 | resource "aws_route" "kubernetes_pods_within_workers" { 78 | depends_on = [ "null_resource.wait_for_worker_instances_to_be_fulfilled" ] 79 | count = "${var.number_of_workers_per_cluster}" 80 | route_table_id = "${aws_route_table.kubernetes_clusters.id}" 81 | destination_cidr_block = "${replace(var.kubernetes_pod_cidr_block, "/^([0-9]{1,3}.[0-9]{1,3}).*/", "$1.${count.index+1}.0/24")}" 82 | instance_id = "${aws_spot_instance_request.kubernetes_workers.*.spot_instance_id[count.index]}" 83 | } 84 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/load_balancer.tf: -------------------------------------------------------------------------------- 1 | resource "aws_elb" "kubernetes_control_plane" { 2 | name = "kubernetes" 3 | subnets = [ "${aws_subnet.kubernetes_control_plane.*.id}" ] 4 | instances = [ "${aws_spot_instance_request.kubernetes_control_plane.*.spot_instance_id}" ] 5 | security_groups = [ "${aws_security_group.kubernetes_control_plane_lb.id}" ] 6 | listener { 7 | instance_port = "${local.kubernetes_internal_port}" 8 | instance_protocol = "TCP" 9 | lb_port = "${local.kubernetes_public_port}" 10 | lb_protocol = "TCP" 11 | } 12 | health_check { 13 | healthy_threshold = 3 14 | unhealthy_threshold = 10 15 | target = "HTTP:80/" 16 | interval = 10 17 | timeout = 5 18 | } 19 | } 20 | 21 | resource "aws_route53_record" "kubernetes_public_address" { 22 | zone_id = "${data.aws_route53_zone.route53_zone_for_domain.zone_id}" 23 | name = "kubernetes" 24 | type = "CNAME" 25 | ttl = "1" 26 | records = [ "${aws_elb.kubernetes_control_plane.dns_name}" ] 27 | } 28 | 29 | output "kubernetes_control_plane_dns_address" { 30 | value = "${aws_route53_record.kubernetes_public_address.fqdn}" 31 | } 32 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/networking.tf: -------------------------------------------------------------------------------- 1 | resource "aws_vpc" "kubernetes_clusters" { 2 | cidr_block = "${var.cidr_block_for_kubernetes_clusters}" 3 | enable_dns_support = true 4 | enable_dns_hostnames = true 5 | tags = "${var.base_tags}" 6 | } 7 | 8 | resource "aws_internet_gateway" "kubernetes_clusters" { 9 | vpc_id = "${aws_vpc.kubernetes_clusters.id}" 10 | tags = "${var.base_tags}" 11 | } 12 | 13 | resource "aws_route_table" "kubernetes_clusters" { 14 | vpc_id = "${aws_vpc.kubernetes_clusters.id}" 15 | route { 16 | cidr_block = "0.0.0.0/0" 17 | gateway_id = "${aws_internet_gateway.kubernetes_clusters.id}" 18 | } 19 | } 20 | 21 | resource "aws_subnet" "kubernetes_control_plane" { 22 | count = "${var.number_of_zones}" 23 | vpc_id = "${aws_vpc.kubernetes_clusters.id}" 24 | availability_zone = "${data.aws_availability_zones.available_to_this_account.names[count.index]}" 25 | cidr_block = "${local.subnet_cidr_blocks[count.index]}" 26 | map_public_ip_on_launch = true 27 | tags = "${var.base_tags}" 28 | } 29 | 30 | resource "aws_route_table_association" "kubernetes_control_plane" { 31 | count = "${var.number_of_zones}" 32 | subnet_id = "${aws_subnet.kubernetes_control_plane.*.id[count.index]}" 33 | route_table_id = "${aws_route_table.kubernetes_clusters.id}" 34 | } 35 | 36 | resource "aws_subnet" "kubernetes_workers" { 37 | count = "${var.number_of_zones}" 38 | vpc_id = "${aws_vpc.kubernetes_clusters.id}" 39 | availability_zone = "${data.aws_availability_zones.available_to_this_account.names[count.index]}" 40 | cidr_block = "${local.worker_subnet_cidr_blocks[count.index]}" 41 | map_public_ip_on_launch = true 42 | tags = "${var.base_tags}" 43 | } 44 | 45 | resource "aws_route_table_association" "kubernetes_workers" { 46 | count = "${var.number_of_zones}" 47 | subnet_id = "${aws_subnet.kubernetes_workers.*.id[count.index]}" 48 | route_table_id = "${aws_route_table.kubernetes_clusters.id}" 49 | } 50 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/security_groups.tf: -------------------------------------------------------------------------------- 1 | resource "aws_security_group" "kubernetes_control_plane_lb" { 2 | name = "kubernetes_lb" 3 | description = "Allows inbound access to this Kubernetes cluster" 4 | tags = "${var.base_tags}" 5 | vpc_id = "${aws_vpc.kubernetes_clusters.id}" 6 | ingress { 7 | from_port = "${local.kubernetes_public_port}" 8 | to_port = "${local.kubernetes_public_port}" 9 | protocol = "tcp" 10 | cidr_blocks = [ "0.0.0.0/0" ] 11 | } 12 | egress { 13 | from_port = 0 14 | to_port = 0 15 | protocol = -1 16 | cidr_blocks = [ "0.0.0.0/0" ] 17 | } 18 | } 19 | 20 | resource "aws_security_group" "kubernetes_clusters" { 21 | name = "kubernetes_control_plane" 22 | vpc_id = "${aws_vpc.kubernetes_clusters.id}" 23 | tags = "${var.base_tags}" 24 | ingress { 25 | from_port = 0 26 | to_port = 0 27 | protocol = "-1" 28 | self = true 29 | } 30 | ingress { 31 | from_port = 0 32 | to_port = 0 33 | protocol = "-1" 34 | cidr_blocks = [ "${var.kubernetes_pod_cidr_block}" ] 35 | } 36 | ingress { 37 | from_port = 22 38 | to_port = 22 39 | protocol = "tcp" 40 | cidr_blocks = [ "${var.provisioning_machine_ip_address}/32" ] 41 | } 42 | ingress { 43 | from_port = "${var.kubernetes_internal_port}" 44 | to_port = "${var.kubernetes_internal_port}" 45 | protocol = "tcp" 46 | security_groups = [ "${aws_security_group.kubernetes_control_plane_lb.id}" ] 47 | } 48 | ingress { 49 | from_port = "80" 50 | to_port = "80" 51 | protocol = "tcp" 52 | security_groups = [ "${aws_security_group.kubernetes_control_plane_lb.id}" ] 53 | } 54 | egress { 55 | from_port = 0 56 | to_port = 0 57 | protocol = "-1" 58 | cidr_blocks = [ "0.0.0.0/0" ] 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/tags/all.json: -------------------------------------------------------------------------------- 1 | { 2 | "Domain": { 3 | "value": "$DOMAIN_NAME", 4 | "propagate": true 5 | }, 6 | "Environment": { 7 | "value": "$ENVIRONMENT_NAME", 8 | "propagate": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/tags/etcd.json: -------------------------------------------------------------------------------- 1 | { 2 | "kubernetes_role": { 3 | "value": "etcd", 4 | "propagate": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/tags/kubernetes_controller.json: -------------------------------------------------------------------------------- 1 | { 2 | "kubernetes_version": { 3 | "value": "$KUBERNETES_VERSION", 4 | "propagate": true 5 | }, 6 | "kubernetes_role": { 7 | "value": "controller", 8 | "propagate": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/tags/kubernetes_worker.json: -------------------------------------------------------------------------------- 1 | { 2 | "kubernetes_version": { 3 | "value": "$KUBERNETES_VERSION", 4 | "propagate": true 5 | }, 6 | "kubernetes_role": { 7 | "value": "worker", 8 | "propagate": true 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kubernetes/control_plane/aws/templates/user_data: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # vim: set filetype=sh : 3 | 4 | # Change the hostname so that the domain is included within. 5 | # This will allow our certificates to validate successfully. 6 | # Clone the source code for our configuration management tool. 7 | infrastructure_code_path=/infra_code 8 | config_management_code_path="$infrastructure_code_path/kubernetes/config/${ kubernetes_configuration_management_tool }" 9 | 10 | git clone "${ kubernetes_configuration_github_repository }" "$infrastructure_code_path" 11 | 12 | if [ "${ kubernetes_configuration_management_tool }" == "ansible" ] 13 | then 14 | cat >"/etc/ansible_vars.yml" </etc/systemd/system/configuration_management.service </root/.aws/credentials <&2 echo "WARNING: No .env file was provided. Using local environment instead." 5 | else 6 | export $(grep -v '^#' "$ENV_FILE" | xargs) 7 | fi 8 | 9 | AWS_REGION="${AWS_REGION?Please provide an AWS region to operate in.}" 10 | AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID?Please provide an access key.}" 11 | AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY?Please provide a secret key.}" 12 | TERRAFORM_STATE_S3_BUCKET="${TERRAFORM_STATE_S3_BUCKET?Please provide the S3 bucket into which state will be stored.}" 13 | TERRAFORM_STATE_S3_KEY="${TERRAFORM_STATE_S3_KEY?Please provide the S3 key within the bucket into which state will be stored.}" 14 | SSH_KEY_S3_BUCKET_NAME="${SSH_KEY_S3_BUCKET_NAME?Please provide the bucket containing our SSH keys within S3.}" 15 | SSH_KEY_S3_KEY_PATH="${SSH_KEY_S3_KEY_PATH?Please provide the path to our SSH keys within S3.}" 16 | KUBERNETES_PUBLIC_PORT="${KUBERNETES_PUBLIC_PORT?Please provide the port to access Kubernetes on.}" 17 | KUBERNETES_INFRASTRUCTURE_SOURCE_PATH="$(git rev-parse --show-toplevel)/kubernetes/control_plane/aws" 18 | TERRAFORM_BACKEND_PATH="${KUBERNETES_INFRASTRUCTURE_SOURCE_PATH}/backend.tf" 19 | TERRAFORM_PROVIDER_PATH="${KUBERNETES_INFRASTRUCTURE_SOURCE_PATH}/provider.tf" 20 | AUTOGENERATED_TERRAFORM_TFVARS_PATH="${KUBERNETES_INFRASTRUCTURE_SOURCE_PATH}/terraform.tfvars" 21 | TEMPLATED_TERRAFORM_TFVARS_PATH="${AUTOGENERATED_TERRAFORM_TFVARS_PATH}.template" 22 | PRIVATE_KEY_PATH_VARIABLE="${PRIVATE_KEY_PATH_VARIABLE:-temporary_ssh_key}" 23 | SSH_TEMP_KEY_NAME="${SSH_TEMP_KEY_FORMAT:-/tmp/kubernetes_cluster_ssh_key}" 24 | 25 | create_s3_backend() { 26 | cat >"$TERRAFORM_BACKEND_PATH" <"${TERRAFORM_PROVIDER_PATH}" <&2 echo "ERROR: Please provide terraform.tfvars template file at $TEMPLATED_TERRAFORM_TFVARS_PATH" 51 | return 1 52 | fi 53 | cat "$TEMPLATED_TERRAFORM_TFVARS_PATH" | \ 54 | while read -r key_value_pair 55 | do 56 | if ! _is_key_value_pair "$key_value_pair" 57 | then 58 | _copy_line_verbatim "$key_value_pair" "$AUTOGENERATED_TERRAFORM_TFVARS_PATH" 59 | continue 60 | fi 61 | terraform_variable=$(echo "$key_value_pair" | cut -f1 -d =) 62 | env_var_to_use=$(echo "$key_value_pair" | cut -f2 -d = | tr -d '"' | tr -d '$') 63 | env_var_value="${!env_var_to_use}" 64 | if [ -z "$env_var_value" ] 65 | then 66 | >&2 echo "WARN: '$env_var_to_use' is not defined. \ 67 | Its associated Terraform variable will be commented out." 68 | _comment_out_missing_terraform_variable "$terraform_variable" "$AUTOGENERATED_TERRAFORM_TFVARS_PATH" 69 | else 70 | _fill_in_templated_value "$terraform_variable" \ 71 | "$AUTOGENERATED_TERRAFORM_TFVARS_PATH" \ 72 | "$env_var_value" 73 | fi 74 | done 75 | } 76 | 77 | _comment_out_missing_terraform_variable() { 78 | terraform_variable="${1?Please provide the Terraform variable.}" 79 | tfvars_path="${2?Please provide the path to the Terraform variables to manipulate.}" 80 | echo "#${terraform_variable}=\"nothing provided\"" >> "$tfvars_path" 81 | } 82 | 83 | _fill_in_templated_value() { 84 | terraform_variable="${1?Please provide the Terraform variable.}" 85 | tfvars_path="${2?Please provide the path to the Terraform variables to manipulate.}" 86 | actual_value="${3?Please provide the actual value being substituted.}" 87 | echo "${terraform_variable}=\"$env_var_value\"" >> "$tfvars_path" 88 | } 89 | 90 | _is_key_value_pair() { 91 | key_value_pair_under_test="$1" 92 | echo "$key_value_pair_under_test" | grep -Eq "^[a-zA-Z0-9_]{1,}=\".*\"$" 93 | } 94 | 95 | _copy_line_verbatim() { 96 | key_value_pair="${1?Please provide a kvp.}" 97 | tfvars_path="${2?Please provide a path to the tfvars file being generated.}" 98 | echo "$key_value_pair" >> "$tfvars_path" 99 | } 100 | 101 | add_our_external_ip_address() { 102 | tfvars_path="${1?Please provide the path to the variables file.}" 103 | if ! ip_address=$(curl -sL 'https://api.ipify.org?format=text') 104 | then 105 | return 1 106 | fi 107 | echo "provisioning_machine_ip_address=\"$ip_address\"" >> "$tfvars_path" 108 | } 109 | 110 | remove_existing_terraform_variables_if_any() { 111 | tfvars_path="${1?Please provide the path to the variables file.}" 112 | rm -f "$tfvars_path" &>/dev/null 113 | } 114 | 115 | _create_s3_ssh_key_bucket_if_not_present() { 116 | return_code=0 117 | if ! aws s3 ls "s3://$SSH_KEY_S3_BUCKET_NAME" &>/dev/null 118 | then 119 | aws s3 mb "s3://$SSH_KEY_S3_BUCKET_NAME" &>/dev/null 120 | return_code=$? 121 | fi 122 | return "$return_code" 123 | } 124 | 125 | _create_and_download_ssh_key_from_s3() { 126 | set -e 127 | path_to_download_to="${1?Please provide the path to download the SSH key into.}" 128 | if [ -f "$path_to_download_to" ] 129 | then 130 | rm -f "$path_to_download_to" &>/dev/null 131 | fi 132 | if ! aws s3 ls "s3://$SSH_KEY_S3_BUCKET_NAME/$SSH_KEY_S3_KEY_PATH/$ENVIRONMENT_NAME" &>/dev/null 133 | then 134 | >&2 echo "WARN: SSH key not found; creating." 135 | ssh-keygen -t rsa -f "$path_to_download_to" -b 2048 -N "" &>/dev/null; 136 | aws s3 cp "$path_to_download_to" "s3://$SSH_KEY_S3_BUCKET_NAME/$SSH_KEY_S3_KEY_PATH/$ENVIRONMENT_NAME" &>/dev/null; 137 | else 138 | aws s3 cp "s3://$SSH_KEY_S3_BUCKET_NAME/$SSH_KEY_S3_KEY_PATH/$ENVIRONMENT_NAME" "$path_to_download_to" &>/dev/null 139 | fi 140 | set +e 141 | } 142 | 143 | delete_preexisting_ssh_keys() { 144 | rm -rf "${SSH_TEMP_KEY_NAME}"* 145 | } 146 | 147 | create_or_retrieve_kubernetes_nodes_key_pair() { 148 | tfvars_path="${1?Please provide the path to the variables file.}" 149 | temporary_ssh_keypair_path="$SSH_TEMP_KEY_NAME" 150 | if ! _create_s3_ssh_key_bucket_if_not_present 151 | then 152 | >&2 echo "ERROR: Unable to create SSH key bucket." 153 | return 1 154 | fi 155 | if ! _create_and_download_ssh_key_from_s3 "$temporary_ssh_keypair_path" 156 | then 157 | >&2 echo "ERROR: Unable to download SSH key from S3." 158 | fi 159 | chmod 600 "$temporary_ssh_keypair_path" 160 | public_key_to_use=$(ssh-keygen -y -f "$temporary_ssh_keypair_path") 161 | echo "kubernetes_cluster_public_key=\"${public_key_to_use}\"" >> "$tfvars_path" 162 | echo "${PRIVATE_KEY_PATH_VARIABLE}=\"${temporary_ssh_keypair_path}\"" >> "$tfvars_path" 163 | echo "INFO: SSH key for instances is located at: $temporary_ssh_keypair_path" 164 | } 165 | 166 | remove_existing_terraform_variables_if_any "$AUTOGENERATED_TERRAFORM_TFVARS_PATH" 167 | if ! add_our_external_ip_address "$AUTOGENERATED_TERRAFORM_TFVARS_PATH" 168 | then 169 | >&2 echo "ERROR: Unable to fetch our external IP address. Check your \ 170 | internet connection." 171 | exit 1 172 | fi 173 | 174 | delete_preexisting_ssh_keys 175 | create_or_retrieve_kubernetes_nodes_key_pair "$AUTOGENERATED_TERRAFORM_TFVARS_PATH" && \ 176 | create_s3_backend && \ 177 | configure_aws_provider && \ 178 | configure_terraform_variables 179 | -------------------------------------------------------------------------------- /kubernetes/include/scripts/generate_tags.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ -z "$ENV_FILE" ] || [ ! -f "$ENV_FILE" ] 3 | then 4 | >&2 echo "WARNING: No .env file was provided. Using local environment instead." 5 | else 6 | export $(grep -v '^#' "$ENV_FILE" | xargs) 7 | fi 8 | TAGS_PATH="${TAGS_PATH?Please provide the path containing the tags to be created (in JSON form).}" 9 | TERRAFORM_TFVARS_PATH="${TERRAFORM_TFVARS_PATH?Please provide the path to your Terraform variable inputs file.}" 10 | 11 | _print_tags_with_trailing_commas_at_end_of_lists() { 12 | echo "$1" | \ 13 | grep -v '^[ \t]\+$' | \ 14 | tr '\n' '\f' | \ 15 | sed 's#,\f]#\f]#g' | \ 16 | sed 's#,\f}#\f}#g' | \ 17 | tr '\f' '\n' 18 | } 19 | 20 | generate_key_value_tags() { 21 | json="${1?Please provide the JSON tree containing the tags to manipulate.}" 22 | echo -e "\n$(echo "$json" | \ 23 | jq -r 'to_entries[] | .key + " = \"" + .value.value + "\","' | \ 24 | sed 's/^/ /')\n" 25 | } 26 | 27 | generate_asg_compatible_tags() { 28 | json="${1?Please provide the JSON tree containing the tags to manipulate.}" 29 | echo -e "\n$(echo "$json" | \ 30 | jq -r 'to_entries[] | " {\n key = \"" + .key + "\"\n value = \"" + .value.value + "\"\n propagate_at_launch = " + (.value.propagate|tostring) + "\n },"' | \ 31 | sed 's/^/ /')\n" 32 | } 33 | 34 | get_tags_json() { 35 | tag_file="${1?Please provide the tag file to read.}" 36 | tag_file_location="${TAGS_PATH}/${tag_file}.json" 37 | if [ ! -f "$tag_file_location" ] 38 | then 39 | >&2 echo "ERROR: File not found: $tag_file_location" 40 | return 1 41 | fi 42 | json=$(cat "$tag_file_location") 43 | environment_variables_found=$(echo "$json" | \ 44 | grep -E '"\$[A-Z0-9_]{1,}"' | \ 45 | sed 's/.*"\$\([A-Z0-9_]\+\)".*/\1/' 46 | ) 47 | for environment_variable in $environment_variables_found 48 | do 49 | environment_variable_value="${!environment_variable}" 50 | if [ ! -z "$environment_variable_value" ] 51 | then 52 | json=$(echo "$json" | sed "s/\"\$${environment_variable}\"/\"$environment_variable_value\"/g") 53 | fi 54 | done 55 | echo "$json" 56 | } 57 | 58 | generate_tag_variables() { 59 | base_tags_json=$(get_tags_json "all") 60 | base_kvp_tags=$(generate_key_value_tags "$base_tags_json") 61 | base_asg_tags=$(generate_asg_compatible_tags "$base_tags_json") 62 | all_tags=$(cat <> "$TERRAFORM_TFVARS_PATH" 98 | -------------------------------------------------------------------------------- /kubernetes/tools/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | MAINTAINER Carlos Nunez 3 | 4 | ENV CERT_UTILS_VERSION=1.2 5 | ENV KUBECTL_VERSION=1.10.2 6 | ENV EXTRA_BINARIES=jq,bash,curl 7 | 8 | # Copy our kubectl initialization script 9 | COPY entrypoint.sh /usr/local/bin/entrypoint.sh 10 | RUN chmod 755 /usr/local/bin/entrypoint.sh 11 | 12 | # Install additional useful utilities 13 | RUN apk add --update-cache $(echo "$EXTRA_BINARIES" | tr ',' ' ') 14 | 15 | # Install cfssl and cfjson for creating a PKI and creating certificate with it. 16 | USER root 17 | RUN for package in cfssl cfssljson; \ 18 | do \ 19 | curl --output "/usr/local/bin/$package" \ 20 | --location \ 21 | "http://pkg.cfssl.org/R${CERT_UTILS_VERSION}/${package}_linux-amd64" && \ 22 | chmod +x "/usr/local/bin/$package"; \ 23 | done 24 | 25 | # Install kubectl so you can do stuff with the k8s control plane 26 | RUN curl --output /usr/local/bin/kubectl \ 27 | --location \ 28 | https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ 29 | chmod +x /usr/local/bin/kubectl 30 | 31 | # Create a scratch directory 32 | RUN mkdir /scratch && chown 1000 /scratch 33 | 34 | # Make the entrypoint bash 35 | WORKDIR /scratch 36 | ENTRYPOINT [ "entrypoint.sh" ] 37 | -------------------------------------------------------------------------------- /kubernetes/tools/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | KUBERNETES_CLUSTER_ADDRESS="${KUBERNETES_CLUSTER_ADDRESS:-127.0.0.1}" 3 | KUBERNETES_CLUSTER_NAME="${KUBERNETES_CLUSTER_NAME:-kubernetes}" 4 | KUBERNETES_CERT_PATH="${KUBERNETES_CERT_PATH:-/certs}" 5 | KUBERNETES_USER_NAME="${KUBERNETES_USER_NAME:-admin}" 6 | KUBERNTES_CLUSTER_CONTEXT="${KUBERNTES_CLUSTER_CONTEXT:-kubernetes}" 7 | USER_CERT="${USER_CERT:-$KUBERNETES_CERT_PATH/admin.pem}" 8 | USER_CERT_PRIVATE_KEY="${USER_CERT_PRIVATE_KEY:-$KUBERNETES_CERT_PATH/admin-key.pem}" 9 | CA_CERT="${CA_CERT:-$KUBERNETES_CERT_PATH/ca.pem}" 10 | usage() { 11 | cat </dev/null 49 | then 50 | >&2 echo "ERROR: kubectl is not installed." 51 | exit 1 52 | fi 53 | } 54 | 55 | ensure_certs_are_provided_or_die() { 56 | for cert in "$USER_CERT" "$USER_CERT_PRIVATE_KEY" "$CA_CERT" 57 | do 58 | if [ ! -f "${cert}" ] 59 | then 60 | >&2 echo "ERROR: Missing certificate: ${cert}" 61 | exit 1 62 | fi 63 | done 64 | } 65 | 66 | set_kubernetes_cluster() { 67 | &>/dev/null kubectl config set-cluster "${KUBERNETES_CLUSTER_NAME}" \ 68 | --certificate-authority=$CA_CERT \ 69 | --embed-certs=true \ 70 | --server=https://${KUBERNETES_CLUSTER_ADDRESS}:6443 71 | } 72 | 73 | set_credentials() { 74 | &>/dev/null kubectl config set-credentials "${KUBERNETES_USER_NAME}" \ 75 | --client-certificate="${USER_CERT}" \ 76 | --client-key="${USER_CERT_PRIVATE_KEY}" 77 | } 78 | 79 | set_cluster_context() { 80 | &>/dev/null kubectl config set-context "$KUBERNTES_CLUSTER_CONTEXT" \ 81 | --cluster="$KUBERNETES_CLUSTER_NAME" \ 82 | --user="$KUBERNETES_USER_NAME" 83 | } 84 | 85 | enable_context() { 86 | &>/dev/null kubectl config use-context "$KUBERNTES_CLUSTER_CONTEXT" 87 | } 88 | 89 | if [ "$1" == "kubectl" ] 90 | then 91 | ensure_kubectl_is_installed_or_die && 92 | ensure_certs_are_provided_or_die && 93 | set_kubernetes_cluster && 94 | set_credentials && 95 | set_cluster_context && 96 | enable_context && 97 | exec "$@" 98 | else 99 | exec "$@" 100 | fi 101 | -------------------------------------------------------------------------------- /kubernetes/tools/tests/1_ensure_kubectl_present.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | @test "Ensure that a Docker image was provided" { 3 | [ "$DOCKER_IMAGE_UNDER_TEST" != "" ] 4 | } 5 | 6 | @test "Ensure that kubectl is present and in the right place" { 7 | expected_exit_code=0 8 | run bash -c "docker run --entrypoint bash \ 9 | "$DOCKER_IMAGE_UNDER_TEST" \ 10 | -c 'kubectl version --client'" 11 | >&2 echo "Test failed. Output: $output" 12 | [ "$status" -eq "$expected_exit_code" ] 13 | [ "$output" != "" ] 14 | } 15 | -------------------------------------------------------------------------------- /kubernetes/tools/tests/2_ensure_additional_binaries_are_present.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | @test "Ensure that required packages are present" { 3 | for package in jq curl 4 | do 5 | expected_exit_code=0 6 | run bash -c "docker run --entrypoint bash \ 7 | "$DOCKER_IMAGE_UNDER_TEST" \ 8 | -c 'which $package'" 9 | >&2 echo "Test failed. Output: $output. Expected: /usr/bin/$package" 10 | [ "$status" -eq "$expected_exit_code" ] 11 | [ "$output" == "/usr/bin/$package" ] 12 | done 13 | for package in cfssl cfssljson 14 | do 15 | expected_exit_code=0 16 | run bash -c "docker run --entrypoint bash \ 17 | "$DOCKER_IMAGE_UNDER_TEST" \ 18 | -c 'which $package'" 19 | >&2 echo "Test failed. Output: $output. Expected: /usr/local/bin/$package" 20 | [ "$status" -eq "$expected_exit_code" ] 21 | [ "$output" == "/usr/local/bin/$package" ] 22 | done 23 | } 24 | --------------------------------------------------------------------------------