├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── addons ├── autoscaler │ └── cluster-autoscaler.yml ├── dashboard │ └── kubernetes-dashboard.yml ├── dns │ ├── kube-dns-autoscaler.yml │ ├── kube-dns-controller.yml │ ├── kube-dns-service.yml.tpl │ └── kube-dns-serviceaccount.yml ├── heapster │ └── heapster.yml ├── logging │ ├── elasticsearch-logging.yml │ ├── fluentd-logging.yml │ └── kibana-logging.yml └── rescheduler │ └── kube-rescheduler.yml ├── circle.yml ├── io.tf ├── makefiles ├── etcd.mk ├── help.mk ├── keypair.mk ├── route53.mk ├── shortcuts.mk ├── ssl.mk └── terraform.mk ├── modules.tf ├── modules ├── bastion │ ├── cloud-config.tf │ ├── cloud-config.yml │ ├── ec2.tf │ └── io.tf ├── etcd │ ├── cloud-config.tf │ ├── cloud-config.yml │ ├── ec2.tf │ ├── elb.tf │ └── io.tf ├── iam │ ├── bastion.tf │ ├── etcd.tf │ ├── io.tf │ ├── pki.tf │ └── worker.tf ├── pki │ ├── cloud-config.tf │ ├── cloud-config.yml │ ├── ec2.tf │ ├── io.tf │ ├── null-resource.tf │ └── route53.tf ├── route53 │ ├── io.tf │ └── route53.tf ├── s3 │ ├── io.tf │ └── s3.tf ├── security │ ├── io.tf │ ├── null-resource.tf │ └── security.tf ├── vpc-existing │ └── io.tf ├── vpc │ ├── io.tf │ ├── private.tf │ ├── public.tf │ └── vpc.tf └── worker │ ├── cloud-config.tf │ ├── cloud-config.yml │ ├── ec2.tf │ └── io.tf ├── modules_override.tf ├── scripts ├── create-admin-certificate ├── create-default-storage-class ├── create-kube-dns-service ├── create-kube-system-configmap ├── create-kubeconfig ├── dashboard ├── delete-addons ├── do-task ├── get-ca ├── init-variables ├── instances ├── myip ├── prereqs ├── retry ├── ssh ├── status ├── test-etcd ├── tmux ├── utils ├── wait-for-cluster └── watch-pods-until ├── test ├── pods │ └── busybox.yml └── redis │ ├── redis-service.yml │ └── redist-deployment.yml └── vpc-existing.tfvars /.gitignore: -------------------------------------------------------------------------------- 1 | .addons 2 | .cfssl 3 | .key-pair 4 | .keypair 5 | .kube 6 | .srl 7 | .ssl 8 | .terraform 9 | 10 | terraform.tfplan 11 | terraform.tfstate* 12 | terraform.tfvars 13 | .terraform.tfstate.lock.info 14 | 15 | templates 16 | *.bak 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 kz8s 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL += -eu 2 | 3 | BLUE := \033[0;34m 4 | GREEN := \033[0;32m 5 | RED := \033[0;31m 6 | NC := \033[0m 7 | 8 | export DIR_KEY_PAIR := .keypair 9 | export DIR_SSL := .cfssl 10 | export DIR_KUBECONFIG := .kube 11 | 12 | # CIDR_PODS: flannel overlay range 13 | # - https://coreos.com/flannel/docs/latest/flannel-config.html 14 | # 15 | # CIDR_SERVICE_CLUSTER: apiserver parameter --service-cluster-ip-range 16 | # - http://kubernetes.io/docs/admin/kube-apiserver/ 17 | # 18 | # CIDR_VPC: vpc subnet 19 | # - http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Subnets.html#VPC_Sizing 20 | # - https://www.terraform.io/docs/providers/aws/r/vpc.html#cidr_block 21 | # 22 | 23 | # ∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨∨ 24 | 25 | export AWS_REGION ?= us-west-2 26 | export COREOS_CHANNEL ?= stable 27 | export COREOS_VM_TYPE ?= hvm 28 | export CLUSTER_NAME ?= test 29 | 30 | export AWS_EC2_KEY_NAME ?= kz8s-$(CLUSTER_NAME) 31 | export AWS_EC2_KEY_PATH := ${DIR_KEY_PAIR}/${AWS_EC2_KEY_NAME}.pem 32 | export INTERNAL_TLD := ${CLUSTER_NAME}.kz8s 33 | 34 | export HYPERKUBE_IMAGE ?= quay.io/coreos/hyperkube 35 | export HYPERKUBE_TAG ?= v1.7.4_coreos.0 36 | 37 | export CIDR_VPC ?= 10.0.0.0/16 38 | export CIDR_PODS ?= 10.2.0.0/16 39 | export CIDR_SERVICE_CLUSTER ?= 10.3.0.0/24 40 | 41 | export K8S_SERVICE_IP ?= 10.3.0.1 42 | export K8S_DNS_IP ?= 10.3.0.10 43 | 44 | export ETCD_IPS ?= 10.0.10.10,10.0.10.11,10.0.10.12 45 | 46 | export PKI_IP ?= 10.0.10.9 47 | 48 | # Alternative: 49 | # CIDR_PODS ?= "172.15.0.0/16" 50 | # CIDR_SERVICE_CLUSTER ?= "172.16.0.0/24" 51 | # K8S_SERVICE_IP ?= 172.16.0.1 52 | # K8S_DNS_IP ?= 172.16.0.10 53 | 54 | # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 55 | 56 | .addons: ; @scripts/do-task "initialize add-ons" ./scripts/init-addons 57 | 58 | ## generate key-pair, variables and then `terraform apply` 59 | all: prereqs create-keypair init apply 60 | @echo "${GREEN}✓ terraform portion of 'make all' has completed ${NC}\n" 61 | @$(MAKE) post-terraform 62 | 63 | .PHONY: post-terraform 64 | post-terraform: 65 | @$(MAKE) instances 66 | @$(MAKE) get-ca 67 | @$(MAKE) create-admin-certificate 68 | @$(MAKE) create-kubeconfig 69 | @$(MAKE) wait-for-cluster 70 | @$(MAKE) create-addons 71 | @$(MAKE) create-busybox 72 | kubectl get nodes -o wide 73 | kubectl --namespace=kube-system get cs 74 | @echo "etcd-0 incorrectly reporting as unhelathy" 75 | @echo "https://github.com/kubernetes/kubernetes/issues/27343" 76 | @echo "https://github.com/kubernetes/kubernetes/pull/39716" 77 | @echo "View nodes:" 78 | @echo "% make nodes" 79 | @echo "---" 80 | @echo "View uninitialized kube-system pods:" 81 | @echo "% make pods" 82 | @echo "---" 83 | @echo "View ec2 instance info:" 84 | @echo "% make instances" 85 | @echo "---" 86 | @echo "Status summaries:" 87 | @echo "% make status" 88 | @echo "---" 89 | @scripts/watch-pods-until 90 | 91 | 92 | ## destroy and remove everything 93 | clean: delete-addons destroy delete-keypair 94 | @-pkill -f "kubectl proxy" ||: 95 | @-rm terraform.tfvars ||: 96 | @-rm terraform.tfplan ||: 97 | @-rm -rf .terraform ||: 98 | @-rm -rf tmp ||: 99 | @-rm -rf ${DIR_SSL} ||: 100 | 101 | ## create kube-system addons 102 | create-addons: 103 | scripts/create-default-storage-class 104 | scripts/create-kube-dns-service 105 | scripts/create-kube-system-configmap 106 | kubectl apply --recursive -f addons 107 | 108 | create-admin-certificate: ; @scripts/do-task "create admin certificate" \ 109 | scripts/create-admin-certificate 110 | 111 | create-busybox: ; @scripts/do-task "create busybox test pod" \ 112 | kubectl create -f test/pods/busybox.yml 113 | 114 | create-kubeconfig: ; @scripts/do-task "create kubeconfig" \ 115 | scripts/create-kubeconfig 116 | 117 | ## start proxy and open kubernetes dashboard 118 | dashboard: ; @./scripts/dashboard 119 | 120 | ## delete addons 121 | delete-addons: ; @-scripts/delete-addons 122 | 123 | ## get ca certificate 124 | get-ca: ; scripts/do-task "get root ca certificate" scripts/get-ca 125 | 126 | ## show instance information 127 | instances: ; @scripts/instances 128 | 129 | ## journalctl on etcd1 130 | journal: ; @scripts/ssh "ssh `terraform output etcd1-ip` journalctl -fl" 131 | 132 | prereqs: ; @scripts/do-task "checking prerequisities" scripts/prereqs 133 | 134 | ## ssh into etcd1 135 | ssh: ; @scripts/ssh "ssh `terraform output etcd1-ip`" 136 | 137 | ## ssh into bastion host 138 | ssh-bastion: ; @scripts/ssh 139 | 140 | ## status 141 | status: instances ; scripts/status 142 | 143 | ## smoke it 144 | test: test-ssl test-route53 test-etcd pods dns 145 | 146 | wait-for-cluster: ; @scripts/do-task "wait-for-cluster" scripts/wait-for-cluster 147 | 148 | include makefiles/*.mk 149 | 150 | .DEFAULT_GOAL := help 151 | .PHONY: all clean create-addons create-admin-certificate create-busybox 152 | .PHONY: delete-addons get-ca instances journal prereqs ssh ssh-bastion ssl 153 | .PHONY: status test wait-for-cluster 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # terraform-aws-coreos-kubernetes 2 | 3 | [![Circle CI](https://circleci.com/gh/kz8s/tack.svg?style=svg)](https://circleci.com/gh/kz8s/tack) 4 | 5 | Opinionated [Terraform](https://terraform.io) module for creating a Highly Available [Kubernetes](http://kubernetes.io) cluster running on 6 | [Container Linux by CoreOS](https://coreos.com) (any channel) in an [AWS 7 | Virtual Private Cloud VPC](https://aws.amazon.com/vpc/). With prerequisites 8 | installed `make all` will simply spin up a default cluster; and, since it is 9 | based on Terraform, customization is much easier than 10 | [CloudFormation](https://aws.amazon.com/cloudformation/). 11 | 12 | The default configuration includes Kubernetes 13 | [add-ons](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons): 14 | DNS, Dashboard and UI. 15 | 16 | ## tl;dr 17 | ```bash 18 | # prereqs 19 | $ brew update && brew install awscli cfssl jq kubernetes-cli terraform 20 | 21 | # build artifacts and deploy cluster 22 | $ make all 23 | 24 | # nodes 25 | $ kubectl get nodes 26 | 27 | # addons 28 | $ kubectl get pods --namespace=kube-system 29 | 30 | # verify dns - run after addons have fully loaded 31 | $ kubectl exec busybox -- nslookup kubernetes 32 | 33 | # open dashboard 34 | $ make dashboard 35 | 36 | # obliterate the cluster and all artifacts 37 | $ make clean 38 | ``` 39 | 40 | ## Component and Tool Versions 41 | 42 | component / tool | version 43 | ---------------- | -------: 44 | Container Linux by CoreOS | 1409.7.0, 1465.3.0, 1492.1.0 45 | kubernetes | 1.7.4 46 | flanneld | 0.7.1 47 | docker | 1.12.6 48 | etcd | 3.1.6 49 | rkt | 1.25.0 50 | terraform | 0.10.0 51 | cfssl | 1.2.0 52 | aws-cli | aws-cli/1.11.129 Python/2.7.10 Darwin/16.7.0 botocore/1.5.92 53 | jq | 1.5 54 | 55 | 56 | ## Features 57 | 58 | * Cluster-internal Certificate Authority infrastructure for TLS certificate generation 59 | * etcd3 60 | 61 | ### AWS 62 | 63 | * [EC2 Key Pair](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) 64 | creation 65 | * AWS VPC Public and Private subnets 66 | * IAM protected S3 bucket for asset distribution 67 | * Bastion Host 68 | * Multi-AZ Auto-Scaling Worker Nodes 69 | * VPC NAT Gateway 70 | * VPC Endpoint for simplified S3 access from EC2 instances 71 | 72 | ### Container Linux by CoreOS 73 | 74 | * etcd3 DNS Discovery Bootstrap 75 | * kubelet runs under rkt (using Container Linux by CoreOS recommended [Kubelet Wrapper Script](https://coreos.com/kubernetes/docs/latest/kubelet-wrapper.html)) 76 | 77 | ### Kubernetes 78 | 79 | * [Highly Available ApiServer Configuration](https://kubernetes.io/docs/admin/high-availability/) 80 | * Service accounts enabled 81 | 82 | ### Terraform 83 | 84 | * Container Linux by CoreOS AMI sourcing 85 | * Terraform Pattern Modules 86 | 87 | ## Prerequisites 88 | 89 | * [AWS Command Line Interface](http://aws.amazon.com/documentation/cli/) 90 | * [CFSSL: CloudFlare's PKI and TLS toolkit](https://cfssl.org/) 91 | * [jq](https://stedolan.github.io/jq/) 92 | * [kubectl](http://kubernetes.io/v1.1/docs/user-guide/kubectl-overview.html) 93 | * [Terraform](https://www.terraform.io/) 94 | 95 | Quick install prerequisites on Mac OS X with [Homebrew](http://brew.sh/): 96 | 97 | ```bash 98 | $ brew update && brew install awscli cfssl jq kubernetes-cli terraform 99 | ``` 100 | 101 | ## Launch Cluster 102 | 103 | `make all` will create: 104 | - AWS Key Pair (PEM file) 105 | - AWS VPC with private and public subnets 106 | - Route 53 internal zone for VPC 107 | - Bastion host 108 | - Certificate Authority server 109 | - etcd3 cluster bootstrapped from Route 53 110 | - High Availability Kubernetes configuration (masters running on etcd nodes) 111 | - Autoscaling worker node group across subnets in selected region 112 | - kube-system namespace and addons: DNS, UI, Dashboard 113 | 114 | ```bash 115 | make all 116 | ``` 117 | 118 | To open dashboard: 119 | 120 | ```bash 121 | make dashboard 122 | ``` 123 | 124 | To display instance information: 125 | 126 | ```bash 127 | make instances 128 | ``` 129 | 130 | To display status: 131 | 132 | ```bash 133 | make status 134 | ``` 135 | 136 | To destroy, remove and generally undo everything: 137 | 138 | ``` 139 | make clean 140 | ``` 141 | 142 | `make all` and `make clean` should be idempotent - should an error occur simply try running 143 | the command again and things should recover from that point. 144 | 145 | ## How Tack works 146 | 147 | ### Tack Phases 148 | 149 | Tack works in three phases: 150 | 151 | 1. Pre-Terraform 152 | 2. Terraform 153 | 3. Post-Terraform 154 | 155 | #### Pre-Terraform 156 | 157 | The purpose of this phase is to prep the environment for Terraform execution. Some tasks are 158 | hard or messy to do in Terraform - a little prep work can go a long way here. Determining 159 | the Container Linux by CoreOS AMI for a given region, channel and VM Type for instance is easy enough to do 160 | with a simple shell script. 161 | 162 | #### Terraform 163 | 164 | Terraform does the heavy lifting of resource creation and sequencing. Tack uses local 165 | modules to partition the work in a logical way. Although it is of course possible to do all 166 | of the Terraform work in a single `.tf` file or collection of `.tf` files, it becomes 167 | unwieldy quickly and impossible to debug. Breaking the work into local modules makes the 168 | flow much easier to follow and provides the basis for composing variable solutions down the track - for example converting the worker Auto Scaling Group to use spot instances. 169 | 170 | #### Post-Terraform 171 | 172 | Once the infrastructure has been configured and instantiated it will take some time for it 173 | to settle. Waiting for the 'master' ELB to become healthy is an example of this. 174 | 175 | ### Components 176 | 177 | Like many great tools, _tack_ has started out as a collection of scripts, makefiles and other tools. As _tack_ matures and patterns crystalize it will evolve to a Terraform plugin and perhaps a Go-based cli tool for 'init-ing' new cluster configurations. The tooling will compose Terraform modules into a solution based on user preferences - think `npm init` or better yet [yeoman](http://yeoman.io/). 178 | 179 | #### TLS Certificates 180 | 181 | * [etcd3 coreos cloudint](https://github.com/coreos/coreos-cloudinit/blob/master/config/etcd.go) 182 | 183 | ```bash 184 | curl --cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/k8s-etcd.pem --key /etc/kubernetes/ssl/k8s-etcd-key.pem https://etcd.test.kz8s:2379/health 185 | openssl x509 -text -noout -in /etc/kubernetes/ssl/ca.pem 186 | openssl x509 -text -noout -in /etc/kubernetes/ssl/k8s-etcd.pem 187 | ``` 188 | 189 | #### ElasticSearch and Kibana 190 | 191 | To access Elasticseach and Kibana first start `kubectl proxy`. 192 | 193 | ```bash 194 | $ kubectl proxy 195 | Starting to serve on localhost:8001 196 | ``` 197 | 198 | * [http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging ](http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/elasticsearch-logging) 199 | * [http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/kibana-logging/app/kibana](http://localhost:8001/api/v1/proxy/namespaces/kube-system/services/kibana-logging/app/kibana) 200 | 201 | ## FAQs 202 | 203 | * [Create an etcd cluster with more than 3 instances](https://github.com/kz8s/tack/wiki/How-to:-change-etcd-cluster-size) 204 | 205 | ## Advanced Features and Configuration 206 | 207 | ### Using an Existing VPC 208 | 209 | If you have an existing VPC you'd like to deploy a cluster into, there is an option for this with _tack_. 210 | 211 | #### Constraints 212 | 213 | * You will need to allocate 3 static IPs for the etcd servers - Choose 3 unused IPs that fall within the IP range of the first subnet specified in `subnet-ids-private` under `vpc-existing.tfvars` 214 | * Your VPC has to have private and public subnets (for now) 215 | * You will need to know the following information: 216 | * VPC CIDR Range (e.g. 192.168.0.0/16) 217 | * VPC Id (e.g. vpc-abc123) 218 | * VPC Internet Gateway Id (e.g. igw-123bbd) 219 | * VPC Public Subnet Ids (e.g. subnet-xyz123,subnet-zyx123) 220 | * VPC Private Subnet Ids (e.g. subnet-lmn123,subnet-opq123) 221 | 222 | #### Enabling Existing VPC Support 223 | 224 | * Edit vpc-existing.tfvars 225 | * Uncomment the blocks with variables and fill in the missing information 226 | * Edit modules_override.tf - This uses the [overrides feature from Terraform](https://www.terraform.io/docs/configuration/override.html) 227 | * Uncomment the vpc module, this will override the reference to the regular VPC module and instead use the stub vpc-existing module which just pulls in the variables from vpc-existing.tfvars 228 | * Edit the Makefile as necessary for CIDR_PODS, CIDR_SERVICE_CLUSTER, etc to match what you need (e.g. avoid collisions with existing IP ranges in your VPC or extended infrastructure) 229 | 230 | #### Testing Existing VPC Support from Scratch 231 | 232 | In order to test existing VPC support, we need to generate a VPC and then try the overrides with it. After that we can clean it all up. These instructions are meant for someone wanting to ensure that the _tack_ existing VPC code works properly. 233 | * Run `make all` to generate a VPC with Terraform 234 | * Edit terraform.tfstate 235 | * Search for the VPC block and cut it out and save it somewhere. Look for "path": ["root","vpc"] 236 | * Run `make clean` to remove everything but the VPC and associated networking (we preserved it in the previous step) 237 | * Edit as per instructions above 238 | * Run `make all` to test out using an existing VPC 239 | * Cleaning up: 240 | * Re-insert the VPC block into terraform.tfstate 241 | * Run `make clean` to clean up everything 242 | 243 | #### Additional Configuration 244 | 245 | * You should to [tag your subnets](https://github.com/kubernetes/kubernetes/blob/master/pkg/cloudprovider/providers/aws/aws.go#66) for internal/external load balancers 246 | 247 | ## Inspiration 248 | 249 | * [Code examples to create Container Linux by CoreOS cluster on AWS with Terraform](https://github.com/xuwang/aws-terraform) by [xuwang](https://github.com/xuwang) 250 | * [kaws: tool for deploying multiple Kubernetes clusters](https://github.com/InQuicker/kaws) 251 | * [Kubernetes on Container Linux by CoreOS](https://github.com/coreos/coreos-kubernetes) 252 | * [Terraform Infrastructure Design Patterns](https://www.opencredo.com/2015/09/14/terraform-infrastructure-design-patterns/) by [Bart Spaans](https://www.opencredo.com/author/bart/) 253 | * [The infrastructure that runs Brandform](https://github.com/brandfolder/infrastructure) 254 | * [AutoScaling your Kubernetes cluster on AWS](https://renzedevries.wordpress.com/2017/01/10/autoscaling-your-kubernetes-cluster-on-aws/) 255 | * [Bash template substitution for manifests - from kayrus/elk-kubernetes](https://github.com/kayrus/elk-kubernetes/blob/master/deploy.sh) 256 | 257 | ## Other Terraform Projects 258 | 259 | * [bakins/kubernetes-coreos-terraform](https://github.com/bakins/kubernetes-coreos-terraform) 260 | * [bobtfish/terraform-aws-coreos-kubernates-cluster](https://github.com/bobtfish/terraform-aws-coreos-kubernates-cluster) 261 | * [chiefy/tf-aws-kubernetes](https://github.com/chiefy/tf-aws-kubernetes) 262 | * [cihangir/terraform-aws-kubernetes](https://github.com/cihangir/terraform-aws-kubernetes) 263 | * [ericandrewlewis/kubernetes-via-terraform](https://github.com/ericandrewlewis/kubernetes-via-terraform) 264 | * [funkymonkeymonk/terraform-demo](https://github.com/funkymonkeymonk/terraform-demo) 265 | * [kelseyhightower/kubestack](https://github.com/kelseyhightower/kubestack) 266 | * [samsung-cnct/kraken](https://github.com/samsung-cnct/kraken) 267 | * [wearemakery/kubestack-aws](https://github.com/wearemakery/kubestack-aws) 268 | * [xuwang/aws-terraform](https://github.com/xuwang/aws-terraform) 269 | 270 | ## References 271 | 272 | * [CFSSL: CloudFlare's PKI and TLS toolkit](https://cfssl.org/) 273 | * [Container Linux by CoreOS - Mounting Storage](https://coreos.com/os/docs/latest/mounting-storage.html) 274 | * [Deploying Container Linux by CoreOS cluster with etcd secured by TLS/SSL](http://blog.skrobul.com/securing_etcd_with_tls/) 275 | * [etcd dns discovery bootstrap](https://coreos.com/etcd/docs/latest/clustering.html#dns-discovery) 276 | * [Generate EC2 Key Pair](https://github.com/xuwang/aws-terraform/blob/master/scripts/aws-keypair.sh) 277 | * [Generate self-signed certificates](https://coreos.com/os/docs/latest/generate-self-signed-certificates.html) 278 | * [kubectl Cheat Sheet](https://kubernetes.io/docs/user-guide/kubectl-cheatsheet/) 279 | * [Makefile `help` target](https://gist.github.com/rcmachado/af3db315e31383502660) 280 | * [Peeking under the hood of Kubernetes on AWS](https://github.com/kubernetes/kubernetes/blob/release-1.2/docs/design/aws_under_the_hood.md) 281 | * [Self documenting Makefile](https://gist.github.com/prwhite/8168133) 282 | * [Setting up etcd to run in production](https://github.com/kelseyhightower/etcd-production-setup) 283 | * [ssl artifact generation](https://github.com/coreos/coreos-kubernetes/tree/master/lib) 284 | * [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) 285 | * [Persistent Storage - Kubernetes on AWS](http://kubernetes-on-aws.readthedocs.io/en/latest/user-guide/using-volumes.html) 286 | * [VPC endpoint Terraform example setup](https://gist.github.com/radeksimko/929a41675323eefed023) 287 | -------------------------------------------------------------------------------- /addons/autoscaler/cluster-autoscaler.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1beta1 3 | kind: Deployment 4 | metadata: 5 | name: cluster-autoscaler 6 | namespace: kube-system 7 | labels: 8 | k8s-app: cluster-autoscaler 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: cluster-autoscaler 14 | template: 15 | metadata: 16 | labels: 17 | app: cluster-autoscaler 18 | annotations: 19 | scheduler.alpha.kubernetes.io/critical-pod: '' 20 | spec: 21 | tolerations: 22 | - key: node-role.kubernetes.io/master 23 | operator: Exists 24 | effect: NoSchedule 25 | - key: "CriticalAddonsOnly" 26 | operator: "Exists" 27 | containers: 28 | - image: gcr.io/google_containers/cluster-autoscaler:v0.5.2 29 | name: cluster-autoscaler 30 | resources: 31 | limits: 32 | cpu: 100m 33 | memory: 300Mi 34 | requests: 35 | cpu: 100m 36 | memory: 300Mi 37 | command: 38 | - ./cluster-autoscaler 39 | - --cloud-provider=aws 40 | - --nodes=1:5:$(WORKER_ASG_NAME) 41 | - --scale-down-delay=5m 42 | - --skip-nodes-with-local-storage=true 43 | - --skip-nodes-with-system-pods=false 44 | - --stderrthreshold=info 45 | - --v=4 46 | - --write-status-configmap=true 47 | env: 48 | - name: AWS_REGION 49 | valueFrom: 50 | configMapKeyRef: 51 | name: kube-system-config 52 | key: AWS_REGION 53 | - name: WORKER_ASG_NAME 54 | valueFrom: 55 | configMapKeyRef: 56 | name: kube-system-config 57 | key: WORKER_ASG_NAME 58 | envFrom: 59 | - configMapRef: 60 | name: kube-system-config 61 | volumeMounts: 62 | - name: ssl-certs 63 | mountPath: /etc/ssl/certs/ca-certificates.crt 64 | readOnly: true 65 | imagePullPolicy: "Always" 66 | volumes: 67 | - name: ssl-certs 68 | hostPath: 69 | path: "/etc/ssl/certs/ca-certificates.crt" 70 | -------------------------------------------------------------------------------- /addons/dashboard/kubernetes-dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | 4 | metadata: 5 | name: kubernetes-dashboard 6 | namespace: kube-system 7 | labels: 8 | k8s-app: kubernetes-dashboard 9 | kubernetes.io/cluster-service: "true" 10 | 11 | spec: 12 | selector: 13 | k8s-app: kubernetes-dashboard 14 | ports: 15 | - port: 80 16 | targetPort: 9090 17 | 18 | --- 19 | 20 | apiVersion: apps/v1beta1 21 | kind: Deployment 22 | 23 | metadata: 24 | name: kubernetes-dashboard 25 | namespace: kube-system 26 | labels: 27 | k8s-app: kubernetes-dashboard 28 | kubernetes.io/cluster-service: "true" 29 | 30 | spec: 31 | selector: 32 | matchLabels: 33 | k8s-app: kubernetes-dashboard 34 | 35 | template: 36 | metadata: 37 | labels: 38 | k8s-app: kubernetes-dashboard 39 | annotations: 40 | scheduler.alpha.kubernetes.io/critical-pod: '' 41 | spec: 42 | tolerations: 43 | - key: node-role.kubernetes.io/master 44 | operator: Exists 45 | effect: NoSchedule 46 | - key: "CriticalAddonsOnly" 47 | operator: "Exists" 48 | containers: 49 | - name: kubernetes-dashboard 50 | image: gcr.io/google_containers/kubernetes-dashboard-amd64:v1.6.3 51 | resources: 52 | # keep request = limit to keep this container in guaranteed class 53 | limits: 54 | cpu: 100m 55 | memory: 50Mi 56 | requests: 57 | cpu: 100m 58 | memory: 50Mi 59 | ports: 60 | - containerPort: 9090 61 | protocol: TCP 62 | livenessProbe: 63 | httpGet: 64 | path: / 65 | port: 9090 66 | initialDelaySeconds: 30 67 | timeoutSeconds: 30 68 | -------------------------------------------------------------------------------- /addons/dns/kube-dns-autoscaler.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta1 2 | kind: Deployment 3 | 4 | metadata: 5 | name: kube-dns-autoscaler 6 | namespace: kube-system 7 | labels: 8 | k8s-app: kube-dns-autoscaler 9 | kubernetes.io/cluster-service: "true" 10 | 11 | spec: 12 | template: 13 | metadata: 14 | labels: 15 | k8s-app: kube-dns-autoscaler 16 | annotations: 17 | scheduler.alpha.kubernetes.io/critical-pod: '' 18 | spec: 19 | tolerations: 20 | - key: node-role.kubernetes.io/master 21 | operator: Exists 22 | effect: NoSchedule 23 | - key: "CriticalAddonsOnly" 24 | operator: "Exists" 25 | containers: 26 | - name: autoscaler 27 | image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.1-r2 28 | resources: 29 | requests: 30 | cpu: "20m" 31 | memory: "10Mi" 32 | command: 33 | - /cluster-proportional-autoscaler 34 | - --namespace=kube-system 35 | - --configmap=kube-dns-autoscaler 36 | # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base 37 | - --target=Deployment/kube-dns 38 | # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. 39 | # If using small nodes, "nodesPerReplica" should dominate. 40 | - --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"preventSinglePointFailure":true}} 41 | - --logtostderr=true 42 | - --v=2 43 | serviceAccountName: kube-dns 44 | -------------------------------------------------------------------------------- /addons/dns/kube-dns-controller.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: apps/v1beta1 16 | kind: Deployment 17 | metadata: 18 | name: kube-dns 19 | namespace: kube-system 20 | labels: 21 | k8s-app: kube-dns 22 | kubernetes.io/cluster-service: 'true' 23 | spec: 24 | # replicas: not specified here: 25 | # 1. In order to make Addon Manager do not reconcile this replicas parameter. 26 | # 2. Default is 1. 27 | # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on. 28 | strategy: 29 | rollingUpdate: 30 | maxSurge: 10% 31 | maxUnavailable: 0 32 | selector: 33 | matchLabels: 34 | k8s-app: kube-dns 35 | template: 36 | metadata: 37 | labels: 38 | k8s-app: kube-dns 39 | annotations: 40 | scheduler.alpha.kubernetes.io/critical-pod: '' 41 | spec: 42 | tolerations: 43 | - key: node-role.kubernetes.io/master 44 | operator: Exists 45 | effect: NoSchedule 46 | - key: "CriticalAddonsOnly" 47 | operator: "Exists" 48 | volumes: 49 | - name: kube-dns-config 50 | configMap: 51 | name: kube-dns 52 | optional: true 53 | containers: 54 | - name: kubedns 55 | image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1 56 | resources: 57 | # TODO: Set memory limits when we've profiled the container for large 58 | # clusters, then set request = limit to keep this container in 59 | # guaranteed class. Currently, this container falls into the 60 | # "burstable" category so the kubelet doesn't backoff from restarting it. 61 | limits: 62 | memory: 170Mi 63 | requests: 64 | cpu: 100m 65 | memory: 70Mi 66 | livenessProbe: 67 | httpGet: 68 | path: /healthcheck/kubedns 69 | port: 10054 70 | scheme: HTTP 71 | initialDelaySeconds: 60 72 | timeoutSeconds: 5 73 | successThreshold: 1 74 | failureThreshold: 5 75 | readinessProbe: 76 | httpGet: 77 | path: /readiness 78 | port: 8081 79 | scheme: HTTP 80 | # we poll on pod startup for the Kubernetes master service and 81 | # only setup the /readiness HTTP server once that's available. 82 | initialDelaySeconds: 3 83 | timeoutSeconds: 5 84 | args: 85 | - --domain=$(CLUSTER_DOMAIN). 86 | - --dns-port=10053 87 | - --config-dir=/kube-dns-config 88 | - --v=2 89 | env: 90 | - name: PROMETHEUS_PORT 91 | value: '10055' 92 | - name: CLUSTER_DOMAIN 93 | valueFrom: 94 | configMapKeyRef: 95 | name: kube-system-config 96 | key: CLUSTER_DOMAIN 97 | envFrom: 98 | - configMapRef: 99 | name: kube-system-config 100 | ports: 101 | - containerPort: 10053 102 | name: dns-local 103 | protocol: UDP 104 | - containerPort: 10053 105 | name: dns-tcp-local 106 | protocol: TCP 107 | - containerPort: 10055 108 | name: metrics 109 | protocol: TCP 110 | volumeMounts: 111 | - name: kube-dns-config 112 | mountPath: /kube-dns-config 113 | - name: dnsmasq 114 | image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1 115 | livenessProbe: 116 | httpGet: 117 | path: /healthcheck/dnsmasq 118 | port: 10054 119 | scheme: HTTP 120 | initialDelaySeconds: 60 121 | timeoutSeconds: 5 122 | successThreshold: 1 123 | failureThreshold: 5 124 | args: 125 | - -v=2 126 | - -logtostderr 127 | - -configDir=/etc/k8s/dns/dnsmasq-nanny 128 | - -restartDnsmasq=true 129 | - -- 130 | - -k 131 | - --cache-size=1000 132 | - --log-facility=- 133 | - --server=/$(CLUSTER_DOMAIN)/127.0.0.1#10053 134 | - --server=/in-addr.arpa/127.0.0.1#10053 135 | - --server=/ip6.arpa/127.0.0.1#10053 136 | env: 137 | - name: CLUSTER_DOMAIN 138 | valueFrom: 139 | configMapKeyRef: 140 | name: kube-system-config 141 | key: CLUSTER_DOMAIN 142 | envFrom: 143 | - configMapRef: 144 | name: kube-system-config 145 | ports: 146 | - containerPort: 53 147 | name: dns 148 | protocol: UDP 149 | - containerPort: 53 150 | name: dns-tcp 151 | protocol: TCP 152 | # see: https://github.com/kubernetes/kubernetes/issues/29055 for details 153 | resources: 154 | requests: 155 | cpu: 150m 156 | memory: 20Mi 157 | volumeMounts: 158 | - name: kube-dns-config 159 | mountPath: /etc/k8s/dns/dnsmasq-nanny 160 | - name: sidecar 161 | image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1 162 | livenessProbe: 163 | httpGet: 164 | path: /metrics 165 | port: 10054 166 | scheme: HTTP 167 | initialDelaySeconds: 60 168 | timeoutSeconds: 5 169 | successThreshold: 1 170 | failureThreshold: 5 171 | args: 172 | - --v=2 173 | - --logtostderr 174 | - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.$(CLUSTER_DOMAIN),5,A 175 | - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.$(CLUSTER_DOMAIN),5,A 176 | env: 177 | - name: CLUSTER_DOMAIN 178 | valueFrom: 179 | configMapKeyRef: 180 | name: kube-system-config 181 | key: CLUSTER_DOMAIN 182 | envFrom: 183 | - configMapRef: 184 | name: kube-system-config 185 | ports: 186 | - containerPort: 10054 187 | name: metrics 188 | protocol: TCP 189 | resources: 190 | requests: 191 | memory: 20Mi 192 | cpu: 10m 193 | dnsPolicy: Default # Don't use cluster DNS. 194 | serviceAccountName: kube-dns 195 | -------------------------------------------------------------------------------- /addons/dns/kube-dns-service.yml.tpl: -------------------------------------------------------------------------------- 1 | # Copyright 2016 The Kubernetes Authors. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: v1 16 | kind: Service 17 | metadata: 18 | name: kube-dns 19 | namespace: kube-system 20 | labels: 21 | k8s-app: kube-dns 22 | kubernetes.io/cluster-service: 'true' 23 | kubernetes.io/name: "KubeDNS" 24 | spec: 25 | selector: 26 | k8s-app: kube-dns 27 | clusterIP: ${DNS_SERVICE_IP} 28 | ports: 29 | - name: dns 30 | port: 53 31 | protocol: UDP 32 | - name: dns-tcp 33 | port: 53 34 | protocol: TCP 35 | -------------------------------------------------------------------------------- /addons/dns/kube-dns-serviceaccount.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: kube-dns 5 | namespace: kube-system 6 | labels: 7 | kubernetes.io/cluster-service: "true" 8 | -------------------------------------------------------------------------------- /addons/heapster/heapster.yml: -------------------------------------------------------------------------------- 1 | kind: Service 2 | apiVersion: v1 3 | 4 | metadata: 5 | name: heapster 6 | namespace: kube-system 7 | labels: 8 | kubernetes.io/cluster-service: "true" 9 | kubernetes.io/name: "Heapster" 10 | 11 | spec: 12 | ports: 13 | - port: 80 14 | targetPort: 8082 15 | selector: 16 | k8s-app: heapster 17 | 18 | --- 19 | 20 | apiVersion: apps/v1beta1 21 | kind: Deployment 22 | 23 | metadata: 24 | name: heapster-v1.3.0 25 | namespace: kube-system 26 | labels: 27 | k8s-app: heapster 28 | kubernetes.io/cluster-service: "true" 29 | version: v1.3.0 30 | 31 | spec: 32 | replicas: 1 33 | selector: 34 | matchLabels: 35 | k8s-app: heapster 36 | version: v1.3.0 37 | template: 38 | metadata: 39 | labels: 40 | k8s-app: heapster 41 | version: v1.3.0 42 | annotations: 43 | scheduler.alpha.kubernetes.io/critical-pod: '' 44 | spec: 45 | tolerations: 46 | - key: node-role.kubernetes.io/master 47 | operator: Exists 48 | effect: NoSchedule 49 | - key: "CriticalAddonsOnly" 50 | operator: "Exists" 51 | containers: 52 | - image: gcr.io/google_containers/heapster:v1.3.0 53 | name: heapster 54 | livenessProbe: 55 | httpGet: 56 | path: /healthz 57 | port: 8082 58 | scheme: HTTP 59 | initialDelaySeconds: 180 60 | timeoutSeconds: 5 61 | resources: 62 | limits: 63 | cpu: 80m 64 | memory: 200Mi 65 | requests: 66 | cpu: 80m 67 | memory: 200Mi 68 | command: 69 | - /heapster 70 | - --source=kubernetes.summary_api:'' 71 | - image: gcr.io/google_containers/addon-resizer:1.7 72 | name: heapster-nanny 73 | resources: 74 | limits: 75 | cpu: 50m 76 | memory: 90Mi 77 | requests: 78 | cpu: 50m 79 | memory: 90Mi 80 | env: 81 | - name: MY_POD_NAME 82 | valueFrom: 83 | fieldRef: 84 | fieldPath: metadata.name 85 | - name: MY_POD_NAMESPACE 86 | valueFrom: 87 | fieldRef: 88 | fieldPath: metadata.namespace 89 | command: 90 | - /pod_nanny 91 | - --cpu=80m 92 | - --extra-cpu=4m 93 | - --memory=200Mi 94 | - --extra-memory=4Mi 95 | - --threshold=5 96 | - --deployment=heapster-v1.3.0 97 | - --container=heapster 98 | - --poll-period=300000 99 | - --estimator=exponential 100 | -------------------------------------------------------------------------------- /addons/logging/elasticsearch-logging.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | 4 | metadata: 5 | name: elasticsearch-logging 6 | namespace: kube-system 7 | labels: 8 | k8s-app: elasticsearch-logging 9 | kubernetes.io/cluster-service: "true" 10 | kubernetes.io/name: "Elasticsearch" 11 | spec: 12 | ports: 13 | - port: 9200 14 | protocol: TCP 15 | targetPort: db 16 | selector: 17 | k8s-app: elasticsearch-logging 18 | 19 | --- 20 | 21 | apiVersion: apps/v1beta1 22 | kind: StatefulSet 23 | 24 | metadata: 25 | name: elasticsearch-logging 26 | namespace: kube-system 27 | labels: 28 | k8s-app: elasticsearch-logging 29 | kubernetes.io/cluster-service: "true" 30 | version: v1 31 | 32 | spec: 33 | replicas: 2 34 | serviceName: elasticsearch-logging 35 | template: 36 | metadata: 37 | labels: 38 | k8s-app: elasticsearch-logging 39 | version: v1 40 | kubernetes.io/cluster-service: "true" 41 | annotations: 42 | scheduler.alpha.kubernetes.io/critical-pod: '' 43 | spec: 44 | tolerations: 45 | - key: node-role.kubernetes.io/master 46 | operator: Exists 47 | effect: NoSchedule 48 | - key: "CriticalAddonsOnly" 49 | operator: "Exists" 50 | containers: 51 | - image: gcr.io/google_containers/elasticsearch:v2.4.1-2 52 | name: elasticsearch-logging 53 | resources: 54 | limits: 55 | cpu: 1000m 56 | requests: 57 | cpu: 100m 58 | ports: 59 | - containerPort: 9200 60 | name: db 61 | protocol: TCP 62 | - containerPort: 9300 63 | name: transport 64 | protocol: TCP 65 | volumeMounts: 66 | - name: es-persistent-storage 67 | mountPath: /data 68 | env: 69 | - name: "NAMESPACE" 70 | valueFrom: 71 | fieldRef: 72 | fieldPath: metadata.namespace 73 | volumeClaimTemplates: 74 | - metadata: 75 | name: es-persistent-storage 76 | annotations: 77 | volume.beta.kubernetes.io/storage-class: default 78 | spec: 79 | accessModes: 80 | - ReadWriteOnce 81 | resources: 82 | requests: 83 | storage: 25Gi 84 | -------------------------------------------------------------------------------- /addons/logging/fluentd-logging.yml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: DaemonSet 3 | 4 | metadata: 5 | name: fluentd 6 | namespace: kube-system 7 | labels: 8 | k8s-app: fluentd-logging 9 | version: v1 10 | kubernetes.io/cluster-service: "true" 11 | 12 | spec: 13 | template: 14 | metadata: 15 | name: fluentd 16 | labels: 17 | k8s-app: fluentd-logging 18 | version: v1 19 | kubernetes.io/cluster-service: "true" 20 | spec: 21 | tolerations: 22 | - key: node-role.kubernetes.io/master 23 | operator: Exists 24 | effect: NoSchedule 25 | containers: 26 | - name: fluentd 27 | image: fluent/fluentd-kubernetes-daemonset:v0.12.33-elasticsearch 28 | env: 29 | - name: FLUENT_ELASTICSEARCH_HOST 30 | value: "elasticsearch-logging" 31 | - name: FLUENT_ELASTICSEARCH_PORT 32 | value: "9200" 33 | resources: 34 | limits: 35 | memory: 200Mi 36 | requests: 37 | cpu: 100m 38 | memory: 200Mi 39 | volumeMounts: 40 | - name: varlog 41 | mountPath: /var/log 42 | - name: varlibdockercontainers 43 | mountPath: /var/lib/docker/containers 44 | readOnly: true 45 | terminationGracePeriodSeconds: 30 46 | volumes: 47 | - name: varlog 48 | hostPath: 49 | path: /var/log 50 | - name: varlibdockercontainers 51 | hostPath: 52 | path: /var/lib/docker/containers 53 | -------------------------------------------------------------------------------- /addons/logging/kibana-logging.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | 4 | metadata: 5 | name: kibana-logging 6 | namespace: kube-system 7 | labels: 8 | k8s-app: kibana-logging 9 | kubernetes.io/cluster-service: "true" 10 | kubernetes.io/name: "Kibana" 11 | 12 | spec: 13 | ports: 14 | - port: 5601 15 | protocol: TCP 16 | targetPort: ui 17 | selector: 18 | k8s-app: kibana-logging 19 | 20 | --- 21 | 22 | apiVersion: apps/v1beta1 23 | kind: Deployment 24 | 25 | metadata: 26 | name: kibana-logging 27 | namespace: kube-system 28 | labels: 29 | k8s-app: kibana-logging 30 | kubernetes.io/cluster-service: "true" 31 | 32 | spec: 33 | replicas: 1 34 | selector: 35 | matchLabels: 36 | k8s-app: kibana-logging 37 | template: 38 | metadata: 39 | labels: 40 | k8s-app: kibana-logging 41 | annotations: 42 | scheduler.alpha.kubernetes.io/critical-pod: '' 43 | spec: 44 | tolerations: 45 | - key: node-role.kubernetes.io/master 46 | operator: Exists 47 | effect: NoSchedule 48 | - key: "CriticalAddonsOnly" 49 | operator: "Exists" 50 | containers: 51 | - name: kibana-logging 52 | image: gcr.io/google_containers/kibana:v4.6.1-1 53 | resources: 54 | # keep request = limit to keep this container in guaranteed class 55 | limits: 56 | cpu: 100m 57 | requests: 58 | cpu: 100m 59 | env: 60 | - name: "ELASTICSEARCH_URL" 61 | value: "http://elasticsearch-logging:9200" 62 | - name: "KIBANA_BASE_URL" 63 | value: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging" 64 | ports: 65 | - containerPort: 5601 66 | name: ui 67 | protocol: TCP 68 | -------------------------------------------------------------------------------- /addons/rescheduler/kube-rescheduler.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1beta1 3 | kind: Deployment 4 | metadata: 5 | name: kube-rescheduler 6 | namespace: kube-system 7 | labels: 8 | k8s-app: kube-rescheduler 9 | kubernetes.io/cluster-service: "true" 10 | kubernetes.io/name: "Rescheduler" 11 | spec: 12 | # `replicas` should always be the default of 1, rescheduler crashes otherwise 13 | template: 14 | metadata: 15 | labels: 16 | k8s-app: kube-rescheduler 17 | annotations: 18 | scheduler.alpha.kubernetes.io/critical-pod: '' 19 | spec: 20 | tolerations: 21 | - key: node-role.kubernetes.io/master 22 | operator: Exists 23 | effect: NoSchedule 24 | - key: "CriticalAddonsOnly" 25 | operator: "Exists" 26 | hostNetwork: true 27 | containers: 28 | - name: kube-rescheduler 29 | image: gcr.io/google-containers/rescheduler:v0.3.0 30 | resources: 31 | requests: 32 | cpu: 10m 33 | memory: 100Mi 34 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | # circle.yml 2 | general: 3 | branches: 4 | only: 5 | - master 6 | - test 7 | 8 | machine: 9 | environment: 10 | CFSSL_VERSION: R1.2 11 | TERRAFORM_VERSION: 0.9.11 12 | 13 | dependencies: 14 | pre: 15 | - wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip 16 | - unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d ~/bin 17 | - curl -s -L -o ~/bin/cfssl https://pkg.cfssl.org/${CFSSL_VERSION}/cfssl_linux-amd64 18 | - curl -s -L -o ~/bin/cfssljson https://pkg.cfssl.org/${CFSSL_VERSION}/cfssljson_linux-amd64 19 | - chmod +x ~/bin/{cfssl,cfssljson} 20 | 21 | test: 22 | pre: 23 | - make ssl 24 | # override: 25 | # - make plan 26 | # post: 27 | # - make clean 28 | 29 | notify: 30 | webhooks: 31 | - url: https://webhooks.gitter.im/e/87c066fe542f9b00f79e 32 | -------------------------------------------------------------------------------- /io.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { region = "${ var.aws["region"] }" } 2 | 3 | # variables 4 | variable "aws" { 5 | default = { 6 | account-id = "" 7 | azs = "" 8 | key-name = "" 9 | region = "" 10 | } 11 | } 12 | variable "cidr" { 13 | default = { 14 | allow-ssh = "0.0.0.0/0" 15 | pods = "10.2.0.0/16" 16 | service-cluster = "10.3.0.0/24" 17 | vpc = "10.0.0.0/16" 18 | } 19 | } 20 | variable "cluster-domain" { default = "cluster.local" } 21 | variable "coreos-aws" { 22 | default = { 23 | ami = "" 24 | channel = "" 25 | type = "" 26 | } 27 | } 28 | variable "dns-service-ip" { default = "10.3.0.10" } 29 | variable "etcd-ips" { default = "10.0.10.10,10.0.10.11,10.0.10.12" } 30 | variable "instance-type" { 31 | default = { 32 | bastion = "t2.nano" 33 | pki = "t2.nano" 34 | etcd = "m3.large" 35 | worker = "m3.large" 36 | } 37 | } 38 | variable "internal-tld" {} 39 | variable "k8s" { 40 | default = { 41 | hyperkube-image = "quay.io/coreos/hyperkube" 42 | hyperkube-tag = "v1.5.1_coreos.0" 43 | } 44 | } 45 | variable "k8s-service-ip" { default = "10.3.0.1" } 46 | variable "name" {} 47 | variable "pki-ip" {} 48 | variable "s3-bucket" {} 49 | variable "vpc-existing" { 50 | default = { 51 | id = "" 52 | gateway-id = "" 53 | subnet-ids-public = "" 54 | subnet-ids-private = "" 55 | } 56 | } 57 | 58 | # outputs 59 | output "azs" { value = "${ var.aws["azs"] }" } 60 | output "bastion-ip" { value = "${ module.bastion.ip }" } 61 | output "cluster-domain" { value = "${ var.cluster-domain }" } 62 | output "dns-service-ip" { value = "${ var.dns-service-ip }" } 63 | output "etcd1-ip" { value = "${ element( split(",", var.etcd-ips), 0 ) }" } 64 | output "external-elb" { value = "${ module.etcd.external-elb }" } 65 | output "internal-tld" { value = "${ var.internal-tld }" } 66 | output "name" { value = "${ var.name }" } 67 | output "pki-ip" { value = "${ module.pki.ip }" } 68 | output "region" { value = "${ var.aws["region"] }" } 69 | output "s3-bucket" { value = "${ module.s3.bucket }" } 70 | output "subnet-ids-private" { value = "${ module.vpc.subnet-ids-private }" } 71 | output "subnet-ids-public" { value = "${ module.vpc.subnet-ids-public }" } 72 | output "worker-autoscaling-group-name" { value = "${ module.worker.autoscaling-group-name }" } 73 | 74 | output "ips" { 75 | value = "${ 76 | map( 77 | "bastion", "${ module.bastion.ip }", 78 | "dns-service", "${ var.dns-service-ip }", 79 | "etcd", "${ var.etcd-ips }", 80 | "pki", "${ module.pki.ip }", 81 | ) 82 | }" 83 | } 84 | -------------------------------------------------------------------------------- /makefiles/etcd.mk: -------------------------------------------------------------------------------- 1 | test-etcd: 2 | ssh -A core@`terraform output bastion-ip` \ 3 | '( curl -s http://etcd.`terraform output internal-tld`:2379/version )' \ 4 | | grep '{"etcdserver":"2.3.2","etcdcluster":"2.3.0"}' 5 | ssh -A core@`terraform output bastion-ip` \ 6 | '( curl -s http://etcd.`terraform output internal-tld`:2379/health )' \ 7 | | grep '{"health": "true"}' 8 | ssh -A core@`terraform output bastion-ip` \ 9 | '( curl -s http://etcd.`terraform output internal-tld`:2379/v2/members )' \ 10 | | grep -o '"name":' | wc -l | grep 3 11 | 12 | .PHONY: test-etcd 13 | -------------------------------------------------------------------------------- /makefiles/help.mk: -------------------------------------------------------------------------------- 1 | ## display this help text 2 | help: 3 | $(info Available targets) 4 | @awk '/^[a-zA-Z\-\_0-9]+:/ { \ 5 | nb = sub( /^## /, "", helpMsg ); \ 6 | if(nb == 0) { \ 7 | helpMsg = $$0; \ 8 | nb = sub( /^[^:]*:.* ## /, "", helpMsg ); \ 9 | } \ 10 | if (nb) \ 11 | print $$1 "\t" helpMsg; \ 12 | } \ 13 | { helpMsg = $$0 }' \ 14 | $(MAKEFILE_LIST) | column -ts $$'\t' | \ 15 | grep --color '^[^ ]*' 16 | 17 | .PHONY: help 18 | -------------------------------------------------------------------------------- /makefiles/keypair.mk: -------------------------------------------------------------------------------- 1 | $(DIR_KEY_PAIR)/: ; mkdir -p $@ 2 | 3 | $(DIR_KEY_PAIR)/$(AWS_EC2_KEY_NAME).pem: | $(DIR_KEY_PAIR)/ 4 | @aws --region ${AWS_REGION} ec2 create-key-pair \ 5 | --key-name ${AWS_EC2_KEY_NAME} \ 6 | --query 'KeyMaterial' \ 7 | --output text \ 8 | > $@ 9 | @chmod 400 $@ 10 | 11 | ## create ec2 key-pair 12 | create-keypair: $(DIR_KEY_PAIR)/$(AWS_EC2_KEY_NAME).pem 13 | 14 | ## delete ec2 key-pair 15 | delete-keypair: 16 | @aws --region ${AWS_REGION} ec2 delete-key-pair --key-name ${AWS_EC2_KEY_NAME} || true 17 | @-rm -rf $(DIR_KEY_PAIR)/ 18 | 19 | .PHONY: create-key-pair delete-key-pair 20 | -------------------------------------------------------------------------------- /makefiles/route53.mk: -------------------------------------------------------------------------------- 1 | test-route53: 2 | @scripts/ssh ${DIR_KEY_PAIR}/${AWS_EC2_KEY_NAME}.pem `terraform output bastion-ip` \ 3 | "( nslookup etcd.`terraform output internal-tld` )" 4 | @scripts/ssh ${DIR_KEY_PAIR}/${AWS_EC2_KEY_NAME}.pem `terraform output bastion-ip` \ 5 | "( dig `terraform output internal-tld` ANY )" 6 | @scripts/ssh ${DIR_KEY_PAIR}/${AWS_EC2_KEY_NAME}.pem `terraform output bastion-ip` \ 7 | "( dig +noall +answer SRV _etcd-server._tcp.`terraform output internal-tld` )" 8 | @scripts/ssh ${DIR_KEY_PAIR}/${AWS_EC2_KEY_NAME}.pem `terraform output bastion-ip` \ 9 | "( dig +noall +answer SRV _etcd-client._tcp.`terraform output internal-tld` )" 10 | @scripts/ssh ${DIR_KEY_PAIR}/${AWS_EC2_KEY_NAME}.pem `terraform output bastion-ip` \ 11 | "( dig +noall +answer etcd.`terraform output internal-tld` )" 12 | 13 | .PHONY: test-route53 14 | -------------------------------------------------------------------------------- /makefiles/shortcuts.mk: -------------------------------------------------------------------------------- 1 | addons: 2 | kubectl get pods --namespace=kube-system 3 | 4 | dns: 5 | kubectl exec busybox -- nslookup kubernetes 6 | 7 | nodes: 8 | kubectl get nodes 9 | 10 | pods: 11 | @kubectl get pods --all-namespaces -o wide 12 | 13 | .PHONY: addons dns nodes pods 14 | -------------------------------------------------------------------------------- /makefiles/ssl.mk: -------------------------------------------------------------------------------- 1 | test-ssl: 2 | openssl x509 -in .cfssl/k8s-admin.pem -noout -text 3 | openssl x509 -in .cfssl/k8s-apiserver.pem -noout -text 4 | openssl x509 -in .cfssl/k8s-worker.pem -noout -text 5 | openssl x509 -in .cfssl/k8s-etcd.pem -noout -text 6 | 7 | .PHONY: test-ssl 8 | -------------------------------------------------------------------------------- /makefiles/terraform.mk: -------------------------------------------------------------------------------- 1 | .terraform: ; terraform get 2 | 3 | terraform.tfvars: 4 | @./scripts/init-variables 5 | 6 | module.%: 7 | @echo "${BLUE}❤ make $@ - commencing${NC}" 8 | @time terraform apply -target $@ 9 | @echo "${GREEN}✓ make $@ - success${NC}" 10 | @sleep 5.2 11 | 12 | ## terraform apply 13 | apply: plan 14 | @echo "${BLUE}❤ terraform apply - commencing${NC}" 15 | terraform apply terraform.tfplan 16 | @echo "${GREEN}✓ make $@ - success${NC}" 17 | 18 | ## terraform destroy 19 | destroy: ; terraform destroy 20 | 21 | ## terraform get 22 | get: ; terraform get 23 | 24 | ## generate variables 25 | init: terraform.tfvars 26 | 27 | ## terraform plan 28 | plan: get init 29 | terraform init 30 | terraform validate -var-file=terraform.tfvars 31 | @echo "${GREEN}✓ terraform validate - success${NC}" 32 | terraform plan -out terraform.tfplan 33 | 34 | ## terraform show 35 | show: ; terraform show 36 | 37 | .PHONY: apply destroy get init module.% plan show 38 | -------------------------------------------------------------------------------- /modules.tf: -------------------------------------------------------------------------------- 1 | module "s3" { 2 | source = "./modules/s3" 3 | 4 | # variables 5 | aws = "${ var.aws }" 6 | bucket = "kz8s-pki-${ var.name }-${ var.aws["account-id"] }-${ var.aws["region"] }" 7 | name = "${ var.name }" 8 | } 9 | 10 | module "vpc" { 11 | source = "./modules/vpc" 12 | depends-id = "" 13 | 14 | # variables 15 | azs = "${ var.aws["azs"] }" 16 | cidr = "${ var.cidr["vpc"] }" 17 | hyperkube-tag = "${ var.k8s["hyperkube-tag"] }" 18 | name = "${ var.name }" 19 | region = "${ var.aws["region"] }" 20 | } 21 | 22 | module "route53" { 23 | source = "./modules/route53" 24 | depends-id = "${ module.vpc.depends-id }" 25 | 26 | # variables 27 | etcd-ips = "${ var.etcd-ips }" 28 | internal-tld = "${ var.internal-tld }" 29 | name = "${ var.name }" 30 | 31 | # modules 32 | vpc-id = "${ module.vpc.id }" 33 | } 34 | 35 | module "pki" { 36 | source = "./modules/pki" 37 | depends-id = "${ module.vpc.depends-id }" 38 | 39 | # variables 40 | ami-id = "${ var.coreos-aws["ami"] }" 41 | aws = "${ var.aws }" 42 | cidr-vpc = "${ var.cidr["vpc"] }" 43 | instance-type = "${ var.instance-type["pki"] }" 44 | internal-tld = "${ var.internal-tld }" 45 | ip = "${ var.pki-ip }" 46 | k8s = "${ var.k8s }" 47 | name = "${ var.name }" 48 | 49 | # modules 50 | instance-profile-name = "${ module.iam.instance-profile-name-pki }" 51 | internal-zone-id = "${ module.route53.internal-zone-id }" 52 | s3-bucket = "${ module.s3.bucket }" 53 | s3-bucket-arn = "${ module.s3.bucket-arn }" 54 | security-group-id = "${ module.security.pki-id }" 55 | subnet-id = "${ element( split(",", module.vpc.subnet-ids-private), 0 ) }" 56 | vpc-id = "${ module.vpc.id }" 57 | } 58 | 59 | module "security" { 60 | source = "./modules/security" 61 | depends-id = "${ module.vpc.depends-id }" 62 | 63 | # variables 64 | cidr-allow-ssh = "${ var.cidr["allow-ssh"] }" 65 | cidr-vpc = "${ var.cidr["vpc"] }" 66 | name = "${ var.name }" 67 | 68 | # modules 69 | vpc-id = "${ module.vpc.id }" 70 | } 71 | 72 | module "iam" { 73 | source = "./modules/iam" 74 | depends-id = "${ module.pki.depends-id }" 75 | 76 | # variables 77 | name = "${ var.name }" 78 | 79 | # modules 80 | s3-bucket-arn = "${ module.s3.bucket-arn }" 81 | } 82 | 83 | module "bastion" { 84 | source = "./modules/bastion" 85 | depends-id = "${ module.vpc.depends-id }" 86 | 87 | # variables 88 | ami-id = "${ var.coreos-aws["ami"] }" 89 | instance-type = "${ var.instance-type["bastion"] }" 90 | internal-tld = "${ var.internal-tld }" 91 | key-name = "${ var.aws["key-name"] }" 92 | name = "${ var.name }" 93 | 94 | # modules 95 | instance-profile-name = "${ module.iam.instance-profile-name-bastion }" 96 | s3-bucket = "${ module.s3.bucket }" 97 | s3-bucket-arn = "${ module.s3.bucket-arn }" 98 | security-group-id = "${ module.security.bastion-id }" 99 | subnet-id = "${ element( split(",", module.vpc.subnet-ids-public), 0 ) }" 100 | vpc-id = "${ module.vpc.id }" 101 | } 102 | 103 | module "etcd" { 104 | source = "./modules/etcd" 105 | depends-id = "${ module.route53.depends-id }" 106 | 107 | # variables 108 | ami-id = "${ var.coreos-aws["ami"] }" 109 | aws = "${ var.aws }" 110 | cluster-domain = "${ var.cluster-domain }" 111 | dns-service-ip = "${ var.dns-service-ip }" 112 | etcd-ips = "${ var.etcd-ips }" 113 | instance-type = "${ var.instance-type["etcd"] }" 114 | internal-tld = "${ var.internal-tld }" 115 | ip-k8s-service = "${ var.k8s-service-ip }" 116 | k8s = "${ var.k8s }" 117 | name = "${ var.name }" 118 | pod-ip-range = "${ var.cidr["pods"] }" 119 | service-cluster-ip-range = "${ var.cidr["service-cluster"] }" 120 | 121 | # modules 122 | etcd-security-group-id = "${ module.security.etcd-id }" 123 | external-elb-security-group-id = "${ module.security.external-elb-id }" 124 | instance-profile-name = "${ module.iam.instance-profile-name-master }" 125 | s3-bucket = "${ module.s3.bucket }" 126 | subnet-id-private = "${ element( split(",", module.vpc.subnet-ids-private), 0 ) }" 127 | subnet-id-public = "${ element( split(",", module.vpc.subnet-ids-public), 0 ) }" 128 | vpc-id = "${ module.vpc.id }" 129 | } 130 | 131 | module "worker" { 132 | source = "./modules/worker" 133 | depends-id = "${ module.route53.depends-id }" 134 | 135 | # variables 136 | ami-id = "${ var.coreos-aws["ami"] }" 137 | aws = "${ var.aws }" 138 | capacity = { 139 | desired = 3 140 | max = 5 141 | min = 1 142 | } 143 | cluster-domain = "${ var.cluster-domain }" 144 | dns-service-ip = "${ var.dns-service-ip }" 145 | instance-type = "${ var.instance-type["worker"] }" 146 | internal-tld = "${ var.internal-tld }" 147 | k8s = "${ var.k8s }" 148 | name = "${ var.name }" 149 | volume_size = { 150 | ebs = 250 151 | root = 52 152 | } 153 | worker-name = "general" 154 | 155 | # modules 156 | instance-profile-name = "${ module.iam.instance-profile-name-worker }" 157 | s3-bucket = "${ module.s3.bucket }" 158 | security-group-id = "${ module.security.worker-id }" 159 | subnet-id = "${ element( split(",", module.vpc.subnet-ids-private), 0 ) }" 160 | vpc-id = "${ module.vpc.id }" 161 | } 162 | -------------------------------------------------------------------------------- /modules/bastion/cloud-config.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "cloud-config" { 2 | template = "${ file( "${ path.module }/cloud-config.yml" )}" 3 | 4 | vars { 5 | internal-tld = "${ var.internal-tld }" 6 | s3-bucket = "${ var.s3-bucket }" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /modules/bastion/cloud-config.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | 6 | locksmith: 7 | endpoint: https://etcd.${ internal-tld }:2379 8 | etcd_cafile: /etc/kubernetes/ssl/ca.pem 9 | etcd_certfile: /etc/kubernetes/ssl/k8s-bastion.pem 10 | etcd_keyfile: /etc/kubernetes/ssl/k8s-bastion-key.pem 11 | 12 | 13 | update: 14 | reboot-strategy: etcd-lock 15 | 16 | 17 | units: 18 | 19 | - name: create-certificates.service 20 | command: start 21 | content: | 22 | [Unit] 23 | After=download-cfssl.service 24 | Requires=download-cfssl.service 25 | Before=flannel.service 26 | RequiredBy=flannel.service 27 | Description=Get ssl artifacts from s3 bucket using IAM role and create local certificates 28 | [Service] 29 | Type=oneshot 30 | RemainAfterExit=yes 31 | ExecStartPre=-/usr/bin/mkdir --parents /etc/kubernetes/ssl 32 | ExecStartPre=/opt/bin/fetch-from-s3 ca.pem 33 | ExecStart=/opt/bin/create-certificates 34 | 35 | - name: download-cfssl.service 36 | command: start 37 | content: | 38 | [Unit] 39 | After=network-online.target 40 | Requires=network-online.target 41 | Before=etcd-member.service 42 | Description=Download cfssl 43 | [Service] 44 | Type=oneshot 45 | RemainAfterExit=yes 46 | ExecStartPre=-/usr/bin/mkdir --parents /etc/kubernetes/ssl 47 | ExecStartPre=-/usr/bin/mkdir --parents /opt/bin 48 | ExecStartPre=/usr/bin/curl -L -o /opt/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 49 | ExecStartPre=/usr/bin/curl -L -o /opt/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 50 | ExecStart=/usr/bin/chmod +x /opt/bin/cfssl /opt/bin/cfssljson 51 | 52 | 53 | write-files: 54 | - path: /etc/environment 55 | permissions: 0644 56 | content: | 57 | ETCDCTL_CA_FILE=/etc/kubernetes/ssl/ca.pem 58 | ETCDCTL_CERT_FILE=/etc/kubernetes/ssl/k8s-bastion.pem 59 | ETCDCTL_ENDPOINTS=https://etcd.${ internal-tld }:2379 60 | ETCDCTL_KEY_FILE=/etc/kubernetes/ssl/k8s-bastion-key.pem 61 | 62 | - path: /opt/bin/host-rkt 63 | permissions: 0755 64 | owner: root:root 65 | content: | 66 | #!/bin/sh 67 | exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@" 68 | 69 | - path: /opt/bin/fetch-from-s3 70 | permissions: 0755 71 | owner: root:root 72 | content: | 73 | #!/bin/bash -e 74 | until /usr/bin/rkt run \ 75 | --net=host \ 76 | --trust-keys-from-https \ 77 | --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \ 78 | --volume=ssl,kind=host,source=/etc/kubernetes/ssl,readOnly=false --mount=volume=ssl,target=/etc/kubernetes/ssl \ 79 | quay.io/coreos/awscli -- aws s3 cp s3://${ s3-bucket }/$1 /etc/kubernetes/ssl 80 | do 81 | echo "retrying" 82 | sleep 5.2 83 | done 84 | echo "✓" 85 | 86 | - path: /opt/bin/create-certificates 87 | permissions: 0755 88 | owner: root:root 89 | content: | 90 | #!/bin/bash -ex 91 | 92 | OUTDIR=/etc/kubernetes/ssl 93 | 94 | function error { 95 | echo "✗ Error on line $1"'!' 96 | exit 1 97 | } 98 | trap 'error $${LINENO}' ERR 99 | 100 | until printf "." && curl -d '{"label":"primary"}' http://pki.${ internal-tld }:8888/api/v1/cfssl/info &>/dev/null 101 | do sleep 5.2; done; echo "✓" 102 | 103 | DNS1="kubernetes" 104 | DNS2="kubernetes.default" 105 | DNS3="kubernetes.default.svc" 106 | DNS4="kubernetes.default.svc.cluster.local" 107 | DEFAULT_HOSTS="$DNS1,$DNS2,$DNS3,$DNS4,127.0.0.1" 108 | 109 | function csr { 110 | cat </dev/null 451 | do sleep 5.2; done; echo "✓" 452 | 453 | DNS1="kubernetes" 454 | DNS2="kubernetes.default" 455 | DNS3="kubernetes.default.svc" 456 | DNS4="kubernetes.default.svc.cluster.local" 457 | DEFAULT_HOSTS="$DNS1,$DNS2,$DNS3,$DNS4,127.0.0.1" 458 | 459 | function csr { 460 | cat </dev/null 20 | ExecStart=/usr/bin/rkt run \ 21 | --net=host \ 22 | --trust-keys-from-https \ 23 | --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \ 24 | --volume=ssl,kind=host,source=/etc/cfssl,readOnly=true --mount=volume=ssl,target=/etc/cfssl \ 25 | quay.io/coreos/awscli -- aws s3 cp /etc/cfssl/service-account-key.pem s3://${ s3-bucket }/ 26 | 27 | - name: download-cfssl.service 28 | command: start 29 | content: | 30 | [Unit] 31 | Description=Download cfssl 32 | ConditionFileNotEmpty=!/opt/bin/cfssl 33 | ConditionFileNotEmpty=!/opt/bin/cfssljson 34 | [Service] 35 | Type=oneshot 36 | RemainAfterExit=yes 37 | ExecStartPre=/usr/bin/mkdir --parents /opt/bin 38 | ExecStartPre=/usr/bin/curl -L -o /opt/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 39 | ExecStartPre=/usr/bin/curl -L -o /opt/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 40 | ExecStart=/usr/bin/chmod +x /opt/bin/cfssl /opt/bin/cfssljson 41 | 42 | - name: generate-rootca.service 43 | command: start 44 | content: | 45 | [Unit] 46 | After=download-cfssl.service 47 | ConditionFileIsExecutable=/opt/bin/cfssl 48 | ConditionFileIsExecutable=/opt/bin/cfssljson 49 | ConditionFileNotEmpty=!/etc/cfssl/ca.pem 50 | Description=Generate rootca and save to s3 51 | [Service] 52 | Type=oneshot 53 | RemainAfterExit=yes 54 | WorkingDirectory=/etc/cfssl 55 | ExecStartPre=/usr/bin/mkdir --parents /etc/cfssl 56 | ExecStartPre=/bin/sh -c "\ 57 | /opt/bin/cfssl gencert -initca ca-csr.json \ 58 | | /opt/bin/cfssljson -bare ca -" 59 | ExecStart=/usr/bin/rkt run \ 60 | --net=host \ 61 | --trust-keys-from-https \ 62 | --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \ 63 | --volume=ssl,kind=host,source=/etc/cfssl,readOnly=true --mount=volume=ssl,target=/etc/cfssl \ 64 | quay.io/coreos/awscli -- aws s3 cp /etc/cfssl/ca.pem s3://${ s3-bucket }/ 65 | 66 | - name: cfssl.service 67 | command: start 68 | content: | 69 | [Unit] 70 | After=download-ssl.service 71 | ConditionFileIsExecutable=/opt/bin/cfssl 72 | Description=Start up cfssl service 73 | [Service] 74 | ExecStart=/opt/bin/cfssl serve \ 75 | -address 0.0.0.0 \ 76 | -ca /etc/cfssl/ca.pem \ 77 | -ca-key /etc/cfssl/ca-key.pem \ 78 | -config /etc/cfssl/ca-config.json 79 | Restart=always 80 | RestartSec=10 81 | 82 | write-files: 83 | - path: /etc/cfssl/ca-csr.json 84 | content: | 85 | { 86 | "CN": "CA", 87 | "key": { "algo": "rsa", "size": 2048 }, 88 | "names": [{ "C": "US", "L": "San Francisco", "O": "Kubernetes", "ST": "California" }] 89 | } 90 | 91 | - path: /etc/cfssl/ca-config.json 92 | content: | 93 | { 94 | "signing": { 95 | "default": { "expiry": "43800h" }, 96 | "profiles": { 97 | "server": { 98 | "expiry": "43800h", 99 | "usages": [ "signing", "key encipherment", "server auth" ] 100 | }, 101 | "client": { 102 | "expiry": "43800h", 103 | "usages": [ "signing", "key encipherment", "client auth" ] 104 | }, 105 | "client-server": { 106 | "expiry": "43800h", 107 | "usages": [ "signing", "key encipherment", "server auth", "client auth" ] 108 | } 109 | } 110 | } 111 | } 112 | 113 | - path: /etc/cfssl/s3-bucket 114 | content: ${ s3-bucket } 115 | -------------------------------------------------------------------------------- /modules/pki/ec2.tf: -------------------------------------------------------------------------------- 1 | resource "aws_instance" "pki" { 2 | 3 | ami = "${ var.ami-id }" 4 | associate_public_ip_address = false 5 | 6 | iam_instance_profile = "${ var.instance-profile-name }" 7 | instance_type = "${ var.instance-type }" 8 | key_name = "${ var.aws["key-name"] }" 9 | 10 | private_ip = "${ var.ip }" 11 | 12 | source_dest_check = true 13 | subnet_id = "${ var.subnet-id }" 14 | 15 | tags { 16 | builtWith = "terraform" 17 | kz8s = "${ var.name }" 18 | depends-id = "${ var.depends-id }" 19 | Name = "kz8s-pki" 20 | role = "pki" 21 | } 22 | 23 | user_data = "${ data.template_file.cloud-config.rendered }" 24 | 25 | vpc_security_group_ids = [ "${ var.security-group-id }" ] 26 | 27 | } 28 | -------------------------------------------------------------------------------- /modules/pki/io.tf: -------------------------------------------------------------------------------- 1 | variable "ami-id" {} 2 | variable "aws" { 3 | type = "map" 4 | } 5 | variable "cidr-vpc" {} 6 | variable "depends-id" {} 7 | variable "instance-profile-name" {} 8 | variable "instance-type" {} 9 | variable "internal-tld" {} 10 | variable "internal-zone-id" {} 11 | variable "k8s" { 12 | type = "map" 13 | } 14 | variable "name" {} 15 | variable "ip" {} 16 | variable "s3-bucket" {} 17 | variable "s3-bucket-arn" {} 18 | variable "security-group-id" {} 19 | variable "subnet-id" {} 20 | variable "vpc-id" {} 21 | 22 | output "depends-id" { value = "${ null_resource.dummy_dependency.id }" } 23 | output "ip" { value = "${ aws_instance.pki.private_ip }" } 24 | -------------------------------------------------------------------------------- /modules/pki/null-resource.tf: -------------------------------------------------------------------------------- 1 | resource "null_resource" "dummy_dependency" { 2 | 3 | depends_on = [ 4 | "aws_instance.pki", 5 | "aws_route53_record.pki", 6 | ] 7 | 8 | } 9 | -------------------------------------------------------------------------------- /modules/pki/route53.tf: -------------------------------------------------------------------------------- 1 | resource "aws_route53_record" "pki" { 2 | 3 | name = "pki" 4 | 5 | records = [ 6 | "${ aws_instance.pki.private_ip }" 7 | ] 8 | 9 | ttl = "300" 10 | type = "A" 11 | zone_id = "${ var.internal-zone-id }" 12 | 13 | } 14 | -------------------------------------------------------------------------------- /modules/route53/io.tf: -------------------------------------------------------------------------------- 1 | variable "depends-id" {} 2 | variable "etcd-ips" {} 3 | variable "internal-tld" {} 4 | variable "name" {} 5 | variable "vpc-id" {} 6 | 7 | output "depends-id" { value = "${null_resource.dummy_dependency.id}" } 8 | output "internal-name-servers" { value = "${ aws_route53_zone.internal.name_servers }" } 9 | output "internal-zone-id" { value = "${ aws_route53_zone.internal.zone_id }" } 10 | -------------------------------------------------------------------------------- /modules/route53/route53.tf: -------------------------------------------------------------------------------- 1 | resource "aws_route53_zone" "internal" { 2 | comment = "Kubernetes [tack] cluster DNS (internal)" 3 | name = "${ var.internal-tld }" 4 | tags { 5 | builtWith = "terraform" 6 | KubernetesCluster = "${ var.name }" 7 | kz8s = "${ var.name }" 8 | Name = "k8s-${ var.name }" 9 | } 10 | vpc_id = "${ var.vpc-id }" 11 | } 12 | 13 | resource "aws_route53_record" "A-etcd" { 14 | name = "etcd" 15 | records = [ "${ split(",", var.etcd-ips) }" ] 16 | ttl = "300" 17 | type = "A" 18 | zone_id = "${ aws_route53_zone.internal.zone_id }" 19 | } 20 | 21 | resource "aws_route53_record" "A-etcds" { 22 | count = "${ length( split(",", var.etcd-ips) ) }" 23 | 24 | name = "etcd${ count.index+1 }" 25 | ttl = "300" 26 | type = "A" 27 | records = [ 28 | "${ element(split(",", var.etcd-ips), count.index) }" 29 | ] 30 | zone_id = "${ aws_route53_zone.internal.zone_id }" 31 | } 32 | 33 | resource "aws_route53_record" "CNAME-master" { 34 | name = "master" 35 | records = [ "etcd.${ var.internal-tld }" ] 36 | ttl = "300" 37 | type = "CNAME" 38 | zone_id = "${ aws_route53_zone.internal.zone_id }" 39 | } 40 | 41 | resource "aws_route53_record" "etcd-client-tcp" { 42 | name = "_etcd-client._tcp" 43 | ttl = "300" 44 | type = "SRV" 45 | records = [ "${ formatlist("0 0 2379 %v", aws_route53_record.A-etcds.*.fqdn) }" ] 46 | zone_id = "${ aws_route53_zone.internal.zone_id }" 47 | } 48 | 49 | resource "aws_route53_record" "etcd-server-tcp" { 50 | name = "_etcd-server-ssl._tcp" 51 | ttl = "300" 52 | type = "SRV" 53 | records = [ "${ formatlist("0 0 2380 %v", aws_route53_record.A-etcds.*.fqdn) }" ] 54 | zone_id = "${ aws_route53_zone.internal.zone_id }" 55 | } 56 | 57 | resource "null_resource" "dummy_dependency" { 58 | depends_on = [ 59 | "aws_route53_record.etcd-server-tcp", 60 | "aws_route53_record.A-etcd", 61 | ] 62 | } 63 | -------------------------------------------------------------------------------- /modules/s3/io.tf: -------------------------------------------------------------------------------- 1 | variable "aws" { 2 | type = "map" 3 | } 4 | variable "bucket" {} 5 | variable "name" {} 6 | 7 | output "bucket" { value = "${ var.bucket }" } 8 | output "bucket-arn" { value = "${ aws_s3_bucket.pki.arn }" } 9 | -------------------------------------------------------------------------------- /modules/s3/s3.tf: -------------------------------------------------------------------------------- 1 | resource "aws_s3_bucket" "pki" { 2 | 3 | acl = "private" 4 | bucket = "${ var.bucket }" 5 | force_destroy = true 6 | 7 | region = "${ var.aws["region"] }" 8 | 9 | tags { 10 | builtWith = "terraform" 11 | KubernetesCluster = "${ var.name }" 12 | kz8s = "${ var.name }" 13 | Name = "${ var.name }" 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /modules/security/io.tf: -------------------------------------------------------------------------------- 1 | variable "cidr-allow-ssh" {} 2 | variable "cidr-vpc" {} 3 | variable "depends-id" {} 4 | variable "name" {} 5 | variable "vpc-id" {} 6 | 7 | output "bastion-id" { value = "${ aws_security_group.bastion.id }" } 8 | output "depends-id" { value = "${ null_resource.dummy_dependency.id }" } 9 | output "etcd-id" { value = "${ aws_security_group.etcd.id }" } 10 | output "external-elb-id" { value = "${ aws_security_group.external-elb.id }" } 11 | output "pki-id" { value = "${ aws_security_group.pki.id }" } 12 | output "worker-id" { value = "${ aws_security_group.worker.id }" } 13 | -------------------------------------------------------------------------------- /modules/security/null-resource.tf: -------------------------------------------------------------------------------- 1 | resource "null_resource" "dummy_dependency" { 2 | 3 | depends_on = [ 4 | "aws_security_group.bastion", 5 | "aws_security_group.etcd", 6 | "aws_security_group.external-elb", 7 | "aws_security_group.pki", 8 | "aws_security_group.worker", 9 | ] 10 | 11 | } 12 | -------------------------------------------------------------------------------- /modules/security/security.tf: -------------------------------------------------------------------------------- 1 | resource "aws_security_group" "bastion" { 2 | description = "kz8s bastion security group" 3 | 4 | egress = { 5 | from_port = 0 6 | to_port = 0 7 | protocol = "-1" 8 | cidr_blocks = [ "0.0.0.0/0" ] 9 | } 10 | 11 | ingress = { 12 | from_port = 22 13 | to_port = 22 14 | protocol = "tcp" 15 | cidr_blocks = [ "${ var.cidr-allow-ssh }" ] 16 | } 17 | 18 | name = "kz8s-bastion-${ var.name }" 19 | 20 | tags { 21 | KubernetesCluster = "${ var.name }" 22 | kz8s = "${ var.name }" 23 | Name = "kz8s-bastion-${ var.name }" 24 | builtWith = "terraform" 25 | } 26 | 27 | vpc_id = "${ var.vpc-id }" 28 | } 29 | 30 | 31 | resource "aws_security_group" "pki" { 32 | 33 | description = "k8s pki security group" 34 | 35 | egress = { 36 | from_port = 0 37 | to_port = 0 38 | protocol = "-1" 39 | cidr_blocks = [ "0.0.0.0/0" ] 40 | } 41 | 42 | ingress = { 43 | from_port = 0 44 | to_port = 0 45 | protocol = "-1" 46 | self = true 47 | cidr_blocks = [ "${ var.cidr-vpc }" ] 48 | } 49 | 50 | name = "kz8s-pki-${ var.name }" 51 | 52 | tags { 53 | KubernetesCluster = "${ var.name }" 54 | kz8s = "${ var.name }" 55 | Name = "kz8s-pki-${ var.name }" 56 | builtWith = "terraform" 57 | } 58 | 59 | vpc_id = "${ var.vpc-id }" 60 | } 61 | 62 | 63 | resource "aws_security_group" "etcd" { 64 | description = "kz8s etcd security group" 65 | 66 | egress = { 67 | from_port = 0 68 | to_port = 0 69 | protocol = "-1" 70 | /*self = true*/ 71 | cidr_blocks = [ "0.0.0.0/0" ] 72 | } 73 | 74 | ingress = { 75 | from_port = 0 76 | to_port = 0 77 | protocol = "-1" 78 | self = true 79 | cidr_blocks = [ "${ var.cidr-vpc }" ] 80 | } 81 | 82 | name = "kz8s-etcd-${ var.name }" 83 | 84 | tags { 85 | KubernetesCluster = "${ var.name }" 86 | kz8s = "${ var.name }" 87 | Name = "kz8s-etcd-${ var.name }" 88 | builtWith = "terraform" 89 | } 90 | 91 | vpc_id = "${ var.vpc-id }" 92 | } 93 | 94 | 95 | resource "aws_security_group" "external-elb" { 96 | description = "kz8s-${ var.name } master (apiserver) external elb" 97 | 98 | egress { 99 | from_port = 0 100 | to_port = 0 101 | protocol = "-1" 102 | /*cidr_blocks = [ "${ var.cidr-vpc }" ]*/ 103 | security_groups = [ "${ aws_security_group.etcd.id }" ] 104 | } 105 | 106 | ingress { 107 | from_port = -1 108 | to_port = -1 109 | protocol = "icmp" 110 | cidr_blocks = [ "0.0.0.0/0" ] 111 | } 112 | 113 | ingress { 114 | from_port = 443 115 | to_port = 443 116 | protocol = "tcp" 117 | cidr_blocks = [ "0.0.0.0/0" ] 118 | } 119 | 120 | name = "kz8s-master-external-elb-${ var.name }" 121 | 122 | tags { 123 | KubernetesCluster = "${ var.name }" 124 | kz8s = "${ var.name }" 125 | Name = "kz8s-master-external-elb-${ var.name }" 126 | builtWith = "terraform" 127 | } 128 | 129 | vpc_id = "${ var.vpc-id }" 130 | } 131 | 132 | resource "aws_security_group" "worker" { 133 | description = "kz8s worker security group" 134 | 135 | egress = { 136 | from_port = 0 137 | to_port = 0 138 | protocol = "-1" 139 | /*self = true*/ 140 | cidr_blocks = [ "0.0.0.0/0" ] 141 | } 142 | 143 | ingress = { 144 | from_port = 0 145 | to_port = 0 146 | protocol = "-1" 147 | self = true 148 | cidr_blocks = [ "${ var.cidr-vpc }" ] 149 | } 150 | 151 | name = "kz8s-worker-${ var.name }" 152 | 153 | tags { 154 | KubernetesCluster = "${ var.name }" 155 | kz8s = "${ var.name }" 156 | Name = "kz8s-worker-${ var.name }" 157 | builtWith = "terraform" 158 | } 159 | 160 | vpc_id = "${ var.vpc-id }" 161 | } 162 | -------------------------------------------------------------------------------- /modules/vpc-existing/io.tf: -------------------------------------------------------------------------------- 1 | # variables 2 | variable "azs" {} 3 | variable "cidr" {} 4 | variable "name" {} 5 | variable "region" {} 6 | variable "hyperkube-tag" {} 7 | 8 | variable "id" {} 9 | variable "gateway-id" {} 10 | variable "depends-id" {} 11 | 12 | variable "subnet-ids-public" {} 13 | variable "subnet-ids-private" {} 14 | 15 | # outputs 16 | output "depends-id" { value = "${ var.id }" } 17 | output "gateway-id" { value = "${ var.gateway-id }" } 18 | output "id" { value = "${ var.id }" } 19 | 20 | output "subnet-ids-public" { value = "${ var.subnet-ids-public }" } 21 | output "subnet-ids-private" { value = "${ var.subnet-ids-private }" } 22 | 23 | -------------------------------------------------------------------------------- /modules/vpc/io.tf: -------------------------------------------------------------------------------- 1 | variable "azs" {} 2 | variable "cidr" {} 3 | variable "hyperkube-tag" {} 4 | variable "depends-id" {} 5 | variable "name" {} 6 | variable "region" {} 7 | 8 | output "depends-id" { value = "${null_resource.dummy_dependency.id}" } 9 | output "gateway-id" { value = "${ aws_internet_gateway.main.id }" } 10 | output "id" { value = "${ aws_vpc.main.id }" } 11 | output "route-table-id" { value = "${ aws_route_table.private.id }" } 12 | output "subnet-ids-private" { value = "${ join(",", aws_subnet.private.*.id) }" } 13 | output "subnet-ids-public" { value = "${ join(",", aws_subnet.public.*.id) }" } 14 | -------------------------------------------------------------------------------- /modules/vpc/private.tf: -------------------------------------------------------------------------------- 1 | resource "aws_eip" "nat" { vpc = true } 2 | 3 | resource "aws_nat_gateway" "nat" { 4 | depends_on = [ 5 | "aws_eip.nat", 6 | "aws_internet_gateway.main", 7 | ] 8 | 9 | allocation_id = "${ aws_eip.nat.id }" 10 | subnet_id = "${ aws_subnet.public.0.id }" 11 | } 12 | 13 | resource "aws_subnet" "private" { 14 | count = "${ length( split(",", var.azs) ) }" 15 | 16 | availability_zone = "${ element( split(",", var.azs), count.index ) }" 17 | cidr_block = "${ cidrsubnet(var.cidr, 8, count.index + 10) }" 18 | vpc_id = "${ aws_vpc.main.id }" 19 | 20 | tags { 21 | "kubernetes.io/role/internal-elb" = "${ var.name }" 22 | builtWith = "terraform" 23 | KubernetesCluster = "${ var.name }" 24 | kz8s = "${ var.name }" 25 | Name = "kz8s-${ var.name }-private" 26 | visibility = "private" 27 | } 28 | } 29 | 30 | resource "aws_route_table" "private" { 31 | vpc_id = "${ aws_vpc.main.id }" 32 | 33 | route { 34 | cidr_block = "0.0.0.0/0" 35 | nat_gateway_id = "${ aws_nat_gateway.nat.id }" 36 | } 37 | 38 | tags { 39 | builtWith = "terraform" 40 | KubernetesCluster = "${ var.name }" 41 | kz8s = "${ var.name }" 42 | Name = "kz8s-${ var.name }" 43 | visibility = "private" 44 | } 45 | } 46 | 47 | resource "aws_route_table_association" "private" { 48 | count = "${ length(split(",", var.azs)) }" 49 | 50 | route_table_id = "${ aws_route_table.private.id }" 51 | subnet_id = "${ element(aws_subnet.private.*.id, count.index) }" 52 | } 53 | -------------------------------------------------------------------------------- /modules/vpc/public.tf: -------------------------------------------------------------------------------- 1 | resource "aws_internet_gateway" "main" { 2 | vpc_id = "${ aws_vpc.main.id }" 3 | 4 | tags { 5 | builtWith = "terraform" 6 | KubernetesCluster = "${ var.name }" 7 | kz8s = "${ var.name }" 8 | Name = "kz8s-${ var.name }" 9 | version = "${ var.hyperkube-tag }" 10 | } 11 | } 12 | 13 | resource "aws_subnet" "public" { 14 | count = "${ length( split(",", var.azs) ) }" 15 | 16 | availability_zone = "${ element( split(",", var.azs), count.index ) }" 17 | cidr_block = "${ cidrsubnet(var.cidr, 8, count.index) }" 18 | vpc_id = "${ aws_vpc.main.id }" 19 | 20 | tags { 21 | "kubernetes.io/role/elb" = "${ var.name }" 22 | builtWith = "terraform" 23 | KubernetesCluster = "${ var.name }" 24 | kz8s = "${ var.name }" 25 | Name = "kz8s-${ var.name }-public" 26 | version = "${ var.hyperkube-tag }" 27 | visibility = "public" 28 | } 29 | } 30 | 31 | resource "aws_route" "public" { 32 | route_table_id = "${ aws_vpc.main.main_route_table_id }" 33 | destination_cidr_block = "0.0.0.0/0" 34 | gateway_id = "${ aws_internet_gateway.main.id }" 35 | } 36 | 37 | resource "aws_route_table_association" "public" { 38 | count = "${ length(split(",", var.azs)) }" 39 | 40 | route_table_id = "${ aws_vpc.main.main_route_table_id }" 41 | subnet_id = "${ element(aws_subnet.public.*.id, count.index) }" 42 | } 43 | -------------------------------------------------------------------------------- /modules/vpc/vpc.tf: -------------------------------------------------------------------------------- 1 | resource "aws_vpc" "main" { 2 | cidr_block = "${ var.cidr }" 3 | 4 | enable_dns_hostnames = true 5 | enable_dns_support = true 6 | 7 | tags { 8 | builtWith = "terraform" 9 | KubernetesCluster = "${ var.name }" 10 | kz8s = "${ var.name }" 11 | Name = "kz8s-${ var.name }" 12 | version = "${ var.hyperkube-tag }" 13 | visibility = "private,public" 14 | } 15 | } 16 | 17 | resource "null_resource" "dummy_dependency" { 18 | depends_on = [ 19 | "aws_vpc.main", 20 | "aws_nat_gateway.nat" 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /modules/worker/cloud-config.tf: -------------------------------------------------------------------------------- 1 | data "template_file" "cloud-config" { 2 | template = "${ file( "${ path.module }/cloud-config.yml" )}" 3 | 4 | vars { 5 | cluster-domain = "${ var.cluster-domain }" 6 | dns-service-ip = "${ var.dns-service-ip }" 7 | hyperkube-image = "${ var.k8s["hyperkube-image"] }" 8 | hyperkube-tag = "${ var.k8s["hyperkube-tag"] }" 9 | internal-tld = "${ var.internal-tld }" 10 | s3-bucket = "${ var.s3-bucket }" 11 | region = "${ var.aws["region"] }" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /modules/worker/cloud-config.yml: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | --- 4 | coreos: 5 | 6 | locksmith: 7 | endpoint: https://etcd.${ internal-tld }:2379 8 | etcd_cafile: /etc/kubernetes/ssl/ca.pem 9 | etcd_certfile: /etc/kubernetes/ssl/k8s-etcd.pem 10 | etcd_keyfile: /etc/kubernetes/ssl/k8s-etcd-key.pem 11 | 12 | flannel: 13 | etcd_cafile: /etc/kubernetes/ssl/ca.pem 14 | etcd_certfile: /etc/kubernetes/ssl/k8s-worker.pem 15 | etcd_endpoints: https://etcd.${ internal-tld }:2379 16 | etcd_keyfile: /etc/kubernetes/ssl/k8s-worker-key.pem 17 | 18 | units: 19 | - name: etcd-member.service 20 | command: start 21 | drop-ins: 22 | - name: 01-wait-for-certs.conf 23 | content: | 24 | [Unit] 25 | After=create-certificates.service 26 | Requires=create-certificates.service 27 | ConditionFileNotEmpty=/etc/kubernetes/ssl/ca.pem 28 | ConditionFileNotEmpty=/etc/kubernetes/ssl/k8s-worker.pem 29 | ConditionFileNotEmpty=/etc/kubernetes/ssl/k8s-worker-key.pem 30 | 31 | - name: 10-environment.conf 32 | content: | 33 | [Service] 34 | Environment="ETCD_SSL_DIR=/etc/kubernetes/ssl" 35 | Environment="ETCD_CERT_FILE=/etc/ssl/certs/k8s-worker.pem" 36 | Environment="ETCD_CLIENT_CERT_AUTH=TRUE" 37 | Environment="ETCD_DISCOVERY_SRV=${ internal-tld }" 38 | Environment="ETCD_KEY_FILE=/etc/ssl/certs/k8s-worker-key.pem" 39 | Environment="ETCD_PEER_CERT_FILE=/etc/ssl/certs/k8s-worker.pem" 40 | Environment="ETCD_PEER_CLIENT_AUTH=true" 41 | Environment="ETCD_PEER_KEY_FILE=/etc/ssl/certs/k8s-worker-key.pem" 42 | Environment="ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/certs/ca.pem" 43 | Environment="ETCD_PROXY=on" 44 | Environment="ETCD_TRUSTED_CA_FILE=/etc/ssl/certs/ca.pem" 45 | 46 | - name: format-ephemeral.service 47 | command: start 48 | content: | 49 | [Unit] 50 | Description=Formats the ephemeral drive 51 | After=dev-xvdf.device 52 | Requires=dev-xvdf.device 53 | [Service] 54 | ExecStart=/usr/sbin/wipefs -f /dev/xvdf 55 | ExecStart=/usr/sbin/mkfs.ext4 -F /dev/xvdf 56 | RemainAfterExit=yes 57 | Type=oneshot 58 | 59 | - name: var-lib-docker.mount 60 | command: start 61 | content: | 62 | [Unit] 63 | Description=Mount ephemeral to /var/lib/docker 64 | Requires=format-ephemeral.service 65 | After=format-ephemeral.service 66 | Before=docker.service 67 | [Mount] 68 | What=/dev/xvdf 69 | Where=/var/lib/docker 70 | Type=ext4 71 | 72 | - name: download-cfssl.service 73 | command: start 74 | content: | 75 | [Unit] 76 | After=network-online.target 77 | Requires=network-online.target 78 | Before=etcd-member.service 79 | Description=Download cfssl 80 | [Service] 81 | Type=oneshot 82 | RemainAfterExit=yes 83 | ExecStartPre=-/usr/bin/mkdir --parents /etc/kubernetes/ssl 84 | ExecStartPre=-/usr/bin/mkdir --parents /opt/bin 85 | ExecStartPre=/usr/bin/curl -L -o /opt/bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 86 | ExecStartPre=/usr/bin/curl -L -o /opt/bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 87 | ExecStart=/usr/bin/chmod +x /opt/bin/cfssl /opt/bin/cfssljson 88 | 89 | - name: create-certificates.service 90 | command: start 91 | content: | 92 | [Unit] 93 | After=download-cfssl.service 94 | Requires=download-cfssl.service 95 | Before=flannel.service 96 | Description=Get ssl artifacts from s3 bucket using IAM role and create local certificates 97 | [Service] 98 | Type=oneshot 99 | RemainAfterExit=yes 100 | ExecStartPre=-/usr/bin/mkdir --parents /etc/kubernetes/ssl 101 | ExecStartPre=/opt/bin/fetch-from-s3 ca.pem 102 | ExecStart=/opt/bin/create-certificates 103 | 104 | - name: flanneld.service 105 | command: start 106 | drop-ins: 107 | - name: 50-network-config.conf 108 | content: | 109 | [Service] 110 | EnvironmentFile=-/etc/environment 111 | Environment="ETCD_SSL_DIR=/etc/kubernetes/ssl" 112 | Restart=always 113 | RestartSec=10 114 | 115 | - name: docker.service 116 | command: start 117 | drop-ins: 118 | - name: 40-flannel.conf 119 | content: | 120 | [Unit] 121 | After=flanneld.service 122 | Requires=flanneld.service 123 | [Service] 124 | Restart=always 125 | RestartSec=10 126 | 127 | - name: prefetch-rkt-hyperkube.service 128 | command: start 129 | content: | 130 | [Unit] 131 | After=network-online.target 132 | Requires=network-online.target 133 | Description=Prefetch rkt Hyperkube 134 | [Service] 135 | Type=oneshot 136 | RemainAfterExit=yes 137 | ExecStartPre=/usr/bin/rkt trust --trust-keys-from-https --prefix=quay.io/coreos/hyperkube 138 | ExecStart=/usr/bin/rkt fetch ${ hyperkube-image }:${ hyperkube-tag } 139 | 140 | - name: prefetch-docker-hyperkube.service 141 | command: start 142 | content: | 143 | [Unit] 144 | After=docker.service 145 | Requires=docker.service 146 | Description=Prefetch docker Hyperkube 147 | [Service] 148 | Type=oneshot 149 | RemainAfterExit=yes 150 | ExecStart=/usr/bin/docker pull ${ hyperkube-image }:${ hyperkube-tag } 151 | 152 | - name: kubelet.service 153 | command: start 154 | content: | 155 | [Unit] 156 | ConditionFileIsExecutable=/usr/lib/coreos/kubelet-wrapper 157 | ConditionFileNotEmpty=/etc/kubernetes/ssl/k8s-worker.pem 158 | ConditionFileNotEmpty=/etc/kubernetes/ssl/k8s-worker-key.pem 159 | After=flanneld.service 160 | After=prefetch-rkt-hyperkube.service 161 | After=prefetch-docker-hyperkube.service 162 | Requires=flanneld.service 163 | [Service] 164 | EnvironmentFile=/etc/environment 165 | Environment="KUBELET_ACI=${ hyperkube-image }" 166 | Environment="KUBELET_VERSION=${ hyperkube-tag }" 167 | Environment="RKT_OPTS=\ 168 | --volume dns,kind=host,source=/etc/resolv.conf \ 169 | --mount volume=dns,target=/etc/resolv.conf \ 170 | --volume rkt,kind=host,source=/opt/bin/host-rkt \ 171 | --mount volume=rkt,target=/usr/bin/rkt \ 172 | --volume var-lib-rkt,kind=host,source=/var/lib/rkt \ 173 | --mount volume=var-lib-rkt,target=/var/lib/rkt \ 174 | --volume stage,kind=host,source=/tmp \ 175 | --mount volume=stage,target=/tmp \ 176 | --volume var-log,kind=host,source=/var/log \ 177 | --mount volume=var-log,target=/var/log" 178 | ExecStartPre=/usr/bin/mkdir -p /var/log/containers 179 | ExecStartPre=/usr/bin/mkdir -p /var/lib/kubelet 180 | ExecStartPre=/usr/bin/mount --bind /var/lib/kubelet /var/lib/kubelet 181 | ExecStartPre=/usr/bin/mount --make-shared /var/lib/kubelet 182 | ExecStartPre=/usr/bin/systemctl is-active flanneld.service 183 | ExecStartPre=/opt/bin/wait-for-apiserver 184 | ExecStart=/usr/lib/coreos/kubelet-wrapper \ 185 | --allow-privileged=true \ 186 | --api-servers=https://master.${ internal-tld } \ 187 | --cert-dir=/etc/kubernetes/ssl \ 188 | --cloud-provider=aws \ 189 | --cluster-dns=${ dns-service-ip } \ 190 | --cluster-domain=${ cluster-domain } \ 191 | --kubeconfig=/etc/kubernetes/kubeconfig.yml \ 192 | --node-labels node-role.kubernetes.io/node \ 193 | --pod-manifest-path=/etc/kubernetes/manifests \ 194 | --register-node=true \ 195 | --tls-cert-file=/etc/kubernetes/ssl/k8s-worker.pem \ 196 | --tls-private-key-file=/etc/kubernetes/ssl/k8s-worker-key.pem 197 | Restart=always 198 | RestartSec=14 199 | [Install] 200 | WantedBy=multi-user.target 201 | 202 | update: 203 | reboot-strategy: etcd-lock 204 | 205 | write-files: 206 | - path: /etc/environment 207 | permissions: 0644 208 | content: | 209 | COREOS_PRIVATE_IPV4=$private_ipv4 210 | ETCD_CA_FILE=/etc/kubernetes/ssl/ca.pem 211 | ETCD_CERT_FILE=/etc/kubernetes/ssl/k8s-worker.pem 212 | ETCD_KEY_FILE=/etc/kubernetes/ssl/k8s-worker-key.pem 213 | ETCDCTL_CA_FILE=/etc/kubernetes/ssl/ca.pem 214 | ETCDCTL_CERT_FILE=/etc/kubernetes/ssl/k8s-worker.pem 215 | ETCDCTL_KEY_FILE=/etc/kubernetes/ssl/k8s-worker-key.pem 216 | 217 | - path: /opt/bin/host-rkt 218 | permissions: 0755 219 | owner: root:root 220 | content: | 221 | #!/bin/sh 222 | exec nsenter -m -u -i -n -p -t 1 -- /usr/bin/rkt "$@" 223 | 224 | - path: /etc/kubernetes/kubeconfig.yml 225 | content: | 226 | apiVersion: v1 227 | kind: Config 228 | clusters: 229 | - name: local 230 | cluster: 231 | certificate-authority: /etc/kubernetes/ssl/ca.pem 232 | server: https://master.${ internal-tld } 233 | users: 234 | - name: kubelet 235 | user: 236 | client-certificate: /etc/kubernetes/ssl/k8s-worker.pem 237 | client-key: /etc/kubernetes/ssl/k8s-worker-key.pem 238 | contexts: 239 | - context: 240 | cluster: local 241 | user: kubelet 242 | name: kubelet-context 243 | current-context: kubelet-context 244 | 245 | - path: /etc/kubernetes/manifests/kube-proxy.yml 246 | content: | 247 | apiVersion: v1 248 | kind: Pod 249 | metadata: 250 | name: kube-proxy 251 | namespace: kube-system 252 | spec: 253 | hostNetwork: true 254 | containers: 255 | - name: kube-proxy 256 | image: ${ hyperkube-image }:${ hyperkube-tag } 257 | command: 258 | - /hyperkube 259 | - proxy 260 | - --kubeconfig=/etc/kubernetes/kubeconfig.yml 261 | - --master=https://master.${ internal-tld } 262 | securityContext: 263 | privileged: true 264 | volumeMounts: 265 | - mountPath: /etc/ssl/certs 266 | name: "ssl-certs" 267 | - mountPath: /etc/kubernetes/kubeconfig.yml 268 | name: "kubeconfig" 269 | readOnly: true 270 | - mountPath: /etc/kubernetes/ssl 271 | name: "etc-kube-ssl" 272 | readOnly: true 273 | - mountPath: /var/run/dbus 274 | name: dbus 275 | readOnly: false 276 | volumes: 277 | - name: "ssl-certs" 278 | hostPath: 279 | path: "/usr/share/ca-certificates" 280 | - name: "kubeconfig" 281 | hostPath: 282 | path: "/etc/kubernetes/kubeconfig.yml" 283 | - name: "etc-kube-ssl" 284 | hostPath: 285 | path: "/etc/kubernetes/ssl" 286 | - name: dbus 287 | hostPath: 288 | path: "/var/run/dbus" 289 | 290 | 291 | - path: /etc/logrotate.d/docker-containers 292 | content: | 293 | /var/lib/docker/containers/*/*.log { 294 | rotate 7 295 | daily 296 | compress 297 | size=1M 298 | missingok 299 | delaycompress 300 | copytruncate 301 | } 302 | 303 | - path: /opt/bin/fetch-from-s3 304 | permissions: 0755 305 | owner: root:root 306 | content: | 307 | #!/bin/bash -e 308 | until /usr/bin/rkt run \ 309 | --net=host \ 310 | --trust-keys-from-https \ 311 | --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \ 312 | --volume=ssl,kind=host,source=/etc/kubernetes/ssl,readOnly=false --mount=volume=ssl,target=/etc/kubernetes/ssl \ 313 | quay.io/coreos/awscli -- aws s3 cp s3://${ s3-bucket }/$1 /etc/kubernetes/ssl 314 | do 315 | echo "retrying" 316 | sleep 5.2 317 | done 318 | echo "✓" 319 | 320 | - path: /opt/bin/wait-for-apiserver 321 | permissions: 0755 322 | owner: root:root 323 | content: | 324 | #!/bin/bash -e 325 | until curl --insecure https://master.${ internal-tld }/ &>/dev/null 326 | do 327 | echo "waiting for apiserver..." 328 | sleep 5.2 329 | done 330 | echo "✓" 331 | 332 | 333 | - path: /opt/bin/create-certificates 334 | permissions: 0755 335 | owner: root:root 336 | content: | 337 | #!/bin/bash -ex 338 | 339 | OUTDIR=/etc/kubernetes/ssl 340 | 341 | function error { 342 | echo "✗ Error on line $1"'!' 343 | exit 1 344 | } 345 | trap 'error $${LINENO}' ERR 346 | 347 | until printf "." && curl -d '{"label":"primary"}' http://pki.${ internal-tld }:8888/api/v1/cfssl/info &>/dev/null 348 | do sleep 5.2; done; echo "✓" 349 | 350 | 351 | DNS1="kubernetes" 352 | DNS2="kubernetes.default" 353 | DNS3="kubernetes.default.svc" 354 | DNS4="kubernetes.default.svc.cluster.local" 355 | DEFAULT_HOSTS="$DNS1,$DNS2,$DNS3,$DNS4,127.0.0.1" 356 | 357 | function csr { 358 | cat </dev/null 30 | do sleep 2.5; done; echo "✓" 31 | 32 | function csr { 33 | cat < ${DIR_KUBECONFIG}/kubeconfig 18 | kubectl config set-cluster cluster-${CLUSTER_NAME} \ 19 | --embed-certs=true \ 20 | --server=https://${MASTER_ELB_URL} \ 21 | --certificate-authority=${CA_PATH} 22 | 23 | kubectl config set-credentials admin-${CLUSTER_NAME} \ 24 | --embed-certs=true \ 25 | --certificate-authority=${CA_PATH} \ 26 | --client-key=${ADMIN_KEY_PATH} \ 27 | --client-certificate=${ADMIN_CERT_PATH} 28 | 29 | kubectl config set-context ${CLUSTER_NAME} \ 30 | --cluster=cluster-${CLUSTER_NAME} \ 31 | --user=admin-${CLUSTER_NAME} 32 | 33 | kubectl config use-context ${CLUSTER_NAME} 34 | EOF 35 | 36 | kubectl config set-cluster cluster-${CLUSTER_NAME} \ 37 | --embed-certs=true \ 38 | --server=https://${MASTER_ELB_URL} \ 39 | --certificate-authority=${CA_PATH} 40 | 41 | kubectl config set-credentials admin-${CLUSTER_NAME} \ 42 | --embed-certs=true \ 43 | --certificate-authority=${CA_PATH} \ 44 | --client-key=${ADMIN_KEY_PATH} \ 45 | --client-certificate=${ADMIN_CERT_PATH} 46 | 47 | kubectl config set-context ${CLUSTER_NAME} \ 48 | --cluster=cluster-${CLUSTER_NAME} \ 49 | --user=admin-${CLUSTER_NAME} 50 | 51 | kubectl config use-context ${CLUSTER_NAME} 52 | -------------------------------------------------------------------------------- /scripts/dashboard: -------------------------------------------------------------------------------- 1 | #!/bin/bash -u 2 | 3 | case "${OSTYPE}" in 4 | linux*) 5 | OPEN="xdg-open" 6 | ;; 7 | darwin*) 8 | OPEN="open" 9 | ;; 10 | esac 11 | 12 | command -v ${OPEN} &> /dev/null 13 | [ $? -ne 0 ] && ( echo "cannot determine 'open' command" ; exit 1 ) 14 | 15 | kubectl cluster-info &> /dev/null 16 | [ $? -ne 0 ] && ( echo "cluster is not healthy" ; exit 1 ) 17 | 18 | kubectl proxy & 19 | echo "✓ kubectl proxy is listening" 20 | 21 | ${OPEN} http://localhost:8001/ui 22 | sleep 1 && echo " " 23 | -------------------------------------------------------------------------------- /scripts/delete-addons: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | CDIR=$(cd `dirname "$0"` && pwd) 4 | source $CDIR/utils 5 | 6 | NAMESPACE=${NAMESPACE:-kube-system} 7 | KUBECTL="kubectl --namespace=\"${NAMESPACE}\"" 8 | 9 | OBJECTS=" 10 | storageclass/default 11 | statefulset/elasticsearch-logging 12 | service/heapster 13 | service/kube-dns 14 | deploy/heapster-v1.4.0 15 | deploy/kube-dns 16 | deploy/kube-dns-autoscaler 17 | deploy/kibana-logging 18 | rc/elasticsearch-logging-v1 19 | daemonset/fluentd 20 | deployment/cluster-autoscaler 21 | service/kubernetes-dashboard 22 | deployment/kubernetes-dashboard 23 | deployment/kube-rescheduler 24 | " 25 | 26 | for object in ${OBJECTS}; do 27 | eval "${KUBECTL} delete --ignore-not-found --now \"${object}\"" 28 | done 29 | 30 | # delete elastsearch persisten volume claims 31 | kubectl delete pvc -l k8s-app=elasticsearch-logging --namespace=kube-system -------------------------------------------------------------------------------- /scripts/do-task: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -lt 2 ]; then 4 | echo not enough args 5 | exit 1 6 | fi 7 | 8 | red='\e[1;31m%s\e[0m\n' 9 | green='\e[1;32m%s\e[0m\n' 10 | yellow='\e[1;33m%s\e[0m\n' 11 | blue='\e[1;34m%s\e[0m\n' 12 | magenta='\e[1;35m%s\e[0m\n' 13 | cyan='\e[1;36m%s\e[0m\n' 14 | 15 | 16 | TEXT=$1 17 | shift 18 | 19 | 20 | printf "$blue" "❤ $TEXT" 21 | 22 | "$@" 23 | EXIT_CODE=$? 24 | 25 | if [ $EXIT_CODE -eq 0 ] 26 | then 27 | printf "$green" "✓ $TEXT - SUCCESS" 28 | else 29 | printf "$red" "✗ $TEXT - FAIL" 30 | fi 31 | 32 | echo 33 | exit $EXIT_CODE 34 | -------------------------------------------------------------------------------- /scripts/get-ca: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | source ${0%/*}/retry 4 | 5 | echo $DIR_SSL 6 | PKI_S3_BUCKET=`terraform output s3-bucket` 7 | CA_PATH="s3://$PKI_S3_BUCKET/ca.pem" 8 | 9 | mkdir -p $DIR_SSL 10 | 11 | _retry "❤ Grabbing $CA_PATH" aws s3 cp $CA_PATH $DIR_SSL 12 | -------------------------------------------------------------------------------- /scripts/init-variables: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | CDIR=$(dirname "${BASH_SOURCE}") 4 | 5 | set -x 6 | 7 | echo $AWS_REGION 8 | echo $COREOS_CHANNEL 9 | echo $COREOS_VM_TYPE 10 | echo $AWS_EC2_KEY_NAME 11 | echo $INTERNAL_TLD 12 | echo $CLUSTER_NAME 13 | CIDR_ALLOW_SSH=`$CDIR/myip` 14 | echo $CIDR_VPC 15 | echo $CIDR_PODS 16 | echo $CIDR_SERVICE_CLUSTER 17 | echo $K8S_SERVICE_IP 18 | echo $K8S_DNS_IP 19 | echo $ETCD_IPS 20 | echo $HYPERKUBE_IMAGE 21 | echo $HYPERKUBE_TAG 22 | echo $PKI_IP 23 | 24 | set +x 25 | 26 | COREOS_AMI_ID=`curl -s \ 27 | $(printf "http://%s.release.core-os.net/amd64-usr/current/coreos_production_ami_%s_%s.txt" \ 28 | $COREOS_CHANNEL $COREOS_VM_TYPE $AWS_REGION)` 29 | 30 | AWS_ACCOUNT_ID=`aws iam get-user --output json \ 31 | | awk '/arn:aws:/{print $2}' \ 32 | | grep -Eo '[[:digit:]]{12}'` 33 | 34 | AWS_REGION_AZS=`aws ec2 describe-availability-zones --region ${AWS_REGION} --output json \ 35 | | jq --raw-output '.AvailabilityZones | map(.ZoneName) | .[]' \ 36 | | xargs \ 37 | | sed -e 's/ /,/g'` 38 | 39 | 40 | cat < terraform.tfvars 41 | # Generated by scripts/init-variables.sh 42 | aws = { 43 | account-id = "${AWS_ACCOUNT_ID}" 44 | azs = "${AWS_REGION_AZS}" 45 | key-name = "${AWS_EC2_KEY_NAME}" 46 | region = "${AWS_REGION}" 47 | } 48 | cidr = { 49 | allow-ssh = "${CIDR_ALLOW_SSH}" 50 | pods = "${CIDR_PODS}" 51 | service-cluster = "${CIDR_SERVICE_CLUSTER}" 52 | vpc = "${CIDR_VPC}" 53 | } 54 | coreos-aws = { 55 | ami = "${COREOS_AMI_ID}" 56 | channel = "${COREOS_CHANNEL}" 57 | type = "${COREOS_VM_TYPE}" 58 | } 59 | k8s = { 60 | hyperkube-image = "${HYPERKUBE_IMAGE}" 61 | hyperkube-tag = "${HYPERKUBE_TAG}" 62 | } 63 | dns-service-ip = "${K8S_DNS_IP}" 64 | internal-tld = "${INTERNAL_TLD}" 65 | k8s-service-ip = "${K8S_SERVICE_IP}" 66 | name = "${CLUSTER_NAME}" 67 | pki-ip = "${PKI_IP}" 68 | s3-bucket = "${AWS_ACCOUNT_ID}-${CLUSTER_NAME}-${AWS_REGION}" 69 | etcd-ips = "$ETCD_IPS" 70 | EOF 71 | 72 | if [ -f ./vpc-existing.tfvars ]; then cat ./vpc-existing.tfvars >>terraform.tfvars ; fi 73 | cat terraform.tfvars 74 | -------------------------------------------------------------------------------- /scripts/instances: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | echo "cluster: $CLUSTER_NAME, region: $AWS_REGION" 4 | 5 | KZ8S="Name=tag:kz8s,Values=$CLUSTER_NAME" 6 | 7 | ( \ 8 | printf "instance-id state name role kz8s private-ip public-ip\n" ;\ 9 | aws ec2 describe-instances \ 10 | --output text \ 11 | --filters "Name=instance-state-name,Values=running" $KZ8S \ 12 | --query 'Reservations[*].Instances[*].[InstanceId,State.Name,Tags[?Key==`Name`].Value|[0],Tags[?Key==`role`].Value|[0],Tags[?Key==`kz8s`].Value|[0],PrivateIpAddress,PublicIpAddress]' \ 13 | --region $AWS_REGION \ 14 | ) | column -t 15 | -------------------------------------------------------------------------------- /scripts/myip: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | IP=$(curl --silent http://whatismyip.akamai.com/) 4 | 5 | if [[ $IP =~ .*:.* ]] 6 | then 7 | echo "0.0.0.0/0" 8 | exit 9 | fi 10 | 11 | echo $IP/32 12 | -------------------------------------------------------------------------------- /scripts/prereqs: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | trap 'echo -e "\n# $BASH_COMMAND"' DEBUG 4 | 5 | aws --version 6 | 7 | cfssl version 8 | 9 | jq --version 10 | 11 | kubectl version --client 12 | 13 | terraform --version 14 | -------------------------------------------------------------------------------- /scripts/retry: -------------------------------------------------------------------------------- 1 | _retry() { 2 | [ -z "${2}" ] && return 1 3 | echo -n ${1} 4 | until printf "." && "${@:2}" &>/dev/null; do sleep 5.2; done; echo "✓" 5 | } 6 | 7 | 8 | # _retry "❤ Trying to connect to cluster with kubectl" kubectl cluster-info 9 | -------------------------------------------------------------------------------- /scripts/ssh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | BASTION_IP=`terraform output bastion-ip` 4 | 5 | COMMAND=$1 6 | 7 | function finish { 8 | [ -z ${SSH_AGENT_PID} ] || kill ${SSH_AGENT_PID} 9 | } 10 | 11 | eval `ssh-agent -s` 12 | trap finish EXIT 13 | 14 | ssh-add ${AWS_EC2_KEY_PATH} 15 | ssh -tA core@${BASTION_IP} ${COMMAND} 16 | -------------------------------------------------------------------------------- /scripts/status: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | kubectl get nodes 4 | kubectl cluster-info 5 | kubectl get pods --all-namespaces 6 | kubectl exec busybox -- nslookup kubernetes 7 | -------------------------------------------------------------------------------- /scripts/test-etcd: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | BASTION_IP=$(terraform output bastion-ip) 4 | 5 | INTERNAL_TLD=$(terraform output internal-tld) 6 | 7 | ETCD_URL="https://etcd.${INTERNAL_TLD}:2379" 8 | 9 | CERTS="--cacert /etc/kubernetes/ssl/ca.pem --cert /etc/kubernetes/ssl/k8s-etcd.pem --key /etc/kubernetes/ssl/k8s-etcd-key.pem" 10 | 11 | KEY_FILE=$(find .keypair/ -name '*.pem' -print -quit) 12 | 13 | SSH_CMD="ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} -p 8822 core@localhost" 14 | 15 | # setup port forwarding 16 | # 17 | ssh-keyscan ${BASTION_IP} 18 | ssh -o StrictHostKeyChecking=no -i ${KEY_FILE} -nNT -L 8822:etcd1.${INTERNAL_TLD}:22 core@${BASTION_IP} & 19 | SSH_PID=$! 20 | 21 | function finish { 22 | [ -z ${SSH_PID} ] || kill ${SSH_PID} 23 | } 24 | trap finish EXIT 25 | 26 | sleep 1 27 | 28 | ${SSH_CMD} -- curl ${CERTS} -s ${ETCD_URL}/version | jq 29 | ${SSH_CMD} -- curl ${CERTS} -s ${ETCD_URL}/health | jq 30 | ${SSH_CMD} -- curl ${CERTS} -s ${ETCD_URL}/v2/members | jq 31 | -------------------------------------------------------------------------------- /scripts/tmux: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TACK_SESSION=${TACK_SESSION:-tack} 4 | 5 | echo $TACK_SESSION 6 | 7 | tmux has-session -t ${TACK_SESSION} 8 | if [ $? -ne 0 ] 9 | then 10 | tmux new-session -s ${TACK_SESSION} -d 11 | 12 | tmux split-window -d -t 0 "watch make instances" 13 | tmux split-window -d -t 0 14 | tmux split-window -d -t 0 "watch make pods" 15 | 16 | tmux select-layout tiled 17 | fi 18 | 19 | tmux attach -t ${TACK_SESSION} 20 | -------------------------------------------------------------------------------- /scripts/utils: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | TACK_ROOT=$(dirname "${BASH_SOURCE}")/.. 8 | 9 | print_red() { 10 | printf '%b' "\033[91m$1\033[0m\n" 11 | } 12 | 13 | print_green() { 14 | printf '%b' "\033[92m$1\033[0m\n" 15 | } 16 | 17 | render_template() { 18 | eval "echo \"$(cat "$1")\"" 19 | } 20 | -------------------------------------------------------------------------------- /scripts/wait-for-cluster: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | ELB=$(terraform output external-elb) 4 | 5 | _retry() { 6 | [ -z "${2}" ] && return 1 7 | echo -n ${1} 8 | until printf "." && "${@:2}" &>/dev/null; do sleep 5.2; done; echo "✓" 9 | } 10 | 11 | 12 | echo "❤ Polling for cluster life - this could take a minute or more" 13 | _retry "❤ Waiting for DNS to resolve for ${ELB}" ping -c1 "${ELB}" 14 | _retry "❤ Curling apiserver external elb" curl --insecure --silent "https://${ELB}" 15 | _retry "❤ Trying to connect to cluster with kubectl" kubectl cluster-info 16 | _retry "❤ Waiting for kube-system namespace" kubectl get namespace kube-system 17 | 18 | 19 | set -x 20 | kubectl cluster-info 21 | -------------------------------------------------------------------------------- /scripts/watch-pods-until: -------------------------------------------------------------------------------- 1 | #!/bin/bash -u 2 | 3 | WATCH=watch 4 | 5 | command -v ${WATCH} &> /dev/null 6 | if [ $? -ne 0 ]; then 7 | printf "not going to 'watch' pods - cannot find 'watch' program, try: \n\n$ brew install watch\n\n" 8 | printf "here is the pod status as of now:\n\n" 9 | kubectl get po --all-namespaces 10 | exit 0 11 | fi 12 | 13 | ${WATCH} -n 1 kubectl get pods --all-namespaces & 14 | WATCH_PID=$! 15 | 16 | function finish { 17 | [ -z ${WATCH_PID} ] || kill ${WATCH_PID} 18 | } 19 | trap finish EXIT 20 | 21 | until [ `kubectl get po --all-namespaces | grep -v Running | wc -l` -eq 1 ] 22 | do sleep 2 23 | done 24 | 25 | sleep 2 26 | -------------------------------------------------------------------------------- /test/pods/busybox.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: busybox 5 | namespace: default 6 | spec: 7 | containers: 8 | - image: busybox 9 | command: 10 | - sleep 11 | - "3600" 12 | imagePullPolicy: IfNotPresent 13 | name: busybox 14 | restartPolicy: Always 15 | -------------------------------------------------------------------------------- /test/redis/redis-service.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: redis 6 | spec: 7 | ports: 8 | - port: 6379 9 | targetPort: 6379 10 | selector: 11 | application: redis 12 | -------------------------------------------------------------------------------- /test/redis/redist-deployment.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1beta1 3 | kind: StatefulSet 4 | metadata: 5 | name: redis 6 | spec: 7 | serviceName: redis 8 | replicas: 2 9 | template: 10 | metadata: 11 | labels: 12 | application: redis 13 | version: 3.2.5 14 | spec: 15 | containers: 16 | - name: redis 17 | image: redis:3.2.5 18 | volumeMounts: 19 | - mountPath: /data 20 | name: redis-data 21 | volumeClaimTemplates: 22 | - metadata: 23 | name: redis-data 24 | annotations: 25 | volume.alpha.kubernetes.io/storage-class: default 26 | spec: 27 | accessModes: 28 | - ReadWriteOnce 29 | resources: 30 | requests: 31 | storage: 7Gi 32 | -------------------------------------------------------------------------------- /vpc-existing.tfvars: -------------------------------------------------------------------------------- 1 | # # This is merged in with terraform.tfvars for override/existing VPC purposes. Only to be used in conjunction with modules_override.tf 2 | 3 | # # The existing VPC CIDR range, ensure that the the etcd, controller and worker IPs are in this range 4 | # cidr.vpc = "10.0.0.0/16" 5 | 6 | # # etcd server static IPs, ensure that they fall within the exisiting VPC public subnet range 7 | # etcd-ips = "10.0.0.10,10.0.0.11,10.0.0.12" 8 | 9 | # # Put your existing VPC info here: 10 | # vpc-existing { 11 | # id = "vpc-" 12 | # gateway-id = "igw-" 13 | # subnet-ids-public = "subnet-,subnet-" 14 | # subnet-ids-private = "subnet-,subnet-" 15 | # } --------------------------------------------------------------------------------