├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── cmd ├── terraform-provider-kubeadm │ └── main.go └── terraform-provisioner-kubeadm │ └── main.go ├── docs ├── Additional_tasks.md ├── FAQ.md ├── Home.md ├── Installation.md ├── Provisioner_kubeadm.md ├── Resource_kubeadm.md ├── Roadmap.md ├── _Sidebar.md └── examples │ ├── README.md │ ├── aws │ ├── Makefile │ ├── README.md │ ├── cloud-init │ │ └── cloud-init.yaml.tpl │ ├── cluster.tf │ ├── test.yaml │ └── variables.tf │ ├── dnd │ ├── Makefile │ ├── README.md │ ├── ci.tfvars │ ├── cluster.tf │ ├── image │ │ ├── Dockerfile │ │ └── daemon.json │ ├── run-example.gif │ └── variables.tf │ ├── libvirt │ ├── Makefile │ ├── README.md │ ├── cloud-init │ │ └── user-data.cfg.tpl │ ├── cluster.tf │ └── variables.tf │ └── lxd │ ├── Makefile │ ├── README.md │ ├── cluster.tf │ ├── images │ ├── build-image.sh │ ├── distrobuilder-opensuse.yaml │ └── distrobuilder-ubuntu.yaml │ ├── support │ └── get-root-device.sh │ ├── variables.sample.tfvars │ └── variables.tf ├── go.mod ├── go.sum ├── internal ├── assets │ ├── cloud_provider_manifest.go │ ├── doc.go │ ├── generated_cni_conf.go │ ├── generated_flannel_manifest.go │ ├── generated_kubeadm_dropin.go │ ├── generated_kubeadm_setup.go │ ├── generated_kubelet_service.go │ ├── generated_kubelet_sysconfig.go │ ├── static │ │ ├── cloud-provider.yml │ │ ├── cni-default.conflist │ │ ├── kube-flannel.yml │ │ ├── kube-flannel.yml-url │ │ ├── kubeadm-dropin.conf │ │ ├── kubeadm-setup.sh │ │ ├── kubelet.sysconfig │ │ ├── service.conf │ │ ├── weave.yml │ │ └── weave.yml-url │ └── weave_manifest.go └── ssh │ ├── base.go │ ├── base_test.go │ ├── cache.go │ ├── cache_test.go │ ├── commands.go │ ├── commands_test.go │ ├── context.go │ ├── context_test.go │ ├── dirs.go │ ├── docker.go │ ├── files.go │ ├── files_test.go │ ├── kubernetes.go │ ├── net.go │ ├── net_test.go │ ├── processes.go │ ├── test_helpers.go │ └── text.go ├── packaging └── suse │ └── make_spec.sh ├── pkg ├── common │ ├── certs.go │ ├── certs_test.go │ ├── common_test.go │ ├── constants.go │ ├── doc.go │ ├── encoding.go │ ├── file.go │ ├── kubeadm.go │ ├── kubeadm_test.go │ ├── net.go │ ├── net_test.go │ ├── provisioner.go │ ├── strings.go │ ├── strings_test.go │ ├── token.go │ └── validation.go ├── provider │ ├── kubeadm_init.go │ ├── kubeadm_init_test.go │ ├── kubeadm_join.go │ ├── module_test.go │ ├── provider.go │ ├── provider_test.go │ └── schema.go └── provisioner │ ├── action_common.go │ ├── action_drain.go │ ├── action_init.go │ ├── action_init_addons.go │ ├── action_init_cni.go │ ├── action_init_cri.go │ ├── action_init_helm.go │ ├── action_init_helm_test.go │ ├── action_join.go │ ├── action_setup.go │ ├── doc.go │ ├── etcd.go │ ├── etcd_test.go │ ├── kubernetes.go │ ├── kubernetes_test.go │ ├── provisioner.go │ ├── provisioner_test.go │ ├── schema.go │ ├── ssh.go │ ├── token.go │ └── token_test.go ├── staticcheck.conf ├── tests └── e2e │ ├── 01-create-increase-reduce │ ├── 00-setup.sh │ ├── 10-add-master.sh │ ├── 15-remove-all-tokens.sh │ ├── 20-add-worker.sh │ ├── 25-show-tokens.sh │ ├── 30-del-master.sh │ ├── 40-del-worker.sh │ ├── 99-teardown.sh │ ├── README.md │ └── common.bash │ ├── Makefile │ ├── README.md │ └── nightly │ └── Makefile └── utils ├── errcheck.sh ├── generate.sh ├── gofmtcheck.sh └── travis.sh /.gitignore: -------------------------------------------------------------------------------- 1 | osc 2 | .idea 3 | .vscode 4 | 5 | terraform.tfstate* 6 | .terraform 7 | *.auto.tfvars 8 | 9 | *.local 10 | *.bak 11 | crash.log 12 | terraform.log 13 | 14 | .terraform.tfstate.lock.info 15 | docker_mirror_* 16 | 17 | *.code-workspace -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | sudo: required 3 | 4 | language: go 5 | go: 6 | - 1.12.x 7 | 8 | 9 | cache: 10 | directories: 11 | # https://docs.travis-ci.com/user/caching/ 12 | # "If you store archives larger than a few hundred megabytes 13 | # in the cache, it is unlikely that you’ll see a big 14 | # speed improvement". 15 | # 16 | # this list of directories seems to speedup things: 17 | # 18 | - $HOME/.cache/go-build 19 | - $HOME/gopath/pkg/mod 20 | - $HOME/.terraform.d/plugins 21 | 22 | #branches: 23 | # only: 24 | # - master 25 | 26 | env: 27 | global: 28 | # Force-enable Go modules. Also force go to use the code in vendor/ 29 | # These will both be unnecessary when Go 1.13 lands. 30 | - GO111MODULE=on 31 | - TF_ACC=true 32 | - LC_ALL=C.UTF-8 33 | - LANG=C.UTF-8 34 | - CI=travis 35 | - IS_CI=true 36 | - E2E_CLEANUP=true 37 | matrix: 38 | # run tests in two different configurations: with flannel and with weave 39 | - TF_VAR_cni=flannel 40 | - TF_VAR_cni=weave 41 | 42 | before_install: 43 | - make ci-setup 44 | 45 | # NOTE: use this `sudo .. su` trick in order to use things like 46 | # the `usermod -a -G lxd` we did on the `before_install` 47 | script: 48 | - sudo -E su $USER -c "make ci-tests" 49 | 50 | after_failure: 51 | - sudo journalctl -e 52 | 53 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MOD_ENV := GO111MODULE=on GO15VENDOREXPERIMENT=1 2 | GO := $(MOD_ENV) go 3 | GOPATH := $(shell go env GOPATH) 4 | GO_NOMOD := GO111MODULE=off go 5 | GOPATH_FIRST := $(shell echo ${GOPATH} | cut -f1 -d':') 6 | GOBIN := $(shell [ -n "${GOBIN}" ] && echo ${GOBIN} || (echo $(GOPATH_FIRST)/bin)) 7 | GO_VERSION := $(shell $(GO) version | sed -e 's/^[^0-9.]*\([0-9.]*\).*/\1/') 8 | GO_VERSION_MAJ := $(shell echo $(GO_VERSION) | cut -f1 -d'.') 9 | GO_VERSION_MIN := $(shell echo $(GO_VERSION) | cut -f2 -d'.') 10 | 11 | # directories with sources 12 | SRC_DIRS = pkg internal 13 | 14 | TEST ?= $$(go list ./... 2>/dev/null |grep -v 'vendor') 15 | TESTARGS ?= -vet=off 16 | GOFMT_FILES ?= $$(find $(SRC_DIRS) -name '*.go' |grep -v vendor) 17 | WEBSITE_REPO = github.com/hashicorp/terraform-website 18 | WIKI_REPO = $(shell echo `pwd`.wiki) 19 | 20 | # for some unknown reason, "provisioners" are only recognized in this directory 21 | PLUGINS_DIR = $$HOME/.terraform.d/plugins 22 | 23 | # the deployment used for running the E2E tests 24 | E2E_ENV := $(shell echo `pwd`)/docs/examples/dnd 25 | 26 | export GOPATH 27 | export GOBIN 28 | export E2E_ENV 29 | 30 | 31 | all: build 32 | 33 | default: build 34 | 35 | build: fmtcheck build-forced 36 | 37 | $(PLUGINS_DIR): 38 | mkdir -p $(PLUGINS_DIR) 39 | 40 | build-forced: $(PLUGINS_DIR) 41 | $(GO) build -v -o $(PLUGINS_DIR)/terraform-provider-kubeadm ./cmd/terraform-provider-kubeadm 42 | $(GO) build -v -o $(PLUGINS_DIR)/terraform-provisioner-kubeadm ./cmd/terraform-provisioner-kubeadm 43 | 44 | generate: 45 | cd internal/assets && $(GO) generate -x 46 | cd pkg/provider && $(GO) generate -x 47 | cd pkg/provisioner && $(GO) generate -x 48 | 49 | install: build-forced 50 | 51 | .PHONY: vendor 52 | vendor: 53 | $(GO) mod tidy 54 | $(GO) mod vendor 55 | 56 | ################################################ 57 | 58 | test: fmtcheck 59 | @echo ">>> Running tests in $(TEST)..." 60 | $(GO) test -race $(TESTARGS) $(TEST) || exit 1 61 | # echo $(TEST) | $(MOD_ENV) xargs -t -n4 go test $(TESTARGS) -timeout=30s -parallel=4 62 | 63 | testacc: fmtcheck 64 | @echo ">>> Running acceptance tests in $(TEST)..." 65 | TF_ACC=1 $(GO) test $(TEST) $(TESTARGS) -timeout 120m 66 | 67 | test-compile: 68 | @if [ "$(TEST)" = "./..." ]; then \ 69 | echo ">>> ERROR: Set TEST to a specific package. For example,"; \ 70 | echo ">>> make test-compile TEST=./pkg/provisioner"; \ 71 | exit 1; \ 72 | fi 73 | $(GO) test -c $(TEST) $(TESTARGS) 74 | 75 | tests-e2e: ci-tests-e2e 76 | e2e: ci-tests-e2e 77 | e2e-cleanup: ci-tests-e2e-cleanup 78 | e2e-destroy: ci-tests-e2e-cleanup 79 | e2e-logs: ci-tests-e2e-logs 80 | 81 | ################################################ 82 | 83 | vet: 84 | @echo ">>> Checking code with 'go vet'" 85 | @$(GO) vet $$($(GO) list ./... | grep -v vendor/) ; if [ $$? -eq 1 ]; then \ 86 | echo ""; \ 87 | echo "Vet found suspicious constructs. Please check the reported constructs"; \ 88 | echo "and fix them if necessary before submitting the code for review."; \ 89 | exit 1; \ 90 | fi 91 | 92 | fmt: 93 | gofmt -w $(GOFMT_FILES) 94 | 95 | fmtcheck: 96 | @sh -c "'$(CURDIR)/utils/gofmtcheck.sh'" 97 | 98 | errcheck: 99 | @sh -c "'$(CURDIR)/utils/errcheck.sh'" 100 | 101 | 102 | ################################################ 103 | # CI targets (for Travis) 104 | 105 | ci-save-env: 106 | # NOTE: "sudo" in travis resets the environment to "safe" values 107 | # (loaded from "/etc/environment"), so we save our current env 108 | # in that file 109 | @env PATH=/snap/bin:${PATH} > /tmp/environment 110 | @echo ">>> Current environment:" 111 | @cat /tmp/environment 112 | @sudo mv -f /tmp/environment /etc/environment 113 | 114 | ci-tests-style: fmtcheck vet errcheck 115 | 116 | ci-tests-unit: test 117 | 118 | ci-tests-e2e: build 119 | @make -C tests/e2e ci-tests 120 | 121 | ci-tests-e2e-cleanup: 122 | @make -C tests/e2e ci-cleanup E2E_CLEANUP="true" 123 | 124 | ci-tests-e2e-logs: 125 | @make -C tests/e2e ci-logs 126 | 127 | ci: ci-tests 128 | 129 | # entrypoints: ci-tests and ci-setup 130 | 131 | ci-tests: ci-tests-unit ci-tests-style ci-tests-e2e 132 | 133 | ci-setup: 134 | @make -C tests/e2e ci-setup 135 | @make ci-save-env 136 | 137 | ################################################ 138 | 139 | wiki: 140 | @echo ">>> Copying markdown file to $(WIKI_REPO)" 141 | @rm -rf $(WIKI_REPO)/* 142 | @rsync -av --delete \ 143 | --exclude=.git \ 144 | --exclude=examples \ 145 | docs/ $(WIKI_REPO)/ 146 | @echo ">>> Done. You must commit changes in the wiki repo!" 147 | 148 | ################################################ 149 | 150 | rpm: 151 | cd osc && osc build 152 | -------------------------------------------------------------------------------- /cmd/terraform-provider-kubeadm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/hashicorp/terraform/plugin" 5 | 6 | "github.com/inercia/terraform-provider-kubeadm/pkg/provider" 7 | ) 8 | 9 | func main() { 10 | plugin.Serve(&plugin.ServeOpts{ 11 | ProviderFunc: provider.Provider, 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /cmd/terraform-provisioner-kubeadm/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/hashicorp/terraform/plugin" 5 | 6 | "github.com/inercia/terraform-provider-kubeadm/pkg/provisioner" 7 | ) 8 | 9 | func main() { 10 | plugin.Serve(&plugin.ServeOpts{ 11 | ProvisionerFunc: provisioner.Provisioner, 12 | }) 13 | } 14 | -------------------------------------------------------------------------------- /docs/Additional_tasks.md: -------------------------------------------------------------------------------- 1 | # Additional tasks 2 | 3 | Once you have created the cluster with Terraform you should do some other tasks in order 4 | to have a production-quality cluster: 5 | 6 | * create some [Pod Security Policy](https://kubernetes.io/docs/concepts/policy/pod-security-policy/) 7 | and apply it before running any workload in the cluster. 8 | * if the CNI plugin supports it, apply some 9 | [Network Security Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/). 10 | * install [Dex](https://github.com/dexidp/dex/blob/master/Documentation/kubernetes.md) 11 | for authentication, and connect it to your LDAP servers, SAML providers, or some 12 | identity provider like GitHub, Google, and Active Directory. Do not distribute the `kubeconfig` 13 | file created for managing this cluster. 14 | -------------------------------------------------------------------------------- /docs/FAQ.md: -------------------------------------------------------------------------------- 1 | ## How does this compare to...? 2 | 3 | ### Terraform with `kubeadm` run in a `remote-exec`? 4 | 5 | The `kubeadm` provider does some things for you: 6 | 7 | * Automatic distribution of certificates among masters. 8 | 9 | With `kubeadm`, before adding a new master to the cluster you have to 10 | 1) `ssh` to one of the master machines where certificates are found 11 | 2) upload the certificates to the API server (with a key) 12 | 3) create the new master before the key expires (usually one hour) 13 | You have to repeat the process if you try to add another master when 14 | the key has expired. 15 | 16 | But all of that is managed automatically by the `kubeadm` provider. 17 | You just increment the `count` of your masters and you are done. 18 | 19 | * Automatic management of _tokens_. 20 | 21 | So if you want to add a new node to the cluster you don't have to 22 | worry about the token you created before, when you ran `kubeadm init`: 23 | the provider will automatically generate a new token when the old 24 | one has expired or has been removed. 25 | 26 | * Automatic draining of nodes, removal from etcd cluster... on node destruction. (see [this issue](https://github.com/inercia/terraform-provider-kubeadm/issues/5)) 27 | 28 | You can install a [destroy-time provisioner](https://www.terraform.io/docs/provisioners/index.html#destroy-time-provisioners) 29 | that will drain the node from the etcd cluster. In case of masters running `etcd`, 30 | it will also remove the etcd instance from the etcd cluster. 31 | 32 | In addition, having `kubeadm` integrated in Terraform means you can use some 33 | attributes generated by the provider in other parts of your code (ie, the 34 | certificates), and in the other direction: you can inject in the `kubeadm` 35 | configuration things that you created on other Terraform resources (ie, 36 | certificates, IPs, etc) 37 | 38 | -------------------------------------------------------------------------------- /docs/Home.md: -------------------------------------------------------------------------------- 1 | # kubeadm provider and provisioner 2 | 3 | The kubeadm provider is used for interacting with kubeadm for creating Kubernetes clusters. 4 | 5 | ## Example Usage 6 | 7 | ```hcl 8 | resource "kubeadm" "main" { 9 | api { 10 | external = "loadbalancer.external.com" 11 | } 12 | 13 | network { 14 | dns_domain = "my_cluster.local" 15 | services = "10.25.0.0/16" 16 | } 17 | } 18 | 19 | # from the libvirt provider 20 | resource "libvirt_domain" "master" { 21 | name = "master" 22 | memory = 1024 23 | ... 24 | provisioner "kubeadm" { 25 | config = "${kubeadm.main.config}" 26 | } 27 | } 28 | 29 | # from the libvirt provider 30 | resource "libvirt_domain" "minion" { 31 | count = 3 32 | name = "minion${count.index}" 33 | ... 34 | provisioner "kubeadm" { 35 | config = "${kubeadm.main.config}" 36 | join = "${libvirt_domain.master.network_interface.0.addresses.0}" 37 | } 38 | } 39 | ``` 40 | 41 | ## Contents 42 | 43 | * [Installation](Installation) instructions. 44 | * Using `kubeadm` in your Terraform scripts: 45 | * The [`resource "kubeadm"`](Resource_kubeadm) configuration block. 46 | * The [`provisioner "kubeadm"`](Provisioner_kubeadm) block. 47 | * [Additional tasks](Additional_tasks) necessary for having a 48 | fully functional Kubernetes cluster, like installing some Pods 49 | Security Policy... 50 | * [Roadmap, TODO and vision](Roadmap). 51 | * [FAQ](FAQ). 52 | * [Examples](examples/README.md) for several providers like 53 | _libvirt_, _LXD_, _AWS_, etc. 54 | -------------------------------------------------------------------------------- /docs/Installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Requirements 4 | 5 | * [Terraform](https://www.terraform.io) 6 | * Go >= 1.12 (for compiling) 7 | 8 | ## From source 9 | 10 | You can either download the source code from github and `make` it or just run these commands: 11 | 12 | ```bash 13 | $ mkdir -p $HOME/.terraform.d/plugins 14 | $ # with go>=1.12 15 | $ go build -v -o $HOME/.terraform.d/plugins/terraform-provider-kubeadm \ 16 | github.com/inercia/terraform-provider-kubeadm/cmd/terraform-provider-kubeadm 17 | $ go build -v -o $HOME/.terraform.d/plugins/terraform-provisioner-kubeadm \ 18 | github.com/inercia/terraform-provider-kubeadm/cmd/terraform-provisioner-kubeadm 19 | ``` 20 | 21 | -------------------------------------------------------------------------------- /docs/Roadmap.md: -------------------------------------------------------------------------------- 1 | # Roadmap and TODO 2 | 3 | * [ ] The ability to customize the Cloud Provider configuration. 4 | * [ ] The ability to load some PSP. 5 | * [ ] Publish the provider in 6 | * [ ] the Terraform [community page](https://www.terraform.io/docs/providers/type/community-index.html). 7 | * [ ] the [awesome kubernetes](https://github.com/ramitsurana/awesome-kubernetes) installers list. 8 | 9 | -------------------------------------------------------------------------------- /docs/_Sidebar.md: -------------------------------------------------------------------------------- 1 | 2 | * [Installation](Installation) 3 | * Configuration 4 | * [`resource "kubeadm"`](Resource_kubeadm) 5 | * [`provisioner "kubeadm"`](Provisioner_kubeadm) 6 | * [Additional tasks](Additional_tasks) 7 | * [Roadmap, TODO and vision](Roadmap) 8 | * [FAQ](FAQ). 9 | * [Examples](examples/README.md) 10 | -------------------------------------------------------------------------------- /docs/examples/README.md: -------------------------------------------------------------------------------- 1 | # Examples 2 | 3 | This folder contains some example Terraform 4 | scripts for using the kubeadm provider and 5 | provisioner with: 6 | 7 | * [AWS](aws/README.md) 8 | * [libvirt](libvirt/README.md) 9 | * [lxd](lxd/README.md) 10 | * [Docker-in-Docker](dnd/README.md) 11 | -------------------------------------------------------------------------------- /docs/examples/aws/Makefile: -------------------------------------------------------------------------------- 1 | 2 | ################################################################### 3 | # CI 4 | ################################################################### 5 | 6 | # entrypoints: ci-setup, ci-cleanup 7 | 8 | ci-setup: 9 | @echo ">>> No setup to do for AWS..." 10 | 11 | ci-cleanup: 12 | terraform init 13 | terraform destroy --auto-approve 14 | rm -f *.log 15 | -------------------------------------------------------------------------------- /docs/examples/aws/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | Example code for creating a Kubernetes cluster in AWS with the 4 | help of the kubeadm provider. 5 | 6 | ## Pre-requisites 7 | 8 | * Some valid credentials (key and secret) for accessing AWS. 9 | 10 | * `kubectl` 11 | 12 | A local kubectl executable. 13 | 14 | ## Contents 15 | 16 | * [Cluster definition](cluster.tf) 17 | * [Variables](variables.tf) 18 | 19 | ## Machine access 20 | 21 | Depending on the distro used (see the variable `var.ami_distro`), 22 | instances have the default user. For Ubuntu it will be the `ubuntu` 23 | user, while `fedora` for the Fedora distro. 24 | 25 | All nodes should be accessible by jumping through a _bastion host_ 26 | with something like: 27 | 28 | ```bash 29 | $ ssh -J ubuntu@ ubuntu@ 30 | ``` 31 | 32 | where these IP addresses can be obtained after `apply`ing 33 | with `terraform output`. 34 | 35 | The private/public keys used for accessing all the instances (as 36 | well as the bastion host) can be customizable with 37 | `var.private_key`, being the default value 38 | `~/.ssh/id_rsa`/`~/.ssh/id_rsa.pub`. 39 | 40 | ## Usage 41 | 42 | Deployment configuration can be done with a variables 43 | file as well as with environment variables. The configuration 44 | file could be something like: 45 | 46 | ```bash 47 | stack_name = "my-k8s" 48 | ami_distro = "ubuntu" 49 | authorized_keys = [ 50 | "ssh-rsa AAAAB3NzaC1yc2E...", 51 | ] 52 | aws_region = "eu-central-1" 53 | aws_az = "eu-central-1a" 54 | aws_access_key = "AKIAZKQ2..." 55 | aws_secret_key = "ORdkX3vw..." 56 | ``` 57 | 58 | or you could provide these values in environmenta variables 59 | when launching `terraform`: 60 | 61 | ```bash 62 | $ TF_VAR_stack_name="my-k8s" \ 63 | TF_VAR_aws_access_key="AKIAZKQ2..." \ 64 | TF_VAR_aws_secret_key="ORdkX3vw..." \ 65 | terraform apply -auto-approve 66 | ``` 67 | 68 | You could be intersested in customizing some variables like: 69 | 70 | * `stack_name`: identifier to make all your resources unique and avoid 71 | clashes with other users of this Terraform project. 72 | * `authorized_keys`: a list of `ssh` public key to populate in the machines, 73 | used for accessing all the nodes. The private key must have been added to 74 | the ssh agent and the agenet must be running. 75 | * `ami_distro`: the Linux distro to use, currently `ubuntu` or `fedora`. 76 | * `aws_region`: name of the region to be used. 77 | * `aws_az`: the AWS Availability Zone 78 | * `vpc_cidr`, `vpc_cidr_public`, `vpc_cidr_private`: the subnet CIDRs 79 | for the VPC and the public and private subnets. 80 | * `master_size` and `worker_size`: the VM size for the masters 81 | and workers. 82 | 83 | ## Topology 84 | 85 | The cluster will be made by these machines: 86 | 87 | * a Load Balancer that redirects requests to the masters. 88 | * a bastion host, used for accessing the machines though `ssh`, 89 | with a public IP and port 22 open. 90 | * `var.masters` master nodes (by default, two), not accessible 91 | from the outside. 92 | * `var.workers` worker nodes (by default, two), not accessible 93 | from the outside. 94 | 95 | ## Some notes on the Terraform code 96 | 97 | There are some constraints imposed by Terraform/AWS that must be 98 | taken into account when using this code as a base for your own 99 | deployments. 100 | 101 | ### Cluster creation order 102 | 103 | Firstly, the cluster creation order in AWS is not _natural_ 104 | for using our provisioner. The problem comes when `kubeadm` is started 105 | in the seeder as part of the provisioning. At some point it tries to 106 | access the API server through the Load Balancer. However, the Load 107 | Balancer creation depends on the very same instance being fully 108 | created _and provisioned_, so there is a deadlock in the 109 | creation process. 110 | 111 | The only solution is to do the `kubeadm` provisioning _after_ 112 | the instance and the Load Balancer have been created, using a 113 | `null_resource` where the `provisioner "kubeadm"` is embeded. 114 | 115 | ### Nodenames 116 | 117 | Nodes must be registered with the private DNS names provided 118 | by AWS. This can be accomplished by using the `private_dns` name 119 | as the `nodename` in the _provisioner_. 120 | 121 | ### Autoscaling groups 122 | 123 | _TODO_ 124 | 125 | 126 | -------------------------------------------------------------------------------- /docs/examples/aws/cloud-init/cloud-init.yaml.tpl: -------------------------------------------------------------------------------- 1 | #cloud-config 2 | 3 | # set locale 4 | locale: en_GB.UTF-8 5 | 6 | # set timezone 7 | timezone: Etc/UTC 8 | 9 | # we must be careful with repos/packages updates: could abort 10 | # any concurrent installation we could do on command line... 11 | repo_update: true 12 | 13 | # set root password 14 | chpasswd: 15 | list: | 16 | root:linux 17 | expire: False 18 | 19 | ssh_authorized_keys: 20 | - ${public_key} 21 | 22 | runcmd: 23 | - [modprobe, br_netfilter] 24 | - [sh, -c, 'echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables'] 25 | - [sysctl, -w, net.ipv4.ip_forward=1] 26 | 27 | final_message: "The system is finally up, after $UPTIME seconds" 28 | 29 | 30 | -------------------------------------------------------------------------------- /docs/examples/aws/test.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | run: ingress-default-backend 6 | name: ingress-default-backend 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | run: ingress-default-backend 12 | template: 13 | metadata: 14 | labels: 15 | run: ingress-default-backend 16 | spec: 17 | containers: 18 | - name: ingress-default-backend 19 | image: gcr.io/google_containers/defaultbackend:1.0 20 | ports: 21 | - containerPort: 8080 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | labels: 27 | run: ingress-default-backend 28 | name: ingress-default-backend 29 | namespace: default 30 | spec: 31 | ports: 32 | - name: port-1 33 | port: 8080 34 | protocol: TCP 35 | targetPort: 8080 36 | selector: 37 | run: ingress-default-backend 38 | 39 | --- 40 | apiVersion: extensions/v1beta1 41 | kind: Deployment 42 | metadata: 43 | labels: 44 | run: http-svc 45 | name: http-svc 46 | spec: 47 | replicas: 2 48 | selector: 49 | matchLabels: 50 | run: http-svc 51 | template: 52 | metadata: 53 | labels: 54 | run: http-svc 55 | spec: 56 | containers: 57 | - name: http-svc 58 | image: gcr.io/google_containers/echoserver:1.3 59 | ports: 60 | - containerPort: 8080 61 | --- 62 | apiVersion: v1 63 | kind: Service 64 | metadata: 65 | labels: 66 | run: http-svc 67 | name: http-svc 68 | namespace: default 69 | spec: 70 | ports: 71 | - name: port-1 72 | port: 8080 73 | protocol: TCP 74 | targetPort: 8080 75 | selector: 76 | run: http-svc 77 | 78 | --- 79 | apiVersion: extensions/v1beta1 80 | kind: Ingress 81 | metadata: 82 | name: app 83 | spec: 84 | rules: 85 | - host: foo.bar 86 | http: 87 | paths: 88 | - path: /app 89 | backend: 90 | serviceName: http-svc 91 | servicePort: 8080 92 | - path: / 93 | backend: 94 | serviceName: ingress-default-backend 95 | servicePort: 8080 96 | --- 97 | 98 | apiVersion: v1 99 | data: 100 | dynamic-scaling: "true" 101 | backend-server-slots-increment: "4" 102 | kind: ConfigMap 103 | metadata: 104 | name: haproxy-configmap 105 | 106 | --- 107 | 108 | apiVersion: extensions/v1beta1 109 | kind: Deployment 110 | metadata: 111 | labels: 112 | run: haproxy-ingress 113 | name: haproxy-ingress 114 | spec: 115 | replicas: 1 116 | selector: 117 | matchLabels: 118 | run: haproxy-ingress 119 | template: 120 | metadata: 121 | labels: 122 | run: haproxy-ingress 123 | spec: 124 | containers: 125 | - name: haproxy-ingress 126 | image: quay.io/jcmoraisjr/haproxy-ingress 127 | args: 128 | - --default-backend-service=default/ingress-default-backend 129 | - --default-ssl-certificate=default/tls-secret 130 | - --configmap=$(POD_NAMESPACE)/haproxy-configmap 131 | - --reload-strategy=native 132 | ports: 133 | - name: http 134 | containerPort: 80 135 | - name: https 136 | containerPort: 443 137 | - name: stat 138 | containerPort: 1936 139 | env: 140 | - name: POD_NAME 141 | valueFrom: 142 | fieldRef: 143 | fieldPath: metadata.name 144 | - name: POD_NAMESPACE 145 | valueFrom: 146 | fieldRef: 147 | fieldPath: metadata.namespace 148 | 149 | 150 | --- 151 | 152 | apiVersion: v1 153 | kind: Service 154 | metadata: 155 | labels: 156 | run: haproxy-ingress 157 | name: haproxy-ingress 158 | namespace: default 159 | spec: 160 | externalIPs: 161 | - 10.245.1.4 162 | ports: 163 | - name: port-1 164 | port: 80 165 | protocol: TCP 166 | targetPort: 80 167 | - name: port-2 168 | port: 443 169 | protocol: TCP 170 | targetPort: 443 171 | - name: port-3 172 | port: 1936 173 | protocol: TCP 174 | targetPort: 1936 175 | selector: 176 | run: haproxy-ingress 177 | 178 | -------------------------------------------------------------------------------- /docs/examples/aws/variables.tf: -------------------------------------------------------------------------------- 1 | variable "stack_name" { 2 | default = "k8s-test" 3 | description = "identifier to make all your resources unique and avoid clashes with other users of this terraform project" 4 | } 5 | 6 | variable "aws_region" { 7 | default = "eu-west-3" 8 | description = "Name of the region to be used" 9 | } 10 | 11 | variable "aws_az" { 12 | type = "string" 13 | description = "AWS Availability Zone" 14 | default = "eu-west-3a" 15 | } 16 | 17 | variable "ami_distro" { 18 | default = "ubuntu" 19 | description = "AMI distro" 20 | } 21 | 22 | variable "kubeconfig" { 23 | default = "kubeconfig.local" 24 | description = "A local copy of the admin kubeconfig created after the cluster initialization" 25 | } 26 | 27 | variable "cni" { 28 | default = "flannel" 29 | description = "CNI driver" 30 | } 31 | 32 | variable "vpc_cidr" { 33 | type = "string" 34 | default = "10.1.0.0/16" 35 | description = "Subnet CIDR" 36 | } 37 | 38 | variable "public_subnet" { 39 | type = "string" 40 | description = "CIDR blocks for each public subnet of vpc" 41 | default = "10.1.1.0/24" 42 | } 43 | 44 | variable "private_subnet" { 45 | type = "string" 46 | description = "Private subnet of vpc" 47 | default = "10.1.4.0/24" 48 | } 49 | 50 | variable "aws_access_key" { 51 | default = "" 52 | description = "AWS access key" 53 | } 54 | 55 | variable "aws_secret_key" { 56 | default = "" 57 | description = "AWS secret key" 58 | } 59 | 60 | variable "master_size" { 61 | default = "t2.micro" 62 | description = "Size of the master nodes" 63 | } 64 | 65 | variable "master_count" { 66 | default = 2 67 | description = "Number of master nodes" 68 | } 69 | 70 | variable "worker_size" { 71 | default = "t2.micro" 72 | description = "Size of the worker nodes" 73 | } 74 | 75 | variable "worker_count" { 76 | default = 2 77 | description = "Number of worker nodes" 78 | } 79 | 80 | variable "tags" { 81 | type = "map" 82 | default = {} 83 | description = "Extra tags used for the AWS resources created" 84 | } 85 | 86 | variable "authorized_keys" { 87 | type = "list" 88 | default = [] 89 | description = "ssh keys to inject into all the nodes. First key will be used for creating a keypair." 90 | } -------------------------------------------------------------------------------- /docs/examples/dnd/Makefile: -------------------------------------------------------------------------------- 1 | IMAGE := dnd-kubeadm 2 | 3 | all: image 4 | 5 | .PHONY: image 6 | image: 7 | @echo "Builing image..." 8 | cd image && docker build -t $(IMAGE) . 9 | @echo "done!" 10 | 11 | # target for testing the image 12 | # once the image is running, you can "ssh -p 8022 root@127.0.0.1" 13 | # with password "linux" 14 | run: 15 | docker run --rm --privileged=true \ 16 | -v $$HOME/.ssh/id_rsa.pub:/root/.ssh/authorized_keys:ro \ 17 | -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ 18 | -p 8022:22 \ 19 | --name $(IMAGE) \ 20 | $(IMAGE) 21 | 22 | ssh: 23 | ssh-keygen -R "[127.0.0.1]:8022" -f $$HOME/.ssh/known_hosts 24 | @echo ">>> Use 'linux' as password..." 25 | ssh -p 8022 root@127.0.0.1 26 | 27 | clean: 28 | sudo rm -rf docker_mirror_* 29 | 30 | ################################################################### 31 | # CI 32 | ################################################################### 33 | 34 | # entrypoints: ci-setup, ci-cleanup 35 | 36 | # install the requiirements for the CI environment 37 | # this assumes we are running in Travis, in a Ubuntu distro 38 | ci-setup: 39 | @echo ">>> Making sure Docker is not running..." 40 | sudo systemctl stop docker || /bin/true 41 | 42 | @echo ">>> Installing a more modern Docker version..." 43 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 44 | sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu `lsb_release -cs` stable" 45 | sudo apt-get update 46 | sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce 47 | 48 | @echo ">>> Re-enabling the Docker service..." 49 | sudo systemctl enable --now docker 50 | 51 | ci-cleanup: 52 | terraform init 53 | terraform destroy --auto-approve 54 | rm -f *.log 55 | -------------------------------------------------------------------------------- /docs/examples/dnd/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | Terraform cluster definition leveraging the Docker provider, using Docker 4 | containers as Kubernetes nodes. 5 | 6 | This configuration can be used for having a convenient way to start a 7 | Kubernetes cluster in your local laptop, using regular Docker containers 8 | as nodes of your cluster. 9 | 10 | ![Run](run-example.gif) 11 | 12 | ## How does it work? 13 | 14 | The Docker daemon can be run _in_ a Docker container in what is usually called 15 | a _DnD_ (_Docker-in-Docker_) configuration. This requires a special [image/Dockerfile](Dockerfile) 16 | that has been tweaked for starting `systemd` as the the _entrypoint_. `systemd` will them start 17 | the Docker daemon as well as the `kubelet`. Once all these elements are running, we can 18 | run `kubeadm` as in any other platform for starting a Kubernetes cluster. 19 | 20 | ## Pre-requisites 21 | 22 | * _Docker_ 23 | 24 | You will need a functional Docker daemon running. Make sure the `${var.daemon}` 25 | is properly set, pointing to a daemon where you can launch containers. 26 | 27 | * `kubectl` 28 | 29 | A local kubectl executable. 30 | 31 | ## Contents 32 | 33 | * [Cluster definition](cluster.tf) 34 | * [Variables](variables.tf) 35 | 36 | ## Machine access 37 | 38 | By default all the machines will have the following users: 39 | 40 | * All the instances have a `root` user with `linux` password. 41 | 42 | ## Topology 43 | 44 | The cluster will be made by these machines: 45 | 46 | * `${var.master_count}` master nodes, with `kubeadm` and the `kubelet` pre-installed. 47 | * `${var.worker_count}` worker nodes, with `kubeadm` and the `kubelet` pre-installed. 48 | 49 | You should be able to `ssh` these machines, and all of them should be able to ping each other. 50 | 51 | ## Status 52 | 53 | There is a bug in the Terraform Docker provider that 54 | prevents containers from being stopped when using `rm=true`, 55 | so there are some problems when re-creating resources. 56 | -------------------------------------------------------------------------------- /docs/examples/dnd/ci.tfvars: -------------------------------------------------------------------------------- 1 | # setting for Travis-CI 2 | 3 | daemon = "unix:///var/run/docker.sock" 4 | 5 | -------------------------------------------------------------------------------- /docs/examples/dnd/image/Dockerfile: -------------------------------------------------------------------------------- 1 | # kind cluster base image, based on the official Kind base image 2 | # 3 | # To this we add systemd, CNI, and other tools needed to run Kubeadm 4 | # 5 | # For systemd + docker configuration used below, see the following references: 6 | # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ 7 | # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ 8 | # https://developers.redhat.com/blog/2016/09/13/running-systemd-in-a-non-privileged-container/ 9 | 10 | ARG BASE_IMAGE="opensuse/leap:15.1" 11 | FROM ${BASE_IMAGE} 12 | 13 | # NOTE: ARCH must be defined again after FROM 14 | # https://docs.docker.com/engine/reference/builder/#scope 15 | ARG ARCH="amd64" 16 | 17 | # Get dependencies 18 | # The base image already has: ssh, apt, snapd 19 | # This is broken down into (each on a line): 20 | # - packages necessary for installing docker 21 | # - packages needed to run services (systemd) 22 | # - packages needed for docker / hyperkube / kubernetes components 23 | # - misc packages (utilities we use in our own tooling) 24 | # Then we cleanup (removing unwanted systemd services) 25 | # Finally we disable kmsg in journald 26 | # https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ 27 | RUN zypper ar --refresh --enable --no-gpgcheck \ 28 | https://download.opensuse.org/tumbleweed/repo/oss extra-repo0 \ 29 | && zypper ar --refresh --enable --no-gpgcheck \ 30 | https://download.opensuse.org/repositories/devel:/kubic/openSUSE_Leap_15.1 kubic \ 31 | && zypper ref -r extra-repo0 \ 32 | && zypper ref -r kubic \ 33 | && zypper in -y --no-recommends \ 34 | ca-certificates curl gpg2 lsb-release \ 35 | systemd systemd-sysvinit libsystemd0 \ 36 | conntrack-tools iptables iproute2 \ 37 | ethtool socat util-linux ebtables udev kmod \ 38 | bash rsync \ 39 | docker \ 40 | openssh \ 41 | kubernetes-kubeadm \ 42 | kubernetes-kubelet \ 43 | kubernetes-client \ 44 | kmod \ 45 | cni \ 46 | cni-plugins \ 47 | && zypper clean -a \ 48 | && rm -f /lib/systemd/system/multi-user.target.wants/* \ 49 | && rm -f /etc/systemd/system/*.wants/* \ 50 | && rm -f /lib/systemd/system/local-fs.target.wants/* \ 51 | && rm -f /lib/systemd/system/sockets.target.wants/*udev* \ 52 | && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \ 53 | && rm -f /lib/systemd/system/basic.target.wants/* \ 54 | && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \ 55 | && systemctl enable docker.service \ 56 | && systemctl enable sshd.service 57 | 58 | # use some customized Docker settings 59 | #COPY daemon.json /etc/docker/daemon.json 60 | 61 | RUN mkdir -p /etc/systemd/system/docker.service.d 62 | 63 | # Install CNI binaries to /opt/cni/bin 64 | RUN mkdir -p /opt/cni && ln -s /usr/lib/cni /opt/cni/bin 65 | 66 | RUN systemctl set-default multi-user.target 67 | 68 | # tweak sshd 69 | RUN rm -f /etc/ssh/ssh_host_*key* 70 | RUN echo "PermitRootLogin yes" >> /etc/ssh/sshd_config 71 | 72 | # set the root password 73 | RUN echo "root:linux" | chpasswd 74 | 75 | # tell systemd that it is in docker (it will check for the container env) 76 | # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ 77 | ENV container docker 78 | 79 | # systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it) 80 | # https://bugzilla.redhat.com/show_bug.cgi?id=1201657 81 | # "--stop-signal=$(kill -l RTMIN+3) 82 | #STOPSIGNAL SIGRTMIN+3 83 | STOPSIGNAL 37 84 | 85 | # wrap systemd with our special entrypoint, see pkg/build for how this is built 86 | # basically this just lets us set up some things before continuing on to systemd 87 | # while preserving that systemd is PID1 88 | # for how we leverage this, see pkg/cluster 89 | # COPY [ "entrypoint/entrypoint", "/usr/local/bin/" ] 90 | # 91 | # We need systemd to be PID1 to run the various services (docker, kubelet, etc.) 92 | # NOTE: this is *only* for documentation, the entrypoint is overridden at runtime 93 | # ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ] 94 | 95 | # the docker graph must be a volume to avoid overlay on overlay 96 | # NOTE: we do this last because changing a volume with a Dockerfile must 97 | # occur before defining it. 98 | # See: https://docs.docker.com/engine/reference/builder/#volume 99 | VOLUME [ "/var/lib/docker" ] 100 | 101 | VOLUME ["/sys/fs/cgroup"] 102 | # VOLUME ["/run"] 103 | 104 | # TODO(bentheelder): deal with systemd MAC address assignment 105 | # https://github.com/systemd/systemd/issues/3374#issuecomment-288882355 106 | # https://github.com/systemd/systemd/issues/3374#issuecomment-339258483 107 | 108 | # Workaround for docker/docker#27202, technique based on comments from docker/docker#9212 109 | #ENTRYPOINT ["/bin/bash", "-c", "exec /sbin/init --log-target=journal 3>&1"] 110 | ENTRYPOINT ["/sbin/init"] 111 | 112 | -------------------------------------------------------------------------------- /docs/examples/dnd/image/daemon.json: -------------------------------------------------------------------------------- 1 | { 2 | "exec-opts": [ 3 | "native.cgroupdriver=systemd" 4 | ], 5 | "log-driver": "json-file", 6 | "log-opts": { 7 | "max-size": "100m" 8 | } 9 | } -------------------------------------------------------------------------------- /docs/examples/dnd/run-example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/inercia/terraform-provider-kubeadm/9169003e70e7a9c7eddf8aa939a881acd4dda8a8/docs/examples/dnd/run-example.gif -------------------------------------------------------------------------------- /docs/examples/dnd/variables.tf: -------------------------------------------------------------------------------- 1 | ##################### 2 | # Cluster variables # 3 | ##################### 4 | 5 | variable "daemon" { 6 | type = "string" 7 | default = "tcp://127.0.0.1:2375/" 8 | 9 | #default = "unix:///var/run/docker.sock" 10 | description = "Docker daemon socket" 11 | } 12 | 13 | variable "img" { 14 | type = "string" 15 | default = "dnd-kubeadm" 16 | description = "Docker image name" 17 | } 18 | 19 | variable "cni" { 20 | default = "flannel" 21 | description = "CNI driver" 22 | } 23 | 24 | variable "master_count" { 25 | default = 1 26 | description = "Number of masters to be created" 27 | } 28 | 29 | variable "worker_count" { 30 | default = 1 31 | description = "Number of workers to be created" 32 | } 33 | 34 | variable "kubeconfig" { 35 | default = "kubeconfig.local" 36 | description = "Local kubeconfig file" 37 | } 38 | 39 | variable "private_key" { 40 | type = "string" 41 | default = "~/.ssh/id_rsa" 42 | description = "filename of ssh private key used for accessing all the nodes. a corresponding .pub file must exist" 43 | } 44 | 45 | variable "ssh_user" { 46 | type = "string" 47 | default = "root" 48 | description = "The SSH user" 49 | } 50 | 51 | variable "ssh_pass" { 52 | type = "string" 53 | default = "linux" 54 | description = "The SSH password" 55 | } 56 | 57 | variable "domain_name" { 58 | type = "string" 59 | default = "test.net" 60 | description = "The domain name for the nodes in the cluster" 61 | } 62 | 63 | variable "manifests" { 64 | type = "list" 65 | default = [] 66 | description = "List of manifests to load after setting up the first master" 67 | } 68 | 69 | variable "nodes_network" { 70 | type = "string" 71 | default = "172.20.0.0/16" 72 | description = "The network (in CIDR) for the nodes in the cluster" 73 | } 74 | -------------------------------------------------------------------------------- /docs/examples/libvirt/Makefile: -------------------------------------------------------------------------------- 1 | 2 | ################################################################### 3 | # CI 4 | ################################################################### 5 | 6 | # entrypoints: ci-setup, ci-cleanup 7 | 8 | ci-setup: 9 | @echo ">>> No setup to do for libvirt..." 10 | 11 | ci-cleanup: 12 | terraform init 13 | terraform destroy --auto-approve 14 | rm -f *.log 15 | -------------------------------------------------------------------------------- /docs/examples/libvirt/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | Terraform cluster definition leveraging the libvirt provider. 4 | 5 | ## Pre-requisites 6 | 7 | * _libvirt_ 8 | 9 | The installation of libvirt is out of the 10 | scope of this document. Please refer 11 | to the instructions for your particular OS. 12 | 13 | * _terraform/libvirt_ provider 14 | 15 | Follow the instuctions for installing 16 | the [Terraform/libvirt provider](https://github.com/dmacvicar/terraform-provider-libvirt) 17 | 18 | * `kubectl` 19 | 20 | A local kubectl executable. 21 | 22 | ## Contents 23 | 24 | * [Cluster definition](cluster.tf) 25 | * [Variables](variables.tf) 26 | 27 | ## Machine access 28 | 29 | By default all the machines will have the following users: 30 | 31 | * All the instances have a `root` user with `linux` password. 32 | 33 | ## Topology 34 | 35 | The topology created for libvirt is currently a bit limited: 36 | 37 | * only one master, with `kubeadm` and the `kubelet` pre-installed. 38 | No load balancer is created, so you are limited to only one master. 39 | * `${var.worker_count}` worker nodes, with `kubeadm` and the `kubelet` pre-installed. 40 | 41 | 42 | You should be able to `ssh` these machines, and all of them should be able to ping each other. 43 | -------------------------------------------------------------------------------- /docs/examples/libvirt/cloud-init/user-data.cfg.tpl: -------------------------------------------------------------------------------- 1 | # set locale 2 | locale: en_GB.UTF-8 3 | 4 | # set timezone 5 | timezone: Etc/UTC 6 | 7 | # set root password 8 | chpasswd: 9 | list: | 10 | root:linux 11 | opensuse:linux 12 | expire: False 13 | 14 | bootcmd: 15 | - ip link set dev eth0 mtu 1400 16 | 17 | final_message: "The system is finally up, after $UPTIME seconds" 18 | -------------------------------------------------------------------------------- /docs/examples/libvirt/cluster.tf: -------------------------------------------------------------------------------- 1 | ##################### 2 | # global 3 | ##################### 4 | 5 | provider "libvirt" { 6 | uri = "qemu:///system" 7 | } 8 | 9 | resource "libvirt_network" "backend" { 10 | name = "${var.name_prefix}net" 11 | mode = "nat" 12 | domain = "local" 13 | addresses = ["10.17.6.0/24"] 14 | } 15 | 16 | resource "libvirt_volume" "base" { 17 | name = "${var.name_prefix}base.img" 18 | source = "${var.image}" 19 | pool = "${var.image_pool}" 20 | } 21 | 22 | data "template_file" "cloud_init_user_data" { 23 | template = "${file("cloud-init/user-data.cfg.tpl")}" 24 | } 25 | 26 | ########################## 27 | # Kubeadm # 28 | ########################## 29 | 30 | resource "kubeadm" "main" { 31 | network { 32 | dns { 33 | domain = "mycluster.com" 34 | } 35 | services = "10.25.0.0/16" 36 | } 37 | 38 | cni { 39 | plugin = "${var.cni}" 40 | } 41 | 42 | runtime { 43 | # note: "crio" seems to have some issues: some pods keep erroring 44 | # in "ContainerCreating", with "failed to get network status for pod sandbox" 45 | # switching to Docker solves those problems... 46 | engine = "docker" 47 | } 48 | 49 | helm { 50 | install = true 51 | } 52 | 53 | dashboard { 54 | install = true 55 | } 56 | } 57 | 58 | ##################### 59 | # kube-master 60 | ##################### 61 | resource "libvirt_volume" "master_volume" { 62 | name = "${var.name_prefix}master.img" 63 | pool = "${var.image_pool}" 64 | base_volume_id = "${libvirt_volume.base.id}" 65 | size = "10737418240" 66 | } 67 | 68 | resource "libvirt_cloudinit_disk" "ci" { 69 | name = "${var.name_prefix}ci.iso" 70 | pool = "${var.image_pool}" 71 | user_data = "${data.template_file.cloud_init_user_data.rendered}" 72 | } 73 | 74 | resource "libvirt_domain" "master" { 75 | name = "${var.name_prefix}master" 76 | memory = 512 77 | cloudinit = "${libvirt_cloudinit_disk.ci.id}" 78 | 79 | disk { 80 | volume_id = "${libvirt_volume.master_volume.id}" 81 | } 82 | 83 | connection { 84 | type = "ssh" 85 | user = "root" 86 | password = "linux" 87 | } 88 | 89 | network_interface { 90 | network_id = "${libvirt_network.backend.id}" 91 | hostname = "${var.name_prefix}master.local" 92 | wait_for_lease = 1 93 | } 94 | 95 | provisioner "kubeadm" { 96 | config = "${kubeadm.main.config}" 97 | kubeconfig = "${var.kubeconfig}" 98 | manifests = "${var.manifests}" 99 | } 100 | } 101 | 102 | output "masters" { 103 | value = ["${libvirt_domain.master.*.network_interface.0.addresses}"] 104 | } 105 | 106 | ##################### 107 | # kube-minion 108 | ##################### 109 | resource "libvirt_volume" "minion_volume" { 110 | count = "${var.worker_count}" 111 | name = "${var.name_prefix}minion${count.index}.img" 112 | pool = "${var.image_pool}" 113 | base_volume_id = "${libvirt_volume.base.id}" 114 | size = "10737418240" 115 | } 116 | 117 | resource "libvirt_domain" "minion" { 118 | count = "${var.worker_count}" 119 | name = "${var.name_prefix}minion${count.index}" 120 | depends_on = ["libvirt_domain.master"] 121 | memory = 512 122 | cloudinit = "${libvirt_cloudinit_disk.ci.id}" 123 | 124 | disk { 125 | volume_id = "${element(libvirt_volume.minion_volume.*.id, count.index)}" 126 | } 127 | 128 | connection { 129 | type = "ssh" 130 | user = "root" 131 | password = "linux" 132 | } 133 | 134 | network_interface { 135 | network_id = "${libvirt_network.backend.id}" 136 | hostname = "${var.name_prefix}minion${count.index}.local" 137 | wait_for_lease = 1 138 | } 139 | 140 | provisioner "kubeadm" { 141 | config = "${kubeadm.main.config}" 142 | join = "${libvirt_domain.master.network_interface.0.addresses.0}" 143 | 144 | #ignore_checks = [ 145 | # "NumCPU", 146 | # "FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", 147 | # "Swap", 148 | #] 149 | } 150 | } 151 | 152 | output "workers" { 153 | value = ["${libvirt_domain.minion.*.network_interface.0.addresses}"] 154 | } 155 | -------------------------------------------------------------------------------- /docs/examples/libvirt/variables.tf: -------------------------------------------------------------------------------- 1 | ##################### 2 | # variables 3 | ##################### 4 | 5 | variable "master_count" { 6 | default = "1" 7 | } 8 | 9 | variable "worker_count" { 10 | default = "1" 11 | } 12 | 13 | variable "cni" { 14 | default = "flannel" 15 | description = "CNI driver" 16 | } 17 | 18 | variable "ssh" { 19 | default = "../ssh/id_rsa" 20 | } 21 | 22 | variable "image" { 23 | default = "https://cloud-images.ubuntu.com/releases/xenial/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img" 24 | } 25 | 26 | variable "image_pool" { 27 | default = "default" 28 | } 29 | 30 | variable "name_prefix" { 31 | type = "string" 32 | default = "kadm-lv-" 33 | description = "Optional prefix to be able to have multiple clusters on one host" 34 | } 35 | 36 | variable "manifests" { 37 | type = "list" 38 | 39 | default = [ 40 | "https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml", 41 | "https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml", 42 | "https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/provider/cloud-generic.yaml", 43 | "https://raw.githubusercontent.com/kubernetes/dashboard/master/aio/deploy/recommended/kubernetes-dashboard.yaml", 44 | ] 45 | 46 | description = "List of manifests to load after setting up the first master" 47 | } 48 | 49 | variable "kubeconfig" { 50 | default = "kubeconfig.local" 51 | description = "Local kubeconfig file" 52 | } -------------------------------------------------------------------------------- /docs/examples/lxd/Makefile: -------------------------------------------------------------------------------- 1 | MOD_ENV := GO111MODULE=on GO15VENDOREXPERIMENT=1 2 | GO := $(MOD_ENV) go 3 | GOPATH := $(shell go env GOPATH) 4 | GO_NOMOD := GO111MODULE=off go 5 | GOPATH_FIRST := $(shell echo ${GOPATH} | cut -f1 -d':') 6 | GOBIN := $(shell [ -n "${GOBIN}" ] && echo ${GOBIN} || (echo $(GOPATH_FIRST)/bin)) 7 | 8 | TF_LXD_PROV_URL := https://github.com/sl1pm4t/terraform-provider-lxd/releases/download/v1.1.3/terraform-provider-lxd_v1.1.3_linux_amd64.zip 9 | TF_LXD_PROV_GOGET := github.com/sl1pm4t/terraform-provider-lxd 10 | 11 | TF_SSH_KEY = ~/.ssh/terraform 12 | PLUGINS_DIR = $$HOME/.terraform.d/plugins 13 | 14 | export TF_LOG=DEBUG 15 | export TF_VAR_private_key=$(TF_SSH_KEY) 16 | export TF_IN_AUTOMATION=1 17 | export GOPATH 18 | export GOBIN 19 | 20 | all: 21 | @echo ">>> This Makefile does not contain targets for users." 22 | 23 | $(TF_SSH_KEY): 24 | @echo ">>> Generating ssh key for Terraform..." 25 | mkdir -p `dirname $(TF_SSH_KEY)` 26 | ssh-keygen -t rsa -N "" -f $(TF_SSH_KEY) 27 | 28 | $(PLUGINS_DIR): 29 | mkdir -p $(PLUGINS_DIR) 30 | 31 | ################################################################### 32 | # CI 33 | ################################################################### 34 | 35 | ci-install-lxd: 36 | # LXD version in Xenial is too old (2.0): we must use the snap 37 | @echo ">>> Installing LXD snap..." 38 | sudo apt remove -y --purge lxd lxd-client 39 | sudo snap install lxd 40 | sudo sh -c 'echo PATH=/snap/bin:${PATH} >> /etc/environment' 41 | sudo lxd waitready 42 | sudo lxd init --auto 43 | sudo usermod -a -G lxd `whoami` 44 | 45 | ci-install-tf-lxd-from-srcs: $(PLUGINS_DIR) 46 | GO111MODULE=on go get -u $(TF_LXD_PROV_GOGET) 47 | cd $(GOPATH_FIRST)/src/$(TF_LXD_PROV_GOGET) 48 | GO111MODULE=on make build 49 | mv -f terraform-provider-lxd* $(PLUGINS_DIR)/terraform-provider-lxd 50 | chmod 755 $(PLUGINS_DIR)/terraform-provider-lxd 51 | 52 | ci-install-tf-lxd-from-zip: $(PLUGINS_DIR) 53 | cd /tmp && \ 54 | rm -f terraform-provider-lxd* && \ 55 | curl -L -O $(TF_LXD_PROV_URL) && \ 56 | unzip -xU terraform-provider-lxd*.zip && \ 57 | rm -f terraform-provider-lxd*.zip && \ 58 | mv -f terraform-provider-lxd* $(PLUGINS_DIR)/terraform-provider-lxd && \ 59 | chmod 755 $(PLUGINS_DIR)/terraform-provider-lxd 60 | 61 | ci-install-tf-lxd: 62 | @echo ">>> Installing the Terraform/LXD provider..." 63 | [ -x $(PLUGINS_DIR)/terraform-provider-lxd ] || \ 64 | make ci-install-tf-lxd-from-zip 65 | @echo ">>> Terraform/LXD provider installed." 66 | 67 | ci-check-env: 68 | @echo ">>> Checking things needed in the Terraform script" 69 | [ -e /dev/mem ] || exit 1 70 | [ -d /lib/modules ] || exit 1 71 | 72 | # entrypoints: ci-setup 73 | 74 | ci-setup: ci-check-env ci-install-tf-lxd ci-install-lxd $(TF_SSH_KEY) 75 | 76 | ci-cleanup: 77 | terraform init 78 | terraform destroy --auto-approve 79 | rm -f *.log 80 | 81 | -------------------------------------------------------------------------------- /docs/examples/lxd/README.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | Terraform cluster definition leveraging the libvirt provider. 4 | 5 | ## Pre-requisites 6 | 7 | * _LXD_ 8 | 9 | The easiest way to install LXD is with a Snap: https://snapcraft.io/lxd. 10 | Just do a `snap install lxd`. Then you will have to add your username to 11 | the `lxd` group (for accessing the LXD socket without being root) with 12 | `sudo usermod -aG lxd $USER`, logout and login again. 13 | 14 | * _terraform/LXD_ 15 | 16 | You whill have to compile the LXD provider by yourself with a Golang compiler 17 | (and your `GOPATH` properly set). 18 | Do a `go get -v -u github.com/sl1pm4t/terraform-provider-lxd`. 19 | Maybe you will have to add the provider to your `~/.terraformrc` if terraform does not find 20 | the provider automatically. For example: 21 | ``` 22 | providers { 23 | lxd = "/src/github.com/sl1pm4t/terraform-provider-lxd/terraform-provider-lxd" 24 | } 25 | ``` 26 | (replacing `` by your GOPATH). 27 | 28 | * `kubectl` 29 | 30 | A local kubectl executable. 31 | 32 | ## Contents 33 | 34 | * [Cluster definition](cluster.tf) 35 | * [Variables](variables.tf) 36 | 37 | ## Machine access 38 | 39 | By default all the machines will have the following users: 40 | 41 | * All the instances have a `root` user with `linux` password. 42 | 43 | ## Topology 44 | 45 | The cluster will be made by these machines: 46 | 47 | * `${var.master_count}` master nodes, with `kubeadm` and the `kubelet` pre-installed. 48 | * `${var.worker_count}` worker nodes, with `kubeadm` and the `kubelet` pre-installed. 49 | 50 | You should be able to `ssh` these machines, and all of them should be able to ping each other. 51 | -------------------------------------------------------------------------------- /docs/examples/lxd/images/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # LXD image build 4 | # 5 | 6 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 7 | [ -z "$GOPATH" ] && echo ">>> FATAL: GOPATH not defined !!!" && exit 1 8 | 9 | FIRST_GOPATH=$(echo `echo ${GOPATH} | cut -f1 -d':'`) 10 | GOBIN=$([ -n "${GOBIN}" ] && echo ${GOBIN} || (echo $(GOPATH)/bin)) 11 | 12 | DISTROBUILDER= 13 | DISTROBUILDER_YAML=$DIR/distrobuilder-opensuse.yaml 14 | DISTROBUILDER_CACHE=$HOME/.cache/distrobuilder 15 | IMAGE_ALIAS="lxd-kubeadm" 16 | 17 | FORCE= 18 | 19 | read -r -d '' HELP <<- EOM 20 | Usage: 21 | 22 | $0 [args] 23 | 24 | where [args] can be: 25 | 26 | --img the name for the image to load in LXD 27 | (default: $IMAGE_ALIAS) 28 | --yaml the YAML definition file 29 | (default: $DISTROBUILDER_YAML) 30 | --distrobuilder the distrobuilder executable 31 | (will be downloaded if no provided/autodetected) 32 | --force force the image (re)creation 33 | 34 | EOM 35 | 36 | while [ $# -gt 0 ] ; do 37 | case $1 in 38 | --img|--alias) 39 | IMAGE_ALIAS=$2 40 | shift 41 | ;; 42 | --distrobuilder|--exe) 43 | DISTROBUILDER=$2 44 | shift 45 | ;; 46 | --yaml|--definition) 47 | DISTROBUILDER_YAML=$2 48 | shift 49 | ;; 50 | --force) 51 | case $2 in 52 | true|TRUE|yes|YES|1) 53 | FORCE=1 54 | ;; 55 | false|FALSE|no|NO|0) 56 | FORCE= 57 | ;; 58 | esac 59 | shift 60 | ;; 61 | --help|-h) 62 | echo "$HELP" 63 | exit 0 64 | ;; 65 | *) 66 | echo "unknown argument $1" 67 | exit 1 68 | ;; 69 | esac 70 | shift 71 | done 72 | 73 | ######################################################################### 74 | 75 | check_distrobuilder() { 76 | if [ -n "$DISTROBUILDER" ] ; then 77 | echo ">>> (distrobuilder provided: $DISTROBUILDER)" 78 | return 79 | fi 80 | 81 | if command distrobuilder &>/dev/null ; then 82 | DISTROBUILDER="$(which distrobuilder)" 83 | echo ">>> (distrobuilder found in the PATH)" 84 | return 85 | fi 86 | 87 | if [ -x $GOBIN/distrobuilder ] ; then 88 | echo ">>> (distrobuilder found at $GOBIN/distrobuilder)" 89 | DISTROBUILDER=$GOBIN/distrobuilder 90 | return 91 | fi 92 | 93 | echo ">>> downloading 'distrobuilder' with 'go get'..." 94 | GO111MODULE=on go get github.com/lxc/distrobuilder/distrobuilder 95 | if [ ! -x $GOBIN/distrobuilder ] ; then 96 | echo "distrobuilder could not be built" 97 | exit 1 98 | fi 99 | echo ">>> 'distrobuilder' installed at $GOBIN/distrobuilder" 100 | DISTROBUILDER=$GOBIN/distrobuilder 101 | } 102 | 103 | ######################################################################### 104 | # LXD images 105 | 106 | image_exists() { 107 | lxc image show $IMAGE_ALIAS &>/dev/null 108 | } 109 | 110 | image_delete() { 111 | echo ">>> deleting any previous image $IMAGE_ALIAS..." 112 | lxc image delete $IMAGE_ALIAS 2>/dev/null || /bin/true 113 | } 114 | 115 | image_artifacts_exist() { 116 | [ -f lxd.tar.xz ] && [ -f rootfs.squashfs ] 117 | } 118 | 119 | image_build() { 120 | echo ">>> building LXD image with $DISTROBUILDER..." 121 | echo ">>> IMPORTANT: distrobuilder will be launched with SUDO !!!" 122 | sudo $DISTROBUILDER \ 123 | --cache-dir=$DISTROBUILDER_CACHE \ 124 | --cleanup=true build-lxd $DISTROBUILDER_YAML 125 | } 126 | 127 | image_import() { 128 | image_delete 129 | 130 | echo ">>> importing LXD image..." 131 | lxc image import lxd.tar.xz rootfs.squashfs --alias $IMAGE_ALIAS 132 | echo ">>> removing leftovers" 133 | rm -f lxd.tar.xz rootfs.squashfs 134 | echo ">>> current list of LXD images:" 135 | lxc image list 136 | } 137 | 138 | image_cleanup() { 139 | echo ">>> cleaning up leftovers..." 140 | # just in case... 141 | } 142 | 143 | [ -n "$FORCE" ] && image_delete 144 | echo ">>> checking if $IMAGE_ALIAS image exists..." 145 | if ! image_exists ; then 146 | echo ">>> $IMAGE_ALIAS does not exist: building..." 147 | check_distrobuilder 148 | trap image_cleanup EXIT 149 | image_artifacts_exist || image_build 150 | image_artifacts_exist && image_import 151 | else 152 | echo ">>> $IMAGE_ALIAS already present!" 153 | fi 154 | 155 | -------------------------------------------------------------------------------- /docs/examples/lxd/images/distrobuilder-opensuse.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # openSUSE definition file 3 | # 4 | ################################################################################### 5 | # NOTE: make sure you REMOVE the old image (with `lxc delete opensuse`) 6 | # AFTER MODIFYING THIS FILE 7 | ################################################################################### 8 | 9 | image: 10 | distribution: openSUSE 11 | release: 15.0 12 | description: openSUSE Leap {{ image.release }} with kubeadm 13 | expiry: 30d 14 | architecture: x86_64 15 | 16 | source: 17 | downloader: opensuse-http 18 | url: https://download.opensuse.org 19 | skip_verification: true 20 | 21 | targets: 22 | lxc: 23 | create-message: | 24 | You just created an openSUSE Leap container (release={{ image.release }}, arch={{ image.architecture }}) 25 | 26 | config: 27 | - type: all 28 | before: 5 29 | content: |- 30 | lxc.include = LXC_TEMPLATE_CONFIG/opensuse.common.conf 31 | 32 | - type: user 33 | before: 5 34 | content: |- 35 | lxc.include = LXC_TEMPLATE_CONFIG/opensuse.userns.conf 36 | 37 | - type: all 38 | after: 4 39 | content: |- 40 | lxc.include = LXC_TEMPLATE_CONFIG/common.conf 41 | 42 | - type: user 43 | after: 4 44 | content: |- 45 | lxc.include = LXC_TEMPLATE_CONFIG/userns.conf 46 | 47 | - type: all 48 | content: |- 49 | lxc.arch = {{ image.architecture_kernel }} 50 | 51 | files: 52 | - path: /etc/hostname 53 | generator: hostname 54 | 55 | - path: /etc/hosts 56 | generator: hosts 57 | 58 | - name: ifcfg-eth0.lxd 59 | path: /etc/sysconfig/network/ifcfg-eth0 60 | generator: template 61 | content: |- 62 | STARTMODE='auto' 63 | BOOTPROTO='dhcp' 64 | HOSTNAME={{ container.name }} 65 | DHCP_HOSTNAME=`hostname` 66 | 67 | # From the version v11.0 kubelet requires to have shared mode for the host mounts. 68 | # There is dirty hack for achieve that, inside LXC-container run: 69 | # see https://medium.com/@kvaps/run-kubernetes-in-lxc-container-f04aa94b6c9c 70 | - path: /etc/rc.local 71 | generator: dump 72 | content: |- 73 | #!/bin/sh -e 74 | mount --make-rshared / 75 | 76 | - path: /etc/cni/net.d/99-loopback.conf 77 | generator: dump 78 | content: |- 79 | { 80 | "cniVersion": "0.2.0", 81 | "name": "lo", 82 | "type": "loopback" 83 | } 84 | 85 | packages: 86 | manager: zypper 87 | update: false 88 | refresh: true 89 | cleanup: true 90 | repositories: 91 | - name: kubic 92 | url: https://download.opensuse.org/repositories/devel:/kubic/openSUSE_Leap_15.1/ 93 | sets: 94 | - packages: 95 | - systemd-sysvinit 96 | - openssh 97 | - sudo 98 | - apparmor-abstractions 99 | - elfutils 100 | - file 101 | - glib2-tools 102 | - gzip 103 | - hardlink 104 | - hostname 105 | - iputils 106 | - net-tools 107 | - openslp 108 | - rsync 109 | - shared-mime-info 110 | - which 111 | - vim 112 | # CaaSP packages 113 | - kubernetes-kubeadm 114 | - kubernetes-kubelet 115 | - kubectl 116 | - haproxy 117 | - docker 118 | action: install 119 | 120 | actions: 121 | - trigger: post-packages 122 | action: |- 123 | #!/bin/sh 124 | 125 | mkdir -p /etc/cni/net.d 126 | 127 | # NOTE: CNI plugins are installed in /usr/lib/cni in OpenSUSE 128 | 129 | # encrypted "linux" 130 | # obtained with `echo "linux" | openssl passwd -1 -stdin` 131 | ROOT_PASSWORD='$1$62xujQ/G$IxTMM4LZimNXF3LFcBawC1' 132 | 133 | echo ">>> setting a trivial password for root and allowing SSH for root" 134 | echo "root:$ROOT_PASSWORD" | /usr/sbin/chpasswd -e 135 | echo "PermitRootLogin yes" >> /etc/ssh/sshd_config 136 | 137 | echo ">>> fixing authorized keys" 138 | mkdir -p /root/.ssh 139 | chmod 700 /root/.ssh 140 | touch /root/.ssh/authorized_keys 141 | chmod 600 /root/.ssh/authorized_keys 142 | 143 | echo ">>> making sure some packages are not installed..." 144 | rpm -e --nodeps docker-kubic docker-libnetwork-kubic docker-runc-kubic || /bin/true 145 | 146 | # /usr/bin/sed -i -e 's/btrfs/overlay2/g' /etc/crio/crio.conf 147 | 148 | echo ">>> enabling some services..." 149 | systemctl enable sshd 150 | systemctl enable docker 151 | systemctl disable crio 152 | 153 | # some commands are really useless in a LXC container, so we create "fake" clones: 154 | for target in /usr/bin/kmod /sbin/sysctl ; do 155 | rm -f $target 156 | ln -s /bin/true $target 157 | done 158 | 159 | exit 0 160 | 161 | environment: 162 | variables: 163 | - key: HOME 164 | value: /root 165 | -------------------------------------------------------------------------------- /docs/examples/lxd/support/get-root-device.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ROOT_DEV=$(mount | grep " on / " | cut -f1 -d" ") 4 | 5 | echo '{"device":"'$ROOT_DEV'"}' -------------------------------------------------------------------------------- /docs/examples/lxd/variables.sample.tfvars: -------------------------------------------------------------------------------- 1 | master_count = 1 2 | worker_count = 1 3 | 4 | # A range that doesn't conflict with the SUSE network 5 | # and allows a similar naming. 6 | #network = "172.30.0.0/22" 7 | #net_mode = "route" 8 | 9 | authorized_keys = [ 10 | "/home/alvaro/.ssh/id_rsa.pub" 11 | ] 12 | 13 | -------------------------------------------------------------------------------- /docs/examples/lxd/variables.tf: -------------------------------------------------------------------------------- 1 | ##################### 2 | # Cluster variables # 3 | ##################### 4 | 5 | variable "img" { 6 | type = "string" 7 | default = "lxd-kubeadm" 8 | description = "image name" 9 | } 10 | 11 | variable "distrobuilder" { 12 | type = "string" 13 | default = "distrobuilder-opensuse.yaml" 14 | description = "image name" 15 | } 16 | 17 | variable "force_img" { 18 | type = "string" 19 | default = "" 20 | description = "force the image re-creation" 21 | } 22 | 23 | variable "master_count" { 24 | default = 1 25 | description = "Number of masters to be created" 26 | } 27 | 28 | variable "worker_count" { 29 | default = 1 30 | description = "Number of workers to be created" 31 | } 32 | 33 | variable "cni" { 34 | default = "flannel" 35 | description = "CNI driver" 36 | } 37 | 38 | variable "kubeconfig" { 39 | default = "kubeconfig.local" 40 | description = "Local kubeconfig file" 41 | } 42 | 43 | variable "name_prefix" { 44 | type = "string" 45 | default = "kubeadm-" 46 | description = "Optional prefix to be able to have multiple clusters on one host" 47 | } 48 | 49 | variable "private_key" { 50 | type = "string" 51 | default = "~/.ssh/id_rsa" 52 | description = "filename of ssh private key used for accessing all the nodes. a corresponding .pub file must exist" 53 | } 54 | 55 | variable "ssh_user" { 56 | type = "string" 57 | default = "root" 58 | description = "The SSH user" 59 | } 60 | 61 | variable "ssh_pass" { 62 | type = "string" 63 | default = "linux" 64 | description = "The SSH password" 65 | } 66 | 67 | variable "domain_name" { 68 | type = "string" 69 | default = "test.net" 70 | description = "The domain name" 71 | } 72 | 73 | variable "manifests" { 74 | type = "list" 75 | default = [] 76 | description = "List of manifests to load after setting up the first master" 77 | } 78 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/inercia/terraform-provider-kubeadm 2 | 3 | require ( 4 | github.com/DATA-DOG/go-sqlmock v1.3.3 // indirect 5 | github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e // indirect 6 | github.com/Masterminds/goutils v1.1.0 // indirect 7 | github.com/Masterminds/semver v1.4.2 // indirect 8 | github.com/Masterminds/sprig v2.20.0+incompatible // indirect 9 | github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 10 | github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect 11 | github.com/cyphar/filepath-securejoin v0.2.2 // indirect 12 | github.com/davecgh/go-spew v1.1.1 13 | github.com/docker/distribution v2.7.1+incompatible // indirect 14 | github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect 15 | github.com/fatih/camelcase v1.0.0 // indirect 16 | github.com/gobuffalo/packr v1.30.1 // indirect 17 | github.com/gobwas/glob v0.2.3 // indirect 18 | github.com/gogo/protobuf v1.2.1 // indirect 19 | github.com/googleapis/gnostic v0.2.0 // indirect 20 | github.com/gookit/color v1.1.7 21 | github.com/hashicorp/terraform v0.12.3 22 | github.com/helm/helm v2.14.3+incompatible 23 | github.com/huandu/xstrings v1.2.0 // indirect 24 | github.com/imdario/mergo v0.3.7 // indirect 25 | github.com/jmoiron/sqlx v1.2.0 // indirect 26 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect 27 | github.com/lithammer/dedent v1.1.0 // indirect 28 | github.com/mitchellh/go-linereader v0.0.0-20190213213312-1b945b3263eb 29 | github.com/opencontainers/go-digest v1.0.0-rc1 // indirect 30 | github.com/pkg/errors v0.8.1 // indirect 31 | github.com/rubenv/sql-migrate v0.0.0-20190717103323-87ce952f7079 // indirect 32 | github.com/spf13/afero v1.2.2 // indirect 33 | github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect 34 | github.com/ziutek/mymysql v1.5.4 // indirect 35 | gopkg.in/gorp.v1 v1.7.2 // indirect 36 | k8s.io/apiextensions-apiserver v0.0.0-20190315093550-53c4693659ed // indirect 37 | k8s.io/apiserver v0.0.0-20190424053242-2200fef3ea67 // indirect 38 | k8s.io/cli-runtime v0.0.0-20190726024606-74a61cd71909 // indirect 39 | k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible 40 | k8s.io/cloud-provider v0.0.0-20190405093944-6c8b65ee8f98 // indirect 41 | k8s.io/cluster-bootstrap v0.0.0-20190626010831-cd8eb24ea488 // indirect 42 | k8s.io/helm v2.14.3+incompatible 43 | k8s.io/kube-proxy v0.0.0-20190314002154-4d735c31b054 // indirect 44 | k8s.io/kubelet v0.0.0-20190314002251-f6da02f58325 // indirect 45 | k8s.io/kubernetes v1.14.1 46 | k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect 47 | vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 // indirect 48 | ) 49 | 50 | replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190626045420-1ec4b74c7bda 51 | 52 | // fix some transitional deps 53 | replace github.com/gogo/protobuf v0.0.0-20190410021324-65acae22fc9 => github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d 54 | 55 | exclude github.com/Sirupsen/logrus v1.4.1 56 | 57 | exclude github.com/Sirupsen/logrus v1.4.0 58 | 59 | exclude github.com/Sirupsen/logrus v1.3.0 60 | 61 | exclude github.com/Sirupsen/logrus v1.2.0 62 | 63 | exclude github.com/Sirupsen/logrus v1.1.1 64 | 65 | exclude github.com/Sirupsen/logrus v1.1.0 66 | 67 | exclude github.com/renstrom/dedent v1.1.0 68 | 69 | go 1.13 70 | -------------------------------------------------------------------------------- /internal/assets/cloud_provider_manifest.go: -------------------------------------------------------------------------------- 1 | // Code generated automatically with go generate; DO NOT EDIT. 2 | 3 | package assets 4 | 5 | const CloudProviderCode = `# from https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/ 6 | 7 | {{- if .cloud_config}} 8 | apiVersion: v1 9 | kind: Secret 10 | metadata: 11 | name: cloud-provider-config 12 | type: Opaque 13 | data: 14 | # "cloud_config" contains the Base64 encoded configuration file 15 | cloud.conf: {{.cloud_config}} 16 | {{- end}} 17 | 18 | --- 19 | apiVersion: v1 20 | kind: ServiceAccount 21 | metadata: 22 | name: cloud-controller-manager 23 | namespace: kube-system 24 | 25 | --- 26 | apiVersion: rbac.authorization.k8s.io/v1 27 | kind: ClusterRoleBinding 28 | metadata: 29 | name: system:cloud-controller-manager 30 | roleRef: 31 | apiGroup: rbac.authorization.k8s.io 32 | kind: ClusterRole 33 | name: cluster-admin 34 | subjects: 35 | - kind: ServiceAccount 36 | name: cloud-controller-manager 37 | namespace: kube-system 38 | 39 | --- 40 | apiVersion: apps/v1 41 | kind: DaemonSet 42 | metadata: 43 | labels: 44 | k8s-app: cloud-controller-manager 45 | name: cloud-controller-manager 46 | namespace: kube-system 47 | spec: 48 | selector: 49 | matchLabels: 50 | k8s-app: cloud-controller-manager 51 | template: 52 | metadata: 53 | labels: 54 | k8s-app: cloud-controller-manager 55 | spec: 56 | serviceAccountName: cloud-controller-manager 57 | containers: 58 | - name: cloud-controller-manager 59 | # for in-tree providers we use k8s.gcr.io/cloud-controller-manager 60 | # this can be replaced with any other image for out-of-tree providers 61 | image: k8s.gcr.io/cloud-controller-manager:v1.8.0 62 | command: 63 | - /usr/local/bin/cloud-controller-manager 64 | - --cloud-provider={{.cloud_provider}} 65 | {{- if .cloud_config}} 66 | - --cloud-config=/etc/kubernetes/cloud/cloud.conf 67 | {{- end}} 68 | - --leader-elect=true 69 | - --use-service-account-credentials 70 | # these flags will vary for every cloud provider 71 | {{- if .cloud_provider_flags}} 72 | - {{.cloud_provider_flags}} 73 | {{- end}} 74 | {{- if .cloud_config}} 75 | volumeMounts: 76 | - name: cloud-provider-config 77 | mountPath: "/etc/kubernetes/cloud" 78 | {{- end}} 79 | 80 | tolerations: 81 | # this is required so CCM can bootstrap itself 82 | - key: node.cloudprovider.kubernetes.io/uninitialized 83 | value: "true" 84 | effect: NoSchedule 85 | # this is to have the daemonset runnable on master nodes 86 | # the taint may vary depending on your cluster setup 87 | - key: node-role.kubernetes.io/master 88 | effect: NoSchedule 89 | # this is to restrict CCM to only run on master nodes 90 | # the node selector may vary depending on your cluster setup 91 | nodeSelector: 92 | node-role.kubernetes.io/master: "" 93 | 94 | {{- if .cloud_config}} 95 | volumes: 96 | - name: cloud-provider-config 97 | secret: 98 | secretName: cloud-provider-config 99 | {{- end}} 100 | ` 101 | -------------------------------------------------------------------------------- /internal/assets/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package assets 16 | 17 | //go:generate ../../utils/generate.sh --out-var KubeadmSetupScriptCode --out-package assets --out-file generated_kubeadm_setup.go ./static/kubeadm-setup.sh 18 | //go:generate ../../utils/generate.sh --out-var KubeletSysconfigCode --out-package assets --out-file generated_kubelet_sysconfig.go ./static/kubelet.sysconfig 19 | //go:generate ../../utils/generate.sh --out-var KubeadmDropinCode --out-package assets --out-file generated_kubeadm_dropin.go ./static/kubeadm-dropin.conf 20 | //go:generate ../../utils/generate.sh --out-var KubeletServiceCode --out-package assets --out-file generated_kubelet_service.go ./static/service.conf 21 | //go:generate ../../utils/generate.sh --out-var CNIDefConfCode --out-package assets --out-file generated_cni_conf.go ./static/cni-default.conflist 22 | //go:generate ../../utils/generate.sh --out-var FlannelManifestCode --out-package assets --out-file generated_flannel_manifest.go ./static/kube-flannel.yml 23 | //go:generate ../../utils/generate.sh --out-var CloudProviderCode --out-package assets --out-file cloud_provider_manifest.go ./static/cloud-provider.yml 24 | //go:generate ../../utils/generate.sh --out-var WeaveManifestCode --out-package assets --out-file weave_manifest.go ./static/weave.yml 25 | -------------------------------------------------------------------------------- /internal/assets/generated_cni_conf.go: -------------------------------------------------------------------------------- 1 | // Code generated automatically with go generate; DO NOT EDIT. 2 | 3 | package assets 4 | 5 | const CNIDefConfCode = `{ 6 | "cniVersion": "0.2.0", 7 | "type": "loopback" 8 | } 9 | ` 10 | -------------------------------------------------------------------------------- /internal/assets/generated_kubeadm_dropin.go: -------------------------------------------------------------------------------- 1 | // Code generated automatically with go generate; DO NOT EDIT. 2 | 3 | package assets 4 | 5 | const KubeadmDropinCode = `# Note: This dropin only works with kubeadm and kubelet v1.11+ 6 | [Service] 7 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" 8 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 9 | 10 | # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically 11 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 12 | 13 | # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use 14 | # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. 15 | EnvironmentFile=-/etc/sysconfig/kubelet 16 | 17 | ExecStart= 18 | ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS 19 | ` 20 | -------------------------------------------------------------------------------- /internal/assets/generated_kubelet_service.go: -------------------------------------------------------------------------------- 1 | // Code generated automatically with go generate; DO NOT EDIT. 2 | 3 | package assets 4 | 5 | const KubeletServiceCode = `[Unit] 6 | Description=kubelet: The Kubernetes Node Agent 7 | Documentation=http://kubernetes.io/docs/ 8 | 9 | [Service] 10 | ExecStart=/usr/bin/kubelet 11 | Restart=always 12 | StartLimitInterval=0 13 | RestartSec=10 14 | 15 | [Install] 16 | WantedBy=multi-user.target 17 | ` 18 | -------------------------------------------------------------------------------- /internal/assets/generated_kubelet_sysconfig.go: -------------------------------------------------------------------------------- 1 | // Code generated automatically with go generate; DO NOT EDIT. 2 | 3 | package assets 4 | 5 | const KubeletSysconfigCode = `# kubelet extra configuration 6 | KUBELET_EXTRA_ARGS="--fail-swap-on=false"` 7 | -------------------------------------------------------------------------------- /internal/assets/static/cloud-provider.yml: -------------------------------------------------------------------------------- 1 | # from https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/ 2 | 3 | {{- if .cloud_config}} 4 | apiVersion: v1 5 | kind: Secret 6 | metadata: 7 | name: cloud-provider-config 8 | type: Opaque 9 | data: 10 | # "cloud_config" contains the Base64 encoded configuration file 11 | cloud.conf: {{.cloud_config}} 12 | {{- end}} 13 | 14 | --- 15 | apiVersion: v1 16 | kind: ServiceAccount 17 | metadata: 18 | name: cloud-controller-manager 19 | namespace: kube-system 20 | 21 | --- 22 | apiVersion: rbac.authorization.k8s.io/v1 23 | kind: ClusterRoleBinding 24 | metadata: 25 | name: system:cloud-controller-manager 26 | roleRef: 27 | apiGroup: rbac.authorization.k8s.io 28 | kind: ClusterRole 29 | name: cluster-admin 30 | subjects: 31 | - kind: ServiceAccount 32 | name: cloud-controller-manager 33 | namespace: kube-system 34 | 35 | --- 36 | apiVersion: apps/v1 37 | kind: DaemonSet 38 | metadata: 39 | labels: 40 | k8s-app: cloud-controller-manager 41 | name: cloud-controller-manager 42 | namespace: kube-system 43 | spec: 44 | selector: 45 | matchLabels: 46 | k8s-app: cloud-controller-manager 47 | template: 48 | metadata: 49 | labels: 50 | k8s-app: cloud-controller-manager 51 | spec: 52 | serviceAccountName: cloud-controller-manager 53 | containers: 54 | - name: cloud-controller-manager 55 | # for in-tree providers we use k8s.gcr.io/cloud-controller-manager 56 | # this can be replaced with any other image for out-of-tree providers 57 | image: k8s.gcr.io/cloud-controller-manager:v1.8.0 58 | command: 59 | - /usr/local/bin/cloud-controller-manager 60 | - --cloud-provider={{.cloud_provider}} 61 | {{- if .cloud_config}} 62 | - --cloud-config=/etc/kubernetes/cloud/cloud.conf 63 | {{- end}} 64 | - --leader-elect=true 65 | - --use-service-account-credentials 66 | # these flags will vary for every cloud provider 67 | {{- if .cloud_provider_flags}} 68 | - {{.cloud_provider_flags}} 69 | {{- end}} 70 | {{- if .cloud_config}} 71 | volumeMounts: 72 | - name: cloud-provider-config 73 | mountPath: "/etc/kubernetes/cloud" 74 | {{- end}} 75 | 76 | tolerations: 77 | # this is required so CCM can bootstrap itself 78 | - key: node.cloudprovider.kubernetes.io/uninitialized 79 | value: "true" 80 | effect: NoSchedule 81 | # this is to have the daemonset runnable on master nodes 82 | # the taint may vary depending on your cluster setup 83 | - key: node-role.kubernetes.io/master 84 | effect: NoSchedule 85 | # this is to restrict CCM to only run on master nodes 86 | # the node selector may vary depending on your cluster setup 87 | nodeSelector: 88 | node-role.kubernetes.io/master: "" 89 | 90 | {{- if .cloud_config}} 91 | volumes: 92 | - name: cloud-provider-config 93 | secret: 94 | secretName: cloud-provider-config 95 | {{- end}} 96 | -------------------------------------------------------------------------------- /internal/assets/static/cni-default.conflist: -------------------------------------------------------------------------------- 1 | { 2 | "cniVersion": "0.2.0", 3 | "type": "loopback" 4 | } 5 | -------------------------------------------------------------------------------- /internal/assets/static/kube-flannel.yml-url: -------------------------------------------------------------------------------- 1 | https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml 2 | -------------------------------------------------------------------------------- /internal/assets/static/kubeadm-dropin.conf: -------------------------------------------------------------------------------- 1 | # Note: This dropin only works with kubeadm and kubelet v1.11+ 2 | [Service] 3 | Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf" 4 | Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml" 5 | 6 | # This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically 7 | EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env 8 | 9 | # This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use 10 | # the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file. 11 | EnvironmentFile=-/etc/sysconfig/kubelet 12 | 13 | ExecStart= 14 | ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS 15 | -------------------------------------------------------------------------------- /internal/assets/static/kubelet.sysconfig: -------------------------------------------------------------------------------- 1 | # kubelet extra configuration 2 | KUBELET_EXTRA_ARGS="--fail-swap-on=false" -------------------------------------------------------------------------------- /internal/assets/static/service.conf: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=kubelet: The Kubernetes Node Agent 3 | Documentation=http://kubernetes.io/docs/ 4 | 5 | [Service] 6 | ExecStart=/usr/bin/kubelet 7 | Restart=always 8 | StartLimitInterval=0 9 | RestartSec=10 10 | 11 | [Install] 12 | WantedBy=multi-user.target 13 | -------------------------------------------------------------------------------- /internal/assets/static/weave.yml-url: -------------------------------------------------------------------------------- 1 | https://cloud.weave.works/k8s/net?k8s-version=1.15 2 | -------------------------------------------------------------------------------- /internal/ssh/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "strconv" 21 | ) 22 | 23 | const ( 24 | // environmental variable that can be used for disabling the cache 25 | cacheEnvVar = "TF_CACHE" 26 | ) 27 | 28 | const ( 29 | // CacheRemoteFileExistsPrefix is the prefix for file checks 30 | CacheRemoteFileExistsPrefix = "remote-file-exists" 31 | 32 | // CacheRemoteDirExistsPrefix is the prefix for dir checks 33 | CacheRemoteDirExistsPrefix = "remote-dir-exists" 34 | ) 35 | 36 | func isCacheDisabled() bool { 37 | enabledStr := os.Getenv(cacheEnvVar) 38 | if len(enabledStr) > 0 { 39 | enabled, _ := strconv.ParseBool(enabledStr) 40 | return !enabled 41 | } 42 | return false 43 | } 44 | 45 | // getFromCacheInContext gets a value from the cache 46 | func getFromCacheInContext(ctx context.Context, key string) (interface{}, bool) { 47 | if isCacheDisabled() { 48 | return nil, false 49 | } 50 | c := getCacheFromContext(ctx) 51 | value, ok := c[key] 52 | Debug("[CACHE] getting %q [found:%t] = %v ", key, ok, value) 53 | return value, ok 54 | } 55 | 56 | // setInCacheInContext sets a value in the cache 57 | func setInCacheInContext(ctx context.Context, key string, value interface{}) { 58 | if isCacheDisabled() { 59 | return 60 | } 61 | c := getCacheFromContext(ctx) 62 | Debug("[CACHE] setting %q = %v", key, value) 63 | c[key] = value 64 | } 65 | 66 | // delInCacheInContext removes akey in the cache 67 | func delInCacheInContext(ctx context.Context, key string) { 68 | if isCacheDisabled() { 69 | return 70 | } 71 | c := getCacheFromContext(ctx) 72 | Debug("[CACHE] deleting %q", key) 73 | delete(c, key) 74 | } 75 | 76 | // DoOnce runs an action if it is not been saved in the cache 77 | // If the action does not produce any error, it is saved in the cache under the `key`. 78 | // If the action produces an error, the error is returned and nothing is saved in the cache, 79 | // so any subsequent DoOnce for the same key will be executed 80 | func DoOnce(key string, action Action) Action { 81 | return DoIf( 82 | CheckNot(CheckInCache(key)), 83 | ActionFunc(func(ctx context.Context) Action { 84 | res := ActionList{action}.Apply(ctx) 85 | if !IsError(res) { 86 | // save in the cache only when no errors happen 87 | setInCacheInContext(ctx, key, true) 88 | } 89 | return res 90 | })) 91 | } 92 | 93 | // DoRemoveFromCache removes some key from the cache 94 | func DoRemoveFromCache(key string) Action { 95 | return ActionFunc(func(ctx context.Context) Action { 96 | delInCacheInContext(ctx, key) 97 | return nil 98 | }) 99 | } 100 | 101 | // DoSetInCache sets some key in the cache 102 | func DoSetInCache(key string, value interface{}) Action { 103 | return ActionFunc(func(ctx context.Context) Action { 104 | setInCacheInContext(ctx, key, value) 105 | return nil 106 | }) 107 | } 108 | 109 | // DoFlushCache flushes the cache 110 | func DoFlushCache() Action { 111 | return ActionFunc(func(ctx context.Context) Action { 112 | if isCacheDisabled() { 113 | return nil 114 | } 115 | sctx := getSSHContext(ctx) 116 | sctx.cache = cache{} 117 | return nil 118 | }) 119 | } 120 | 121 | ///////////////////////////////////////////////////////////////////////////// 122 | // Checks 123 | ///////////////////////////////////////////////////////////////////////////// 124 | 125 | // CheckInCache returns true if the key is in the cache 126 | func CheckInCache(key string) CheckerFunc { 127 | return CheckerFunc(func(ctx context.Context) (bool, error) { 128 | _, ok := getFromCacheInContext(ctx, key) 129 | if ok { 130 | return true, nil 131 | } 132 | return false, nil 133 | }) 134 | } 135 | 136 | // CheckOnce checks if there is a cached result for the `key`. If not, 137 | // runs the check, storing the result in the cache 138 | func CheckOnce(key string, check Checker) CheckerFunc { 139 | return CheckerFunc(func(ctx context.Context) (bool, error) { 140 | value, ok := getFromCacheInContext(ctx, key) 141 | if ok { 142 | return value.(bool), nil 143 | } 144 | 145 | res, err := check.Check(ctx) 146 | if err != nil { 147 | return false, err 148 | } 149 | 150 | setInCacheInContext(ctx, key, res) 151 | return res, nil 152 | }) 153 | } 154 | -------------------------------------------------------------------------------- /internal/ssh/cache_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "testing" 21 | ) 22 | 23 | func TestIsCacheDisabled(t *testing.T) { 24 | prevValue := os.Getenv(cacheEnvVar) 25 | 26 | os.Setenv(cacheEnvVar, "1") 27 | if isCacheDisabled() { 28 | t.Fatalf("Error: cache is disabled with %s = 1", cacheEnvVar) 29 | } 30 | 31 | os.Setenv(cacheEnvVar, "0") 32 | if !isCacheDisabled() { 33 | t.Fatalf("Error: cache is not disabled with %s = 0", cacheEnvVar) 34 | } 35 | 36 | os.Setenv(cacheEnvVar, prevValue) 37 | } 38 | 39 | func TestCacheBasic(t *testing.T) { 40 | if isCacheDisabled() { 41 | t.Skip("DoOnce not tested: cache is disabled.") 42 | return 43 | } 44 | 45 | ctx := NewTestingContext() 46 | 47 | t.Log("Setting some value in the cache...") 48 | actions := ActionList{ 49 | DoSetInCache("test", "value"), 50 | DoSetInCache("thor", "value"), 51 | DoSetInCache("loki", "value"), 52 | } 53 | res := actions.Apply(ctx) 54 | if IsError(res) { 55 | t.Fatalf("Error: error detected: %s", res) 56 | } 57 | 58 | count := 0 59 | inc := ActionFunc(func(context.Context) Action { 60 | t.Log("incrementing the counter...") 61 | count++ 62 | return nil 63 | }) 64 | 65 | t.Log("Checking that value is in the cache...") 66 | actions = ActionList{ 67 | DoIf(CheckInCache("test"), inc), 68 | DoIf(CheckInCache("foo"), inc), 69 | DoIf(CheckInCache("bar"), inc), 70 | } 71 | res = actions.Apply(ctx) 72 | if IsError(res) { 73 | t.Fatalf("Error: error detected: %s", res) 74 | } 75 | if count != 1 { 76 | t.Fatalf("Error: unexpected value in counter: %d, expected: %d", count, 1) 77 | } 78 | 79 | t.Log("Flushing the cache and checking again...") 80 | count = 0 81 | actions = ActionList{ 82 | DoFlushCache(), 83 | DoIf(CheckInCache("test"), inc), 84 | DoIf(CheckInCache("thor"), inc), 85 | DoIf(CheckInCache("loki"), inc), 86 | } 87 | res = actions.Apply(ctx) 88 | if IsError(res) { 89 | t.Fatalf("Error: error detected: %s", res) 90 | } 91 | if count != 0 { 92 | t.Fatalf("Error: unexpected value in counter: %d, expected: %d", count, 0) 93 | } 94 | } 95 | 96 | func TestDoOnce(t *testing.T) { 97 | if isCacheDisabled() { 98 | t.Skip("DoOnce not tested: cache is disabled.") 99 | return 100 | } 101 | 102 | count := 0 103 | inc := ActionFunc(func(context.Context) Action { 104 | t.Log("incrementing the counter...") 105 | count++ 106 | return nil 107 | }) 108 | 109 | actions := ActionList{ 110 | DoOnce("increment", inc), 111 | DoOnce("increment", inc), 112 | DoOnce("increment", inc), 113 | DoOnce("increment", inc), 114 | } 115 | 116 | ctx := NewTestingContext() 117 | res := actions.Apply(ctx) 118 | if IsError(res) { 119 | t.Fatalf("Error: error detected: %s", res) 120 | } 121 | if count != 1 { 122 | t.Fatalf("Error: unexpected number of increments: %d, expected: %d", count, 1) 123 | } 124 | } 125 | 126 | func TestDoOnceWithError(t *testing.T) { 127 | if isCacheDisabled() { 128 | t.Skip("DoOnceWithError not tested: cache is disabled ") 129 | return 130 | } 131 | 132 | count := 0 133 | failed := 0 134 | 135 | inc := ActionFunc(func(context.Context) Action { 136 | t.Log("incrementing the counter...") 137 | count++ 138 | return nil 139 | }) 140 | 141 | incFailed := ActionFunc(func(context.Context) Action { 142 | t.Log("returning an error") 143 | failed++ 144 | return ActionError("failed to increase the counter") 145 | }) 146 | 147 | ctx := NewTestingContext() 148 | 149 | actions := ActionList{ 150 | // failed actions do not store anything on the cache 151 | DoOnce("increment", incFailed), 152 | // ... thihs shoult not be run: the previous error was returned 153 | DoOnce("increment", inc), 154 | } 155 | res := actions.Apply(ctx) 156 | if !IsError(res) { 157 | t.Fatalf("Error: no error detected: %s", res) 158 | } 159 | if failed != 1 { 160 | t.Fatalf("Error: unexpected number of failed: %d, expected: %d", failed, 1) 161 | } 162 | if count != 0 { 163 | t.Fatalf("Error: unexpected number of increments: %d, expected: %d", count, 0) 164 | } 165 | 166 | count = 0 167 | failed = 0 168 | actions = ActionList{ 169 | // ... then we run once 170 | DoOnce("increment", inc), 171 | // ... and these actions should not be run 172 | DoOnce("increment", inc), 173 | DoOnce("increment", inc), 174 | } 175 | res = actions.Apply(ctx) 176 | if IsError(res) { 177 | t.Fatalf("Error: error detected: %s", res) 178 | } 179 | if count != 1 { 180 | t.Fatalf("Error: unexpected number of increments: %d, expected: %d", count, 1) 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /internal/ssh/commands_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestCheckBinaryExists(t *testing.T) { 22 | responses := []string{ 23 | " /usr/bin/kubeadm\r ", 24 | "CONDITION_SUCCEEDED", 25 | } 26 | 27 | ctx := NewTestingContextWithResponses(responses) 28 | exists, err := CheckBinaryExists("kubeadm").Check(ctx) 29 | if err != nil { 30 | t.Fatalf("Error: %s", err) 31 | } 32 | if !exists { 33 | t.Fatalf("Error: unexpected result for exists: %t", exists) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /internal/ssh/context.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/hashicorp/terraform/communicator" 21 | ) 22 | 23 | const ( 24 | sshContextKey = contextKey("ssh") 25 | ) 26 | 27 | // UIOutput is the interface that must be implemented to output 28 | // data to the end user. 29 | type UIOutput interface { 30 | Output(string) 31 | } 32 | 33 | type OutputFunc func(s string) 34 | 35 | func (f OutputFunc) Output(s string) { f(s) } 36 | 37 | /////////////////////////////////////////////////////////////////////////////////////////////// 38 | 39 | type cache map[string]interface{} 40 | 41 | /////////////////////////////////////////////////////////////////////////////////////////////// 42 | 43 | // sshContext is the "internal" context we pass around 44 | type sshContext struct { 45 | useSudo bool 46 | userOutput UIOutput 47 | execOutput UIOutput 48 | comm communicator.Communicator 49 | cache cache 50 | leftovers []string 51 | } 52 | 53 | // WithValues creates a new "internal" SSH context 54 | func WithValues(ctx context.Context, userOutput UIOutput, execOutput UIOutput, comm communicator.Communicator, useSudo bool) context.Context { 55 | return context.WithValue(ctx, sshContextKey, &sshContext{ 56 | useSudo: useSudo, 57 | userOutput: userOutput, 58 | execOutput: execOutput, 59 | comm: comm, 60 | cache: cache{}, 61 | leftovers: []string{}, 62 | }) 63 | } 64 | 65 | func getSSHContext(ctx context.Context) *sshContext { 66 | sshc, ok := ctx.Value(sshContextKey).(*sshContext) 67 | if !ok { 68 | panic("could not get SSH context info info from context") 69 | } 70 | return sshc 71 | } 72 | 73 | // GetUseSudoFromContext gets the "should we use sudo?" value 74 | func GetUseSudoFromContext(ctx context.Context) bool { 75 | return getSSHContext(ctx).useSudo 76 | } 77 | 78 | // GetUserOutputFromContext gets the user output 79 | func GetUserOutputFromContext(ctx context.Context) UIOutput { 80 | return getSSHContext(ctx).userOutput 81 | } 82 | 83 | // GetExecOutputFromContext gets the exec output from the current context 84 | func GetExecOutputFromContext(ctx context.Context) UIOutput { 85 | return getSSHContext(ctx).execOutput 86 | } 87 | 88 | // GetCommFromContext gets the communicator from the current context 89 | func GetCommFromContext(ctx context.Context) communicator.Communicator { 90 | return getSSHContext(ctx).comm 91 | } 92 | 93 | // getCacheFromContext gets the cache from the current context 94 | func getCacheFromContext(ctx context.Context) cache { 95 | return getSSHContext(ctx).cache 96 | } 97 | -------------------------------------------------------------------------------- /internal/ssh/context_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | -------------------------------------------------------------------------------- /internal/ssh/dirs.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | // DoMkdir creates a remote directory 22 | func DoMkdir(path string) Action { 23 | mkdirCmd := fmt.Sprintf("mkdir -p %s", path) 24 | return ActionList{ 25 | DoMessageDebug(fmt.Sprintf("Making sure directory %q exists", path)), 26 | DoExec(mkdirCmd), 27 | } 28 | } 29 | 30 | // DoMkdirOnce creates a remote directory (only once). 31 | // We don't really delete any remote directory, so running a `mkdir` for a 32 | // remote directory only once can be considered safe. 33 | func DoMkdirOnce(dir string) Action { 34 | return DoOnce( 35 | CacheRemoteDirExistsPrefix+"-"+dir, 36 | DoMkdir(dir)) 37 | } 38 | 39 | // CheckDirExists checks that a directory exists 40 | func CheckDirExists(path string) CheckerFunc { 41 | return CheckExec(fmt.Sprintf("[ -d '%s' ]", path)) 42 | } 43 | -------------------------------------------------------------------------------- /internal/ssh/docker.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "errors" 21 | "fmt" 22 | "strings" 23 | ) 24 | 25 | const ( 26 | // docker command for getting a container id 27 | dockerGetContainer = "docker ps --filter name=^/%s -q" 28 | ) 29 | 30 | var ( 31 | // ErrContainerNotFound is the container has not been found 32 | ErrContainerNotFound = errors.New("container not found") 33 | ) 34 | 35 | // GetContainer returns the ID of a container 36 | func GetContainer(ctx context.Context, pattern string) (string, error) { 37 | 38 | cmd := fmt.Sprintf(dockerGetContainer, pattern) 39 | var buf bytes.Buffer 40 | if err := DoSendingExecOutputToWriter(DoExec(cmd), &buf).Apply(ctx); IsError(err) { 41 | return "", err 42 | } 43 | 44 | output := buf.String() 45 | if len(output) == 0 { 46 | return "", ErrContainerNotFound 47 | } 48 | 49 | output = strings.ReplaceAll(output, "\n", "") 50 | output = strings.TrimSpace(output) 51 | 52 | Debug("GetContainer(%s) output: %q", pattern, output) 53 | return output, nil 54 | } 55 | 56 | // DoDockerExec runs a `docker exec` command in a container 57 | func DoDockerExec(pattern string, command string) Action { 58 | return ActionFunc(func(ctx context.Context) Action { 59 | cid, err := GetContainer(ctx, pattern) 60 | if err != nil { 61 | return ActionError(err.Error()) 62 | } 63 | 64 | // build the full `docker exec` command to run 65 | dockerCommand := fmt.Sprintf("docker exec -ti '%s' /bin/sh -c '%s'", cid, command) 66 | 67 | Debug("Running command in container %q: '%s'", cid, dockerCommand) 68 | return DoExec(dockerCommand) 69 | }) 70 | } 71 | 72 | // CheckContainerRunning checks if we can get the CID for a pattern 73 | func CheckContainerRunning(pattern string) CheckerFunc { 74 | return CheckerFunc(func(ctx context.Context) (bool, error) { 75 | cid, err := GetContainer(ctx, pattern) 76 | if err != nil { 77 | return false, nil 78 | } 79 | if cid == "" { 80 | return false, nil 81 | } 82 | return true, nil 83 | }) 84 | } 85 | -------------------------------------------------------------------------------- /internal/ssh/files_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "os" 19 | "testing" 20 | ) 21 | 22 | func TestTempFilenames(t *testing.T) { 23 | name1, err := GetTempFilename() 24 | if err != nil { 25 | t.Fatalf("Error: %s", err) 26 | } 27 | name2, err := GetTempFilename() 28 | if err != nil { 29 | t.Fatalf("Error: %s", err) 30 | } 31 | names := []struct { 32 | name string 33 | res bool 34 | }{ 35 | { 36 | name: name1, 37 | res: true, 38 | }, 39 | { 40 | name: name2, 41 | res: true, 42 | }, 43 | { 44 | name: "/tmp/something.tmp", 45 | res: false, 46 | }, 47 | } 48 | 49 | for _, testCase := range names { 50 | isTemp := IsTempFilename(testCase.name) 51 | if isTemp != testCase.res { 52 | t.Fatalf("Error: %q detected as temp=%t when we expected temp=%t", testCase.name, isTemp, testCase.res) 53 | } 54 | } 55 | } 56 | 57 | func TestCheckLocalFileExists(t *testing.T) { 58 | ctx := NewTestingContext() 59 | 60 | name1, err := GetTempFilename() 61 | if err != nil { 62 | t.Fatalf("Error: %s", err) 63 | } 64 | defer func() { 65 | DoTry(DoDeleteLocalFile(name1)).Apply(ctx) 66 | }() 67 | 68 | f, err := os.Create(name1) 69 | if err != nil { 70 | t.Fatalf("Error: %s", err) 71 | } 72 | _, err = f.Write([]byte("something")) 73 | if err != nil { 74 | t.Fatalf("Error: %s", err) 75 | } 76 | 77 | exists, err := CheckLocalFileExists(name1).Check(ctx) 78 | if err != nil { 79 | t.Fatalf("Error: %s", err) 80 | } 81 | if !exists { 82 | t.Fatalf("Error: unexpected result for exists: %t", exists) 83 | } 84 | } 85 | 86 | func TestCheckFileExists(t *testing.T) { 87 | ctx := NewTestingContext() 88 | name1, err := GetTempFilename() 89 | if err != nil { 90 | t.Fatalf("Error: %s", err) 91 | } 92 | 93 | // return a CONDITION_SUCCEEDED 94 | ctx = NewTestingContextWithResponses([]string{"CONDITION_SUCCEEDED"}) 95 | exists, err := CheckFileExists(name1).Check(ctx) 96 | if err != nil { 97 | t.Fatalf("Error: %s", err) 98 | } 99 | if !exists { 100 | t.Fatalf("Error: unexpected result for exists: %t", exists) 101 | } 102 | } 103 | 104 | func TestDoUploadReaderToFile(t *testing.T) { 105 | ctx, uploads := NewTestingContextForUploads([]string{}) 106 | 107 | dst := "/tmp/something.txt" 108 | s := "this is a test" 109 | 110 | actions := ActionList{ 111 | DoUploadBytesToFile([]byte(s), dst), 112 | } 113 | if res := actions.Apply(ctx); IsError(res) { 114 | t.Fatalf("Error: when running actions: %s", res) 115 | } 116 | if len(*uploads) == 0 { 117 | t.Fatalf("Error: upload not found in %+v", uploads) 118 | } 119 | } 120 | 121 | func TestLeftovers(t *testing.T) { 122 | ctx := NewTestingContextWithResponses([]string{}) 123 | 124 | actions := ActionList{ 125 | DoAddLeftover("/tmp/test1"), 126 | DoAddLeftover("/tmp/test2"), 127 | DoCleanupLeftovers(), 128 | } 129 | if res := actions.Apply(ctx); IsError(res) { 130 | t.Fatalf("Error: when running actions: %s", res) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /internal/ssh/net.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "regexp" 19 | ) 20 | 21 | // AllMatchesIPv4 return all matches of IPs in a string 22 | func AllMatchesIPv4(s string) (ips []string) { 23 | re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) 24 | submatchall := re.FindAllString(s, -1) 25 | for _, ip := range submatchall { 26 | if ip == "127.0.0.1" { 27 | continue 28 | } 29 | ips = append(ips, ip) 30 | } 31 | return 32 | } 33 | -------------------------------------------------------------------------------- /internal/ssh/net_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestAllMatchesIPv4(t *testing.T) { 22 | str1 := `Proxy Port Last Check Proxy Speed Proxy Country Anonymity 118.99.81.204 23 | 118.99.81.204 8080 34 sec Indonesia - Tangerang Transparent 2.184.31.2 8080 58 sec 24 | Iran Transparent 93.126.11.189 8080 1 min Iran - Esfahan Transparent 202.118.236.130 25 | 7777 1 min China - Harbin Transparent 62.201.207.9 8080 1 min Iraq Transparent` 26 | 27 | str1Addresseses := []string{ 28 | "118.99.81.204", 29 | "118.99.81.204", 30 | "2.184.31.2", 31 | "93.126.11.189", 32 | "202.118.236.130", 33 | "62.201.207.9", 34 | } 35 | 36 | inList := func(lst []string, target string) bool { 37 | for _, v := range lst { 38 | if v == target { 39 | return true 40 | } 41 | } 42 | return false 43 | } 44 | 45 | ips := AllMatchesIPv4(str1) 46 | for _, expected := range str1Addresseses { 47 | if !inList(ips, expected) { 48 | t.Fatalf("Error: %q is not in the result", expected) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /internal/ssh/processes.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "fmt" 19 | ) 20 | 21 | // CheckProcessRunning checks that a process is running with the help of `ps` 22 | // FIXME: this is not really reliable, as it looks for a string in tne output 23 | // of `ps`, and that string can be part of some other command... 24 | func CheckProcessRunning(process string) CheckerFunc { 25 | check := fmt.Sprintf(`[ -n "$(ps ax | grep %s | grep -v grep)" ]`, process) 26 | return CheckExec(check) 27 | } 28 | 29 | // DoRestartService restart a systemctl service 30 | func DoRestartService(service string) Action { 31 | return ActionList{ 32 | DoMessageInfo(fmt.Sprintf("Restarting service %s", service)), 33 | DoExec(fmt.Sprintf("systemctl --no-pager restart '%s'", service)), 34 | } 35 | } 36 | 37 | // DoEnableService enables a systemctl service 38 | func DoEnableService(service string) Action { 39 | return ActionList{ 40 | DoMessageInfo(fmt.Sprintf("Enabling service %s", service)), 41 | DoExec(fmt.Sprintf("systemctl --no-pager enable '%s'", service)), 42 | } 43 | } 44 | 45 | // CheckServiceExists checks that service exists 46 | func CheckServiceExists(service string) CheckerFunc { 47 | Debug("will check if service '%s' exists", service) 48 | exists := fmt.Sprintf("systemctl --no-pager status '%s' 2>/dev/null", service) 49 | return CheckExec(exists) 50 | } 51 | 52 | // CheckServiceActive checks that service exists and is active 53 | func CheckServiceActive(service string) CheckerFunc { 54 | inactive := fmt.Sprintf("systemctl --no-pager status '%s' 2>/dev/null | grep Active | grep -q inactive", service) 55 | return CheckNot( 56 | CheckAnd(CheckServiceExists(service), 57 | CheckExec(inactive))) 58 | } 59 | -------------------------------------------------------------------------------- /internal/ssh/test_helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "io" 21 | "io/ioutil" 22 | "time" 23 | 24 | "github.com/hashicorp/terraform/communicator" 25 | "github.com/hashicorp/terraform/communicator/remote" 26 | "github.com/hashicorp/terraform/terraform" 27 | ) 28 | 29 | type DummyOutput struct{} 30 | 31 | func (_ DummyOutput) Output(s string) { 32 | fmt.Print(s) 33 | } 34 | 35 | type DummyCommunicator struct { 36 | } 37 | 38 | func (_ DummyCommunicator) Connect(terraform.UIOutput) error { 39 | Debug("DummyCommunicator: Connect()") 40 | return nil 41 | } 42 | 43 | func (_ DummyCommunicator) Disconnect() error { 44 | Debug("DummyCommunicator: Disconnect()") 45 | return nil 46 | } 47 | 48 | func (_ DummyCommunicator) Timeout() time.Duration { 49 | Debug("DummyCommunicator: Timeout()") 50 | return 1 * time.Hour 51 | } 52 | 53 | func (_ DummyCommunicator) ScriptPath() string { 54 | Debug("DummyCommunicator: ScriptPath()") 55 | return "" 56 | } 57 | 58 | func (dc DummyCommunicator) Start(cmd *remote.Cmd) error { 59 | Debug("DummyCommunicator: Start(%s)", cmd.Command) 60 | return nil 61 | } 62 | 63 | func (_ DummyCommunicator) Upload(string, io.Reader) error { 64 | Debug("DummyCommunicator: Upload()") 65 | return nil 66 | } 67 | 68 | func (_ DummyCommunicator) UploadScript(string, io.Reader) error { 69 | Debug("DummyCommunicator: UploadScript()") 70 | return nil 71 | } 72 | 73 | func (_ DummyCommunicator) UploadDir(string, string) error { 74 | Debug("DummyCommunicator: UploadDir()") 75 | return nil 76 | } 77 | 78 | //////////////////////////////////////////////////////////////////////////// 79 | 80 | func NewTestingContextWithCommunicator(comm communicator.Communicator) context.Context { 81 | ctx := context.Background() 82 | out := DummyOutput{} 83 | return WithValues(ctx, out, out, comm, false) 84 | } 85 | 86 | func NewTestingContext() context.Context { 87 | return NewTestingContextWithCommunicator(DummyCommunicator{}) 88 | } 89 | 90 | type dummyCommunicatorWithResponses struct { 91 | DummyCommunicator 92 | 93 | responses []string 94 | counter *int 95 | uploads *(map[string]string) 96 | } 97 | 98 | func (dc dummyCommunicatorWithResponses) Start(cmd *remote.Cmd) error { 99 | cmd.Init() 100 | if (*dc.counter) >= len(dc.responses) { 101 | cmd.Stdout.Write([]byte("")) 102 | } else { 103 | cmd.Stdout.Write([]byte(dc.responses[*dc.counter])) 104 | } 105 | cmd.SetExitStatus(0, nil) 106 | *dc.counter++ 107 | return nil 108 | } 109 | 110 | func (dc dummyCommunicatorWithResponses) Upload(dst string, r io.Reader) error { 111 | all, _ := ioutil.ReadAll(r) 112 | (*dc.uploads)[dst] = string(all) 113 | return nil 114 | } 115 | 116 | func (dc dummyCommunicatorWithResponses) UploadScript(dst string, r io.Reader) error { 117 | all, _ := ioutil.ReadAll(r) 118 | (*dc.uploads)[dst] = string(all) 119 | return nil 120 | } 121 | 122 | func (_ dummyCommunicatorWithResponses) UploadDir(string, string) error { 123 | return nil 124 | } 125 | 126 | func NewTestingContextWithResponses(responses []string) context.Context { 127 | // we must keep the "counter" out of the communicator object as this 128 | // object is inmutable... :-/ 129 | counter := 0 130 | comm := dummyCommunicatorWithResponses{ 131 | responses: responses, 132 | counter: &counter, 133 | } 134 | return NewTestingContextWithCommunicator(comm) 135 | } 136 | 137 | // NewTestingContextForUploads creates a new context prepared 138 | // for doing fake uploads 139 | func NewTestingContextForUploads(responses []string) (context.Context, *map[string]string) { 140 | // we must keep the "counter" out of the communicator object as this 141 | // object is inmutable... :-/ 142 | uploads := map[string]string{} 143 | counter := 0 144 | comm := dummyCommunicatorWithResponses{ 145 | responses: responses, 146 | counter: &counter, 147 | uploads: &uploads, 148 | } 149 | return NewTestingContextWithCommunicator(comm), &uploads 150 | } 151 | -------------------------------------------------------------------------------- /internal/ssh/text.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ssh 16 | 17 | import ( 18 | "bytes" 19 | "text/template" 20 | ) 21 | 22 | // ReplaceInTemplate performs replacements in an input text 23 | func ReplaceInTemplate(text string, replacements map[string]interface{}) (string, error) { 24 | tmpl, err := template.New("template").Parse(text) 25 | if err != nil { 26 | return "", err 27 | } 28 | 29 | b := bytes.Buffer{} 30 | if err := tmpl.Execute(&b, replacements); err != nil { 31 | return "", err 32 | } 33 | return b.String(), nil 34 | } 35 | -------------------------------------------------------------------------------- /packaging/suse/make_spec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z "$1" ]; then 4 | cat < ${NAME}.spec 21 | # 22 | # spec file for package $NAME 23 | # 24 | # Copyright (c) $YEAR Alvaro Saurin 25 | # 26 | # All modifications and additions to the file contributed by third parties 27 | # remain the property of their copyright owners, unless otherwise agreed 28 | # upon. The license for this file, and modifications and additions to the 29 | # file, is the same license as for the pristine package itself (unless the 30 | # license for the pristine package is not an Open Source License, in which 31 | # case the license is the MIT License). An "Open Source License" is a 32 | # license that conforms to the Open Source Definition (Version 1.9) 33 | # published by the Open Source Initiative. 34 | 35 | # Please submit bugfixes or comments via http://bugs.opensuse.org/ 36 | # 37 | 38 | # Make sure that the binary is not getting stripped. 39 | %{go_nostrip} 40 | 41 | Name: $NAME 42 | Version: $VERSION 43 | Release: 0 44 | License: MPL-2.0 45 | Summary: Experimental Terraform plugin for kubeadm 46 | Url: https://github.com/inercia/terraform-kubeadm/ 47 | Group: System/Management 48 | Source: %{name}-%{version}.tar.xz 49 | BuildRoot: %{_tmppath}/%{name}-%{version}-build 50 | 51 | BuildRequires: golang-packaging 52 | BuildRequires: libvirt-devel 53 | BuildRequires: xz 54 | BuildRequires: go >= 1.6 55 | 56 | Requires: terraform >= 0.8.5 57 | Requires: genisoimage 58 | %{go_provides} 59 | 60 | %description 61 | Terraform plugin for using kubeadm for creating kubernetes clusters. 62 | 63 | %prep 64 | %setup -q -n %{name}-%{version} 65 | 66 | %build 67 | %goprep github.com/inercia/terraform-kubeadm 68 | echo ">>>> making" 69 | export GOPATH=%{_builddir}/go 70 | export GOBIN=$GOPATH/bin 71 | make -C $GOPATH/src/github.com/inercia/terraform-kubeadm 72 | 73 | %install 74 | 75 | echo ">>>> installing" 76 | install -m 755 -d %{buildroot}%{_bindir} 77 | install -p -m 755 -t %{buildroot}%{_bindir} %{_builddir}/go/bin/terraform-{provider,provisioner}-kubeadm 78 | 79 | rm -rf %{buildroot}/%{_libdir}/go/contrib 80 | 81 | %files 82 | %defattr(-,root,root,-) 83 | %doc README.md 84 | %{_bindir}/terraform-{provider,provisioner}-kubeadm 85 | 86 | %changelog 87 | EOF 88 | -------------------------------------------------------------------------------- /pkg/common/certs_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/davecgh/go-spew/spew" 22 | ) 23 | 24 | func TestCertsSerialization(t *testing.T) { 25 | etcdCrtContents := "1234567890" 26 | 27 | certsMap := map[string]interface{}{ 28 | "ca_crt": "-- BEGIN PUBLIC KEY ---\n SOME-CERT ...", 29 | "ca_key": "-- BEGIN PRIVATE KEY ---\n SOME-KEY ...", 30 | "sa_crt": "-- BEGIN PUBLIC KEY ---\n SOME-CERT ...", 31 | "sa_key": "-- BEGIN PRIVATE KEY ---\n SOME-KEY ...", 32 | "etcd_crt": etcdCrtContents, 33 | "etcd_key": "-- BEGIN PRIVATE KEY ---\n SOME-KEY ...", 34 | "proxy_crt": "-- BEGIN PUBLIC KEY ---\n SOME-CERT ...", 35 | "proxy_key": "-- BEGIN PRIVATE KEY ---\n SOME-KEY ...", 36 | } 37 | 38 | certsConfig := CertsConfig{} 39 | if certsConfig.HasAllCertificates() { 40 | t.Fatalf("Error: certsConfig seems to be filled") 41 | } 42 | 43 | err := certsConfig.FromMap(certsMap) 44 | if err != nil { 45 | t.Fatalf("Error: %v", err) 46 | } 47 | fmt.Printf("certs config object:\n%s", spew.Sdump(certsConfig)) 48 | 49 | if !certsConfig.HasAllCertificates() { 50 | t.Fatalf("Error: certsConfig seems to be empty") 51 | } 52 | 53 | certsMap2, err := certsConfig.ToMap() 54 | if err != nil { 55 | t.Fatalf("Error: %v", err) 56 | } 57 | fmt.Printf("certs config map:\n%s", spew.Sdump(certsMap2)) 58 | certContents, ok := certsMap2["etcd_crt"] 59 | if !ok { 60 | t.Fatalf("Error: etcd_crt not in map") 61 | } 62 | if certContents != etcdCrtContents { 63 | t.Fatalf("Error: etcd_crt does not match") 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/common/common_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import "github.com/davecgh/go-spew/spew" 18 | 19 | func init() { 20 | spew.Config.Indent = "\t" 21 | } 22 | -------------------------------------------------------------------------------- /pkg/common/constants.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "github.com/inercia/terraform-provider-kubeadm/internal/assets" 19 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 20 | ) 21 | 22 | const ( 23 | DefPodCIDR = "10.244.0.0/16" 24 | 25 | DefServiceCIDR = "10.96.0.0/12" 26 | 27 | // kubernetes version to deploy 28 | // notes: * leaving it empty leads to some instabililties 29 | // * 1.15.1 seems to be broken (some etcd problems) 30 | DefKubernetesVersion = "v1.15.0" 31 | 32 | DefDNSDomain = "cluster.local" 33 | 34 | DefRuntimeEngine = "docker" 35 | 36 | DefKubeadmInitConfPath = "/etc/kubernetes/kubeadm-init.conf" 37 | 38 | DefKubeadmJoinConfPath = "/etc/kubernetes/kubeadm-join.conf" 39 | 40 | DefCniConfDir = "/etc/cni/net.d" 41 | 42 | DefCniLookbackConfPath = "/etc/cni/net.d/99-loopback.conf" 43 | 44 | DefCniBinDir = "/opt/cni/bin" 45 | 46 | DefFlannelBackend = "vxlan" 47 | 48 | DefFlannelImageVersion = "v0.11.0" 49 | 50 | // Full path where we should upload the kubelet sysconfig file 51 | DefKubeletSysconfigPath = "/etc/sysconfig/kubelet" 52 | 53 | // Full path where we should upload the kubelet.service file 54 | DefKubeletServicePath = "/usr/lib/systemd/system/kubelet.service" 55 | 56 | // Full path where we should upload the kubeadm dropin file 57 | DefKubeadmDropinPath = "/usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf" 58 | 59 | // Default PKI dir 60 | DefPKIDir = "/etc/kubernetes/pki" 61 | 62 | DefAPIServerPort = 6443 63 | 64 | // manifest for loading the dashboard 65 | DefDashboardManifest = "https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml" 66 | 67 | // kubeadm executable in the machines (we assume it is in some standard path) 68 | DefKubeadmPath = "kubeadm" 69 | 70 | // kubectl executable in the machines (we assume it is in some standard path) 71 | DefKubectlPath = "kubectl" 72 | 73 | // resolv.conf for pods when upstream servers are provided 74 | DefResolvUpstreamConf = "/etc/resolv.conf-kubeadm" 75 | ) 76 | 77 | var ( 78 | // CNIPluginsManifestsTemplates is the map of manifests for different CNI drivers 79 | CNIPluginsManifestsTemplates = map[string]ssh.Manifest{ 80 | "flannel": {Inline: assets.FlannelManifestCode}, 81 | "weave": {Inline: assets.WeaveManifestCode}, 82 | } 83 | 84 | // CNIPluginsList gets the list of supported CNI plugins (will be filled by the init()) 85 | CNIPluginsList = []string{} 86 | ) 87 | 88 | var ( 89 | // DefaultCriSocket info 90 | DefCriSocket = map[string]string{ 91 | "docker": "/var/run/dockershim.sock", 92 | "crio": "/var/run/crio/crio.sock", 93 | "containerd": "/var/run/containerd/containerd.sock", 94 | } 95 | 96 | DefIgnorePreflightChecks = []string{ 97 | "NumCPU", 98 | "FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", 99 | "Swap", 100 | "FileExisting-crictl", 101 | "Port-10250", 102 | "SystemVerification", // for ignoring docker graph=btrfs 103 | "IsPrivilegedUser", 104 | "NumCPU", // we will not always have >=2 CPUs in our VMs 105 | } 106 | 107 | DefKubeletSettings = map[string]string{ 108 | "network-plugin": "cni", 109 | } 110 | ) 111 | 112 | // cloud-provider configuration and constants 113 | var ( 114 | // DefSupportedCloudProviders is the list of Cloud Providers supported 115 | DefSupportedCloudProviders = []string{ 116 | "aws", 117 | "azure", 118 | "cloudstack", 119 | "gce", 120 | "openstack", 121 | "ovirt", 122 | "photon", 123 | "vsphere", 124 | } 125 | 126 | // DefCloudConfigMandatory is the list of Cloud Providers where the cloud-config is mandatory 127 | DefCloudConfigMandatory = []string{ 128 | "openstack", 129 | } 130 | 131 | // DefCloudConfigFilename is the default cloud config inn the nodes 132 | DefCloudConfigFilename = "/etc/kubernetes/cloud.conf" 133 | ) 134 | 135 | func init() { 136 | for k := range CNIPluginsManifestsTemplates { 137 | CNIPluginsList = append(CNIPluginsList, k) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/common/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // 16 | // "common" package holds shared code and types. 17 | // 18 | package common 19 | -------------------------------------------------------------------------------- /pkg/common/encoding.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "encoding/base64" 19 | ) 20 | 21 | // ToTerraformSafeString converts some (possibly binary) data to a 22 | // string that can be stored in the Terraform state 23 | func ToTerraformSafeString(data []byte) string { 24 | return base64.URLEncoding.EncodeToString(data) 25 | } 26 | 27 | // FromTerraformSafeString converts some Terraform state data to 28 | // (possibly binary) data. 29 | func FromTerraformSafeString(data string) ([]byte, error) { 30 | return base64.URLEncoding.DecodeString(data) 31 | } 32 | -------------------------------------------------------------------------------- /pkg/common/file.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "io/ioutil" 19 | "net/url" 20 | "strings" 21 | ) 22 | 23 | const ( 24 | isURL = iota 25 | isFile 26 | ) 27 | 28 | // GetFileType identifies if a string represents a file or a URL 29 | func GetFileType(r string) (int, error) { 30 | switch { 31 | case strings.HasPrefix(strings.ToLower(r), "http://") || strings.HasPrefix(strings.ToLower(r), "https://"): 32 | if _, err := url.ParseRequestURI(r); err != nil { 33 | return 0, err 34 | } 35 | return isURL, nil 36 | 37 | default: 38 | return isFile, nil 39 | } 40 | } 41 | 42 | // GetSafeLocalTempDirectory returns a temporary, safe directory 43 | func GetSafeLocalTempDirectory() (string, error) { 44 | // create a temporary directory for the certificates and try to download them 45 | // TODO: maybe we should use os.UserCacheDir() for the dir... 46 | t, err := ioutil.TempDir("", "terraform") 47 | if err != nil { 48 | return "", err 49 | } 50 | return t, nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/common/kubeadm_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | ) 21 | 22 | func TestInitConfigSerialization(t *testing.T) { 23 | configContents := ` 24 | apiVersion: kubeadm.k8s.io/v1beta1 25 | bootstrapTokens: 26 | - groups: 27 | - system:bootstrappers:kubeadm:default-node-token 28 | token: 82eb2m.999999idy9l74yha 29 | ttl: 24h0m0s 30 | usages: 31 | - signing 32 | - authentication 33 | kind: InitConfiguration 34 | nodeRegistration: 35 | criSocket: /var/run/dockershim.sock 36 | taints: 37 | - effect: NoSchedule 38 | key: node-role.kubernetes.io/master 39 | --- 40 | apiServer: 41 | timeoutForControlPlane: 4m0s 42 | apiVersion: kubeadm.k8s.io/v1beta1 43 | certificatesDir: /etc/kubernetes/pki 44 | clusterName: kubernetes 45 | controlPlaneEndpoint: "" 46 | controllerManager: {} 47 | dns: 48 | type: CoreDNS 49 | etcd: 50 | local: 51 | dataDir: /var/lib/etcd 52 | imageRepository: k8s.gcr.io 53 | kind: ClusterConfiguration 54 | kubernetesVersion: v1.14.1 55 | networking: 56 | dnsDomain: cluster.local 57 | serviceSubnet: 10.96.0.0/12 58 | ` 59 | 60 | initConfig, err := YAMLToInitConfig([]byte(configContents)) 61 | if err != nil { 62 | t.Fatalf("Error: %v", err) 63 | } 64 | 65 | if initConfig.BootstrapTokens[0].Token.String() != "82eb2m.999999idy9l74yha" { 66 | t.Fatalf("Error: wrong bootstrap token: %v", initConfig.BootstrapTokens[0].Token.String()) 67 | } 68 | if initConfig.KubernetesVersion != "v1.14.1" { 69 | t.Fatalf("Error: wrong kubernetes version: %v", initConfig.KubernetesVersion) 70 | } 71 | 72 | configContentsAgain, err := InitConfigToYAML(initConfig) 73 | if err != nil { 74 | t.Fatalf("Error: %v", err) 75 | } 76 | fmt.Printf("----------------- init configuration ---------------- \n%s", configContentsAgain) 77 | 78 | if len(configContentsAgain) == 0 { 79 | t.Fatalf("wrong serialized contents: %s", configContentsAgain) 80 | } 81 | } 82 | 83 | func TestJoinConfigSerialization(t *testing.T) { 84 | configContents := ` 85 | apiVersion: kubeadm.k8s.io/v1beta1 86 | caCertPath: /etc/kubernetes/pki/ca.crt 87 | discovery: 88 | bootstrapToken: 89 | token: e5927b.cd71ba4602956ef3 90 | unsafeSkipCAVerification: true 91 | timeout: 15m0s 92 | tlsBootstrapToken: "e5927b.cd71ba4602956ef3" 93 | kind: JoinConfiguration 94 | controlPlane: 95 | nodeRegistration: 96 | localAPIEndpoint: 97 | advertiseAddress: 10.10.0.1 98 | ` 99 | 100 | joinConfig, err := YAMLToJoinConfig([]byte(configContents)) 101 | if err != nil { 102 | t.Fatalf("Error: %v", err) 103 | } 104 | if joinConfig.Discovery.BootstrapToken.Token != "e5927b.cd71ba4602956ef3" { 105 | t.Fatalf("Error: wrong bootstrap token: %v", joinConfig.Discovery.BootstrapToken.Token) 106 | } 107 | 108 | configContentsAgain, err := JoinConfigToYAML(joinConfig) 109 | if err != nil { 110 | t.Fatalf("Error: %v", err) 111 | } 112 | fmt.Printf("----------------- join configuration ---------------- \n%s", configContentsAgain) 113 | } 114 | -------------------------------------------------------------------------------- /pkg/common/net.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "fmt" 19 | "net" 20 | "strconv" 21 | "strings" 22 | ) 23 | 24 | // AddressWithPort return an address as expectedHost:expectedPort (setting a default expectedPort p if there was no expectedPort specified) 25 | func AddressWithPort(name string, p int) string { 26 | if strings.IndexByte(name, ':') < 0 { 27 | return net.JoinHostPort(name, fmt.Sprintf("%d", p)) 28 | } 29 | return name 30 | } 31 | 32 | func SplitHostPort(hp string, defaultPort int) (string, int, error) { 33 | if strings.Count(hp, ":") == 0 && defaultPort > 0 { 34 | hp = fmt.Sprintf("%s:%d", hp, defaultPort) 35 | } 36 | h, p, err := net.SplitHostPort(hp) 37 | if err != nil { 38 | return "", 0, err 39 | } 40 | 41 | pi, err := strconv.Atoi(p) 42 | if err != nil { 43 | return "", 0, err 44 | } 45 | 46 | return h, pi, nil 47 | } 48 | -------------------------------------------------------------------------------- /pkg/common/net_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestSplitHostPort(t *testing.T) { 22 | 23 | testsCases := []struct { 24 | addr string 25 | defPort int 26 | expectedHost string 27 | expectedPort int 28 | }{ 29 | { 30 | "some.place:4545", 31 | 0, 32 | "some.place", 33 | 4545, 34 | }, 35 | { 36 | "some.place", 37 | 25, 38 | "some.place", 39 | 25, 40 | }, 41 | { 42 | "some.place:2525", 43 | 8080, 44 | "some.place", 45 | 2525, 46 | }, 47 | } 48 | 49 | for _, testCase := range testsCases { 50 | h, p, err := SplitHostPort(testCase.addr, testCase.defPort) 51 | if err != nil { 52 | t.Fatalf("Error: %v", err) 53 | } 54 | if h != testCase.expectedHost { 55 | t.Fatalf("Error: expectedHost does not match: %q != %q", h, testCase.expectedHost) 56 | } 57 | if p != testCase.expectedPort { 58 | t.Fatalf("Error: expectedPort does not match: %q != %q", p, testCase.expectedPort) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pkg/common/provisioner.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "github.com/hashicorp/terraform/helper/schema" 19 | ) 20 | 21 | // Rationale: 22 | // 23 | // The "provisioner" does not have access to the "resource kubeadm", so we 24 | // must pass configuration in some way from one to the other. 25 | // 26 | // ProvisionerConfigElements is the list of configuration options that are 27 | // passed from the "provider" to the "provisioner". 28 | // 29 | // This dictionary is passed to templates as well, so you can use ie, {{.token}} 30 | // or {{.flannel_backend}} in the templates that loaded in the cluster later on 31 | // (for exmaple, in the CNI manifest) 32 | // 33 | // FIXME: it seems we cannot use types other than "strings": Terraform just skips those fields otherwise 34 | // 35 | var ProvisionerConfigElements = map[string]*schema.Schema{ 36 | "init": { 37 | Type: schema.TypeString, 38 | // Computed: true, 39 | Optional: true, 40 | }, 41 | "join": { 42 | Type: schema.TypeString, 43 | // Computed: true, 44 | Optional: true, 45 | }, 46 | "kube_version": { 47 | Type: schema.TypeString, 48 | // Computed: true, 49 | Optional: true, 50 | }, 51 | "kubeconfig": { 52 | Type: schema.TypeString, 53 | // Computed: true, 54 | Optional: true, 55 | }, 56 | "token": { 57 | Type: schema.TypeString, 58 | // Computed: true, 59 | Optional: true, 60 | Sensitive: true, 61 | }, 62 | "cni_plugin": { 63 | Type: schema.TypeString, 64 | // Computed: true, 65 | Optional: true, 66 | }, 67 | "cni_plugin_manifest": { 68 | Type: schema.TypeString, 69 | // Computed: true, 70 | Optional: true, 71 | }, 72 | "cni_bin_dir": { 73 | Type: schema.TypeString, 74 | // Computed: true, 75 | Optional: true, 76 | }, 77 | "cni_conf_dir": { 78 | Type: schema.TypeString, 79 | // Computed: true, 80 | Optional: true, 81 | }, 82 | "cni_pod_cidr": { 83 | Type: schema.TypeString, 84 | // Computed: true, 85 | Optional: true, 86 | }, 87 | "dns_upstream": { 88 | Type: schema.TypeString, 89 | // Computed: true, 90 | Optional: true, 91 | }, 92 | "flannel_backend": { 93 | Type: schema.TypeString, 94 | Optional: true, 95 | Description: "the flannel backend", 96 | }, 97 | "flannel_image_version": { 98 | Type: schema.TypeString, 99 | Optional: true, 100 | Description: "the flannel image version", 101 | }, 102 | "helm_enabled": { 103 | Type: schema.TypeBool, 104 | // Computed: true, 105 | Optional: true, 106 | }, 107 | "cloud_provider": { 108 | Type: schema.TypeString, 109 | // Computed: true, 110 | Optional: true, 111 | }, 112 | "cloud_config": { 113 | Type: schema.TypeString, 114 | // Computed: true, 115 | Optional: true, 116 | }, 117 | "cloud_provider_flags": { 118 | Type: schema.TypeString, 119 | // Computed: true, 120 | Optional: true, 121 | }, 122 | "dashboard_enabled": { 123 | Type: schema.TypeBool, 124 | // Computed: true, 125 | Optional: true, 126 | }, 127 | "config_path": { 128 | Type: schema.TypeString, 129 | // Computed: true, 130 | Optional: true, 131 | }, 132 | "certs_dir": { 133 | Type: schema.TypeString, 134 | Optional: true, 135 | Description: "the directory for certificates", 136 | }, 137 | //////////////////////////////////////////////////////////// 138 | // certificates 139 | //////////////////////////////////////////////////////////// 140 | "ca_crt": { 141 | Type: schema.TypeString, 142 | // Computed: true, 143 | Optional: true, 144 | Sensitive: true, 145 | }, 146 | "ca_key": { 147 | Type: schema.TypeString, 148 | // Computed: true, 149 | Optional: true, 150 | Sensitive: true, 151 | }, 152 | "sa_crt": { 153 | Type: schema.TypeString, 154 | // Computed: true, 155 | Optional: true, 156 | Sensitive: true, 157 | }, 158 | "sa_key": { 159 | Type: schema.TypeString, 160 | // Computed: true, 161 | Optional: true, 162 | Sensitive: true, 163 | }, 164 | "etcd_crt": { 165 | Type: schema.TypeString, 166 | // Computed: true, 167 | Optional: true, 168 | Sensitive: true, 169 | }, 170 | "etcd_key": { 171 | Type: schema.TypeString, 172 | // Computed: true, 173 | Optional: true, 174 | Sensitive: true, 175 | }, 176 | "proxy_crt": { 177 | Type: schema.TypeString, 178 | // Computed: true, 179 | Optional: true, 180 | Sensitive: true, 181 | }, 182 | "proxy_key": { 183 | Type: schema.TypeString, 184 | // Computed: true, 185 | Optional: true, 186 | Sensitive: true, 187 | }, 188 | } 189 | 190 | // GetProvisionerConfig returns the config for a provisioner, stored in the "config" attribute 191 | func GetProvisionerConfig(d *schema.ResourceData) map[string]interface{} { 192 | return d.Get("config").(map[string]interface{}) 193 | } 194 | -------------------------------------------------------------------------------- /pkg/common/strings.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | // StringSliceUnique removes duplicates in a string slice 18 | func StringSliceUnique(slice []string) []string { 19 | keys := make(map[string]bool) 20 | list := []string{} 21 | for _, entry := range slice { 22 | if _, value := keys[entry]; !value { 23 | keys[entry] = true 24 | list = append(list, entry) 25 | } 26 | } 27 | return list 28 | } 29 | -------------------------------------------------------------------------------- /pkg/common/strings_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "testing" 19 | ) 20 | 21 | func TestStringSliceUnique(t *testing.T) { 22 | 23 | equal := func(a, b []string) bool { 24 | if len(a) != len(b) { 25 | return false 26 | } 27 | for i, v := range a { 28 | if v != b[i] { 29 | return false 30 | } 31 | } 32 | return true 33 | } 34 | 35 | testsCases := []struct { 36 | input []string 37 | expected []string 38 | }{ 39 | { 40 | []string{"hello", "world"}, 41 | []string{"hello", "world"}, 42 | }, 43 | { 44 | []string{"hello", "hello", "world"}, 45 | []string{"hello", "world"}, 46 | }, 47 | { 48 | []string{"hello", "hello", "world", "world"}, 49 | []string{"hello", "world"}, 50 | }, 51 | } 52 | 53 | for _, testCase := range testsCases { 54 | out := StringSliceUnique(testCase.input) 55 | if !equal(testCase.expected, out) { 56 | t.Fatalf("Error: expected output does not match: %q != %q", out, testCase.expected) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /pkg/common/token.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "crypto/rand" 19 | "encoding/hex" 20 | "fmt" 21 | 22 | kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" 23 | ) 24 | 25 | const ( 26 | TokenIDBytes = 3 27 | 28 | TokenSecretBytes = 8 29 | 30 | TokenRegex = `[a-z0-9]{6}\.[a-z0-9]{16}` 31 | ) 32 | 33 | func randBytes(length int) (string, error) { 34 | b := make([]byte, length) 35 | _, err := rand.Read(b) 36 | if err != nil { 37 | return "", err 38 | } 39 | return hex.EncodeToString(b), nil 40 | } 41 | 42 | // GetRandomToken generates a new token with a token ID that is valid as a 43 | // Kubernetes DNS label. 44 | // For more info, see kubernetes/pkg/util/validation/validation.go. 45 | func GetRandomToken() (string, error) { 46 | tokenID, err := randBytes(TokenIDBytes) 47 | if err != nil { 48 | return "", err 49 | } 50 | 51 | tokenSecret, err := randBytes(TokenSecretBytes) 52 | if err != nil { 53 | return "", err 54 | } 55 | 56 | return fmt.Sprintf("%s.%s", tokenID, tokenSecret), nil 57 | } 58 | 59 | func NewBootstrapToken(token string) (kubeadmapi.BootstrapToken, error) { 60 | var err error 61 | bto := kubeadmapi.BootstrapToken{} 62 | bto.Token, err = kubeadmapi.NewBootstrapTokenString(token) 63 | if err != nil { 64 | return kubeadmapi.BootstrapToken{}, err 65 | } 66 | return bto, err 67 | } 68 | 69 | func NewRandomBootstrapToken() (kubeadmapi.BootstrapToken, error) { 70 | t, err := GetRandomToken() 71 | if err != nil { 72 | return kubeadmapi.BootstrapToken{}, err 73 | } 74 | return NewBootstrapToken(t) 75 | } 76 | -------------------------------------------------------------------------------- /pkg/common/validation.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package common 16 | 17 | import ( 18 | "fmt" 19 | "net" 20 | "net/url" 21 | "path/filepath" 22 | "regexp" 23 | 24 | "github.com/hashicorp/terraform/helper/validation" 25 | ) 26 | 27 | const dnsRegex = `^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$` 28 | 29 | var DnsRegexMatcher = regexp.MustCompile(dnsRegex) 30 | 31 | // ValidateDNSName is a regular expression for validating a DNS name 32 | var ValidateDNSName = validation.StringMatch(DnsRegexMatcher, 33 | "the DNS name does not follow RFC 952 and RFC 1123 requirements") 34 | 35 | // ValidateDNSNameOrIP is a regular expression for validating a DNS name or an IP 36 | var ValidateDNSNameOrIP = validation.Any(validation.SingleIP(), ValidateDNSName) 37 | 38 | func ValidateAbsPath(v interface{}, k string) (ws []string, errors []error) { 39 | if !filepath.IsAbs(v.(string)) { 40 | errors = append(errors, fmt.Errorf("%q is not an absolute path", k)) 41 | } 42 | return 43 | } 44 | 45 | func ValidateHostPort(v interface{}, k string) (ws []string, errors []error) { 46 | _, _, err := net.SplitHostPort(v.(string)) 47 | errors = append(errors, fmt.Errorf("%q is not an valid 'expectedHost:expectedPort': %s", k, err)) 48 | return 49 | } 50 | 51 | // ValidateURL validates a URL 52 | func ValidateURL(v interface{}, k string) (ws []string, errors []error) { 53 | if _, err := url.ParseRequestURI(v.(string)); err != nil { 54 | errors = append(errors, fmt.Errorf("%q does not seem a valid URL: %s", k, err)) 55 | } 56 | return 57 | } 58 | -------------------------------------------------------------------------------- /pkg/provider/kubeadm_init_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provider 16 | 17 | import ( 18 | "fmt" 19 | "testing" 20 | 21 | "github.com/hashicorp/terraform/helper/schema" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 24 | ) 25 | 26 | func TestKubeadmInitConfigSerialization(t *testing.T) { 27 | d := schema.ResourceData{} 28 | 29 | token := "82eb2m.999999idy9l74yha" 30 | 31 | d.Set("api.0.internal", "10.10.0.1") 32 | d.Set("network.0.dns.0.domain", "my-local.cluster") 33 | 34 | initConfig, err := dataSourceToInitConfig(&d, token) 35 | if err != nil { 36 | t.Fatalf("could not create initConfig from dataSource: %s", err) 37 | } 38 | 39 | if initConfig.BootstrapTokens[0].Token.String() != token { 40 | t.Fatalf("Error: wrong bootstrap token: %v", initConfig.BootstrapTokens[0].Token.String()) 41 | } 42 | 43 | initConfigBytes, err := common.InitConfigToYAML(initConfig) 44 | if err != nil { 45 | t.Fatalf("Error: %v", err) 46 | } 47 | fmt.Printf("----------------- init configuration ---------------- \n%s", initConfigBytes) 48 | 49 | } 50 | -------------------------------------------------------------------------------- /pkg/provider/kubeadm_join.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provider 16 | 17 | import ( 18 | "fmt" 19 | 20 | "github.com/hashicorp/terraform/helper/schema" 21 | kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 24 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 25 | ) 26 | 27 | // dataSourceToJoinConfig copies some settings to a Join configuration 28 | func dataSourceToJoinConfig(d *schema.ResourceData, token string) (*kubeadmapi.JoinConfiguration, error) { 29 | joinConfig := &kubeadmapi.JoinConfiguration{ 30 | NodeRegistration: kubeadmapi.NodeRegistrationOptions{ 31 | KubeletExtraArgs: common.DefKubeletSettings, 32 | }, 33 | Discovery: kubeadmapi.Discovery{ 34 | BootstrapToken: &kubeadmapi.BootstrapTokenDiscovery{ 35 | Token: token, 36 | UnsafeSkipCAVerification: true, 37 | }, 38 | }, 39 | } 40 | 41 | if _, ok := d.GetOk("runtime.0"); ok { 42 | if runtimeEngineOpt, ok := d.GetOk("runtime.0.engine"); ok { 43 | if socket, ok := common.DefCriSocket[runtimeEngineOpt.(string)]; ok { 44 | ssh.Debug("setting CRI socket '%s'", socket) 45 | joinConfig.NodeRegistration.KubeletExtraArgs["container-runtime-endpoint"] = fmt.Sprintf("unix://%s", socket) 46 | joinConfig.NodeRegistration.CRISocket = socket 47 | } else { 48 | return nil, fmt.Errorf("unknown runtime engine %s", runtimeEngineOpt.(string)) 49 | } 50 | } 51 | 52 | if _, ok := d.GetOk("runtime.0.extra_args.0"); ok { 53 | if args, ok := d.GetOk("runtime.0.extra_args.0.kubelet"); ok { 54 | joinConfig.NodeRegistration.KubeletExtraArgs = args.(map[string]string) 55 | } 56 | } 57 | } 58 | 59 | if _, ok := d.GetOk("network.0"); ok { 60 | if _, ok := d.GetOk("network.0.dns.0"); ok { 61 | if dnsUpstreamOpt, ok := d.GetOk("network.0.dns.0.upstream"); ok { 62 | dnsUp := dnsUpstreamOpt.([]interface{}) 63 | if len(dnsUp) > 0 { 64 | joinConfig.NodeRegistration.KubeletExtraArgs["resolv-conf"] = common.DefResolvUpstreamConf 65 | } 66 | } 67 | } 68 | } 69 | 70 | // check if we have some cloud-provider 71 | if cloudProvRaw, ok := d.GetOk("cloud.0.provider"); ok && len(cloudProvRaw.(string)) > 0 { 72 | joinConfig.NodeRegistration.KubeletExtraArgs["cloud-provider"] = "external" 73 | } 74 | 75 | return joinConfig, nil 76 | } 77 | -------------------------------------------------------------------------------- /pkg/provider/module_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provider 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/hashicorp/terraform/helper/schema" 21 | "github.com/hashicorp/terraform/terraform" 22 | ) 23 | 24 | var testAccProviders map[string]terraform.ResourceProvider 25 | var testAccProvider *schema.Provider 26 | 27 | func init() { 28 | testAccProvider = Provider().(*schema.Provider) 29 | testAccProviders = map[string]terraform.ResourceProvider{ 30 | "kubeadm": testAccProvider, 31 | } 32 | } 33 | 34 | func TestProvider(t *testing.T) { 35 | if err := Provider().(*schema.Provider).InternalValidate(); err != nil { 36 | t.Fatalf("err: %s", err) 37 | } 38 | } 39 | 40 | func TestProvider_impl(t *testing.T) { 41 | var _ = Provider() 42 | } 43 | 44 | func testAccPreCheck(t *testing.T) { 45 | } 46 | -------------------------------------------------------------------------------- /pkg/provisioner/action_drain.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/hashicorp/terraform/helper/schema" 21 | 22 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 23 | ) 24 | 25 | func doRemoveNode(d *schema.ResourceData) ssh.Action { 26 | return ssh.ActionList{ 27 | ssh.DoMessageInfo("Preparing to remove node from cluster..."), 28 | ssh.DoTry(doDrainKubernetesNode(d)), 29 | ssh.DoTry(doRemoveIfMember(d)), 30 | } 31 | } 32 | 33 | // doDrainKubernetesNode drains a Kubernetes node 34 | func doDrainKubernetesNode(d *schema.ResourceData) ssh.Action { 35 | localKubeNode := ssh.KubeNode{} 36 | 37 | actions := ssh.ActionList{ 38 | ssh.DoMessageInfo("Checking if we must drain the node from the Kubernetes cluster..."), 39 | DoGetNodename(d, &localKubeNode), 40 | ssh.ActionFunc(func(ctx context.Context) ssh.Action { 41 | if localKubeNode.IsEmpty() { 42 | return ssh.DoMessageWarn("could not find Kubernetes nodename for this node") 43 | } 44 | // drain the node with "nodename" 45 | return ssh.ActionList{ 46 | doKubectlDrainNode(d, localKubeNode.Nodename), 47 | ssh.DoMessageInfo("Kubernetes node %q has been drained", localKubeNode.Nodename), 48 | doKubectlDeleteNode(d, localKubeNode.Nodename), 49 | ssh.DoMessageInfo("Kubernetes node %q has been deleted", localKubeNode.Nodename), 50 | } 51 | }), 52 | } 53 | return actions 54 | } 55 | -------------------------------------------------------------------------------- /pkg/provisioner/action_init.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "fmt" 19 | "time" 20 | 21 | "github.com/hashicorp/terraform/helper/schema" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 24 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 25 | ) 26 | 27 | // doKubeadmInit runs the `kubeadm init` 28 | func doKubeadmInit(d *schema.ResourceData) ssh.Action { 29 | extraArgs := []string{"--skip-token-print"} 30 | 31 | // get the join configuration 32 | initConfig, _, err := common.InitConfigFromResourceData(d) 33 | if err != nil { 34 | return ssh.ActionError(fmt.Sprintf("could not get a valid 'config' for join'ing: %s", err)) 35 | } 36 | 37 | // ... update the nodename 38 | initConfig.NodeRegistration.Name = getNodenameFromResourceData(d) 39 | 40 | // ... and update the `config.join` section 41 | if err := common.InitConfigToResourceData(d, initConfig); err != nil { 42 | return ssh.ActionError(err.Error()) 43 | } 44 | 45 | actions := ssh.ActionList{ 46 | // * if a "admin.conf" is there and the cluster is alive, do nothing 47 | // (just try to reload CNI, Helm and so) 48 | // * if a partial setup is detected (ie, cluster is not alive but some manifests are there...) 49 | // try to reset the node 50 | // * in any other case, do a regular "kubeadm init" 51 | doDeleteLocalKubeconfig(d), 52 | ssh.DoIfElse( 53 | checkAdminConfAlive(d), 54 | ssh.ActionList{ 55 | ssh.DoMessageInfo("There is a 'admin.conf' in this master pointing to a live cluster: skipping any setup"), 56 | }, 57 | ssh.ActionList{ 58 | ssh.DoRetry( 59 | ssh.Retry{Times: 3, Interval: 15 * time.Second}, 60 | ssh.ActionList{ 61 | doMaybeResetMaster(d, common.DefKubeadmInitConfPath), 62 | doUploadCerts(d), // (we must upload certs because a "kubeadm reset" wipes them...) 63 | ssh.DoMessageInfo("Initializing the cluster with 'kubadm init'..."), 64 | doKubeadm(d, common.DefKubeadmInitConfPath, "init", extraArgs...), 65 | }, 66 | ), 67 | }, 68 | ), 69 | // we always download the kubeconfig and try to do a "kubeactl apply -f" of manifests 70 | doDownloadKubeconfig(d), 71 | doLoadCNI(d), 72 | doLoadDashboard(d), 73 | doLoadHelm(d), 74 | doLoadCloudProviderManager(d), 75 | doLoadExtraManifests(d), 76 | } 77 | return actions 78 | } 79 | 80 | // doMaybeResetMaster maybe "reset"s the master with kubeadm if 81 | // it is detected as "partially" setup: 82 | // ie, /etc/kubernetes/kubeadm-*.conf exist AND /etc/kubernetes/manifests/* exist 83 | func doMaybeResetMaster(d *schema.ResourceData, kubeadmConfigFilename string) ssh.Action { 84 | return ssh.DoIf( 85 | ssh.CheckOr( 86 | ssh.CheckFileExists(kubeadmConfigFilename), 87 | ssh.CheckFileExists("/etc/kubernetes/manifests/kube-apiserver.yaml"), 88 | ssh.CheckFileExists("/etc/kubernetes/manifests/kube-controller-manager.yaml"), 89 | ssh.CheckFileExists("/etc/kubernetes/manifests/kube-scheduler.yaml"), 90 | ), 91 | ssh.ActionList{ 92 | ssh.DoMessageWarn("previous kubeadm config file found: resetting node"), 93 | doExecKubeadmWithConfig(d, "reset", "", "--force"), 94 | ssh.DoDeleteFile(kubeadmConfigFilename), 95 | ssh.DoFlushCache(), 96 | }) 97 | } 98 | -------------------------------------------------------------------------------- /pkg/provisioner/action_init_addons.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | 21 | "github.com/hashicorp/terraform/helper/schema" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 24 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 25 | ) 26 | 27 | // doLoadDashboard loads the dashboard (if enabled) 28 | func doLoadDashboard(d *schema.ResourceData) ssh.Action { 29 | opt, ok := d.GetOk("config.dashboard_enabled") 30 | if !ok { 31 | return ssh.DoMessageWarn("the Dashboard will not be loaded") 32 | } 33 | enabled, err := strconv.ParseBool(opt.(string)) 34 | if err != nil { 35 | return ssh.ActionError("could not parse dashboard_enabled in provisioner") 36 | } 37 | if !enabled { 38 | return ssh.DoMessageWarn("The Dashboard will not be loaded") 39 | } 40 | if common.DefDashboardManifest == "" { 41 | return ssh.DoMessageWarn("No manifest for Dashboard: the Dashboard will not be loaded") 42 | } 43 | return ssh.ActionList{ 44 | ssh.DoMessageInfo(fmt.Sprintf("Loading Dashboard from %q", common.DefDashboardManifest)), 45 | doRemoteKubectlApply(d, []ssh.Manifest{{URL: common.DefDashboardManifest}}), 46 | } 47 | } 48 | 49 | // doLoadExtraManifests loads some extra manifests 50 | func doLoadExtraManifests(d *schema.ResourceData) ssh.Action { 51 | manifestsOpt, ok := d.GetOk("manifests") 52 | if !ok { 53 | return nil 54 | } 55 | manifests := []ssh.Manifest{} 56 | for _, v := range manifestsOpt.([]interface{}) { 57 | manifests = append(manifests, ssh.NewManifest(v.(string))) 58 | } 59 | if len(manifests) == 0 { 60 | return ssh.DoMessageWarn("Could not find valid manifests to load") 61 | } 62 | return ssh.ActionList{ 63 | ssh.DoMessageInfo(fmt.Sprintf("Loading %d extra manifests", len(manifests))), 64 | doRemoteKubectlApply(d, manifests), 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/provisioner/action_init_cni.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | 21 | "github.com/hashicorp/terraform/helper/schema" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 24 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 25 | ) 26 | 27 | // doLoadCNI loads the CNI driver 28 | func doLoadCNI(d *schema.ResourceData) ssh.Action { 29 | manifest := ssh.Manifest{} 30 | var message ssh.Action 31 | 32 | if cniPluginManifestOpt, ok := d.GetOk("config.cni_plugin_manifest"); ok { 33 | cniPluginManifest := strings.TrimSpace(cniPluginManifestOpt.(string)) 34 | if len(cniPluginManifest) > 0 { 35 | manifest = ssh.NewManifest(cniPluginManifest) 36 | if manifest.Inline != "" { 37 | return ssh.ActionError(fmt.Sprintf("%q not recognized as URL or local filename", cniPluginManifest)) 38 | } 39 | message = ssh.DoMessageInfo(fmt.Sprintf("Loading CNI plugin from %q", cniPluginManifest)) 40 | } 41 | } else { 42 | if cniPluginOpt, ok := d.GetOk("config.cni_plugin"); ok { 43 | cniPlugin := strings.TrimSpace(strings.ToLower(cniPluginOpt.(string))) 44 | if len(cniPlugin) > 0 { 45 | ssh.Debug("verifying CNI plugin: %s", cniPlugin) 46 | if m, ok := common.CNIPluginsManifestsTemplates[cniPlugin]; ok { 47 | ssh.Debug("CNI plugin: %s", cniPlugin) 48 | manifest = m 49 | } else { 50 | panic("unknown CNI driver: should have been caught at the validation stage") 51 | } 52 | message = ssh.DoMessageInfo(fmt.Sprintf("Loading CNI plugin %q", cniPlugin)) 53 | } 54 | } 55 | } 56 | 57 | if manifest.IsEmpty() { 58 | return ssh.DoMessageWarn("no CNI driver is going to be loaded") 59 | } 60 | 61 | err := manifest.ReplaceConfig(common.GetProvisionerConfig(d)) 62 | if err != nil { 63 | return ssh.ActionError(fmt.Sprintf("could not replace variables in manifest: %s", err)) 64 | } 65 | 66 | return ssh.ActionList{ 67 | message, 68 | doRemoteKubectlApply(d, []ssh.Manifest{manifest}), 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /pkg/provisioner/action_init_cri.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "github.com/inercia/terraform-provider-kubeadm/internal/assets" 19 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 20 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 21 | ) 22 | 23 | // doPrepareCRI preparse the CRI in the target node 24 | func doPrepareCRI() ssh.Action { 25 | return ssh.ActionList{ 26 | ssh.DoUploadBytesToFile([]byte(assets.CNIDefConfCode), common.DefCniLookbackConfPath), 27 | // we must reload the containers runtime engine after changing the CNI configuration 28 | ssh.DoIf( 29 | ssh.CheckServiceExists("crio.service"), 30 | ssh.DoRestartService("crio.service")), 31 | ssh.DoIf( 32 | ssh.CheckServiceExists("docker.service"), 33 | ssh.DoRestartService("docker.service")), 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/provisioner/action_init_helm.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "fmt" 19 | "strconv" 20 | "strings" 21 | 22 | "github.com/hashicorp/terraform/helper/schema" 23 | "k8s.io/helm/cmd/helm/installer" 24 | 25 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 26 | ) 27 | 28 | const ( 29 | defHelmReplicas = 1 30 | defHelmNamespace = "kube-system" 31 | // defHelmNodeselector = "node-role.kubernetes.io/master=" 32 | defHelmNodeselector = "" 33 | ) 34 | 35 | // doLoadHelm loads Helm (if enabled) 36 | func doLoadHelm(d *schema.ResourceData) ssh.Action { 37 | opt, ok := d.GetOk("config.helm_enabled") 38 | if !ok { 39 | return ssh.DoMessageWarn("Helm will not be loaded") 40 | } 41 | enabled, err := strconv.ParseBool(opt.(string)) 42 | if err != nil { 43 | return ssh.ActionError("could not parse helm_enabled in provisioner") 44 | } 45 | if !enabled { 46 | return ssh.DoMessageWarn("Helm will not be loaded") 47 | } 48 | 49 | opts := installer.Options{ 50 | Namespace: defHelmNamespace, 51 | AutoMountServiceAccountToken: true, 52 | EnableHostNetwork: false, 53 | NodeSelectors: defHelmNodeselector, 54 | UseCanary: false, 55 | Replicas: defHelmReplicas, 56 | // TODO: we shoud have options for enabling TLS and so... 57 | } 58 | manifests, err := installer.TillerManifests(&opts) 59 | if err != nil { 60 | return ssh.ActionError(fmt.Sprintf("could not get Tiller manifests for installing Helm: %s", err)) 61 | } 62 | 63 | allManifests := strings.Join(manifests, "\n---\n") 64 | ssh.Debug("Helm manifests: %s", allManifests) 65 | 66 | kubeconfig := getKubeconfigFromResourceData(d) 67 | 68 | return ssh.ActionList{ 69 | ssh.DoMessageInfo("Loading Helm..."), 70 | doRemoteKubectlApply(d, []ssh.Manifest{{Inline: allManifests}}), 71 | ssh.DoMessageInfo("Now you should initialize the client with 'helm --kubeconfig=%s init'", kubeconfig), 72 | ssh.DoMessageInfo("Then you can install charts with something like 'helm install --kubeconfig=%s --generate-name ...'", kubeconfig), 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /pkg/provisioner/action_init_helm_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "strings" 19 | "testing" 20 | 21 | "github.com/hashicorp/terraform/helper/schema" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 24 | ) 25 | 26 | func TestDoLoadHelm(t *testing.T) { 27 | // responses from the fake remote machine 28 | responses := []string{ 29 | "CONDITION_SUCCEEDED", 30 | } 31 | 32 | t.Skip("FIXME: it seems config.helm_enabled is not properly set...") 33 | d := schema.ResourceData{} 34 | _ = d.Set("install.0.kubectl_path", "") 35 | _ = d.Set("config.helm_enabled", "true") 36 | 37 | ctx, uploads := ssh.NewTestingContextForUploads(responses) 38 | actions := ssh.ActionList{ 39 | doLoadHelm(&d), 40 | } 41 | res := actions.Apply(ctx) 42 | if ssh.IsError(res) { 43 | t.Fatalf("Error: %s", res.Error()) 44 | } 45 | t.Logf("Uploads: %+v", *uploads) 46 | if len(*uploads) == 0 { 47 | t.Fatalf("Error: no uploads performed") 48 | } 49 | for _, manifest := range *uploads { 50 | if !strings.Contains(manifest, "tiller") { 51 | t.Fatalf("Error: no 'tiller' found in manifest:\n%s", manifest) 52 | } 53 | break 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/provisioner/action_join.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "time" 21 | 22 | "github.com/hashicorp/terraform/helper/schema" 23 | kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" 24 | 25 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 26 | "github.com/inercia/terraform-provider-kubeadm/pkg/common" 27 | ) 28 | 29 | const ( 30 | // retry 6 times to join 31 | joinRetryTimes = 6 32 | 33 | // ... waiting 30 seconds between each try 34 | joinRetryInterval = 30 * time.Second 35 | ) 36 | 37 | // doKubeadmJoinWorker runs the `kubeadm join` 38 | func doKubeadmJoinWorker(d *schema.ResourceData) ssh.Action { 39 | // get the join configuration 40 | joinConfig, _, err := common.JoinConfigFromResourceData(d) 41 | if err != nil { 42 | return ssh.ActionError(fmt.Sprintf("could not get a valid 'config' for join'ing: %s", err)) 43 | } 44 | 45 | // ... update the nodename 46 | joinConfig.NodeRegistration.Name = getNodenameFromResourceData(d) 47 | 48 | // ... and update the `config.join` section 49 | if err := common.JoinConfigToResourceData(d, joinConfig); err != nil { 50 | return ssh.ActionError(err.Error()) 51 | } 52 | 53 | actions := ssh.ActionList{ 54 | ssh.DoRetry( 55 | ssh.Retry{Times: joinRetryTimes, Interval: joinRetryInterval}, 56 | ssh.ActionList{ 57 | doCheckLocalKubeconfigExists(d), 58 | }), 59 | ssh.DoRetry( 60 | ssh.Retry{Times: joinRetryTimes, Interval: joinRetryInterval}, 61 | ssh.ActionList{ 62 | doRefreshToken(d), 63 | }), 64 | ssh.DoRetry( 65 | ssh.Retry{Times: joinRetryTimes, Interval: joinRetryInterval}, 66 | ssh.ActionList{ 67 | doMaybeResetWorker(d, common.DefKubeadmJoinConfPath), 68 | ssh.DoMessageInfo("Trying to join the cluster as a worker with 'kubadm join'..."), 69 | doKubeadm(d, common.DefKubeadmJoinConfPath, "join"), 70 | }), 71 | } 72 | return actions 73 | } 74 | 75 | // doKubeadmJoinControlPlane runs the `kubeadm join` for another control-plane machine 76 | func doKubeadmJoinControlPlane(d *schema.ResourceData) ssh.Action { 77 | // get the joinConfiguration from the 'config.join' in the ResourceData 78 | joinConfig, _, err := common.JoinConfigFromResourceData(d) 79 | if err != nil { 80 | return ssh.ActionError(fmt.Sprintf("could not get a valid 'config' for join'ing: %s", err)) 81 | } 82 | 83 | // check that we have a stable control plane endpoint 84 | initConfig, _, err := common.InitConfigFromResourceData(d) 85 | if err != nil { 86 | return ssh.ActionError(fmt.Sprintf("could not get a valid 'config' for join'ing: %s", err)) 87 | } 88 | if len(initConfig.ClusterConfiguration.ControlPlaneEndpoint) == 0 { 89 | return ssh.ActionError("Cannot create additional masters when the 'kubeadm..api.external' is empty") 90 | } 91 | 92 | // add a local Control-Plane section to the JoinConfiguration (that means a new master will be started here) 93 | endpoint := kubeadmapi.APIEndpoint{} 94 | if hp, ok := d.GetOk("listen"); ok { 95 | h, p, err := common.SplitHostPort(hp.(string), common.DefAPIServerPort) 96 | if err != nil { 97 | return ssh.ActionError(fmt.Sprintf("could not parse listen address %q: %s", hp.(string), err)) 98 | } 99 | endpoint = kubeadmapi.APIEndpoint{AdvertiseAddress: h, BindPort: int32(p)} 100 | } else { 101 | endpoint = kubeadmapi.APIEndpoint{AdvertiseAddress: "", BindPort: common.DefAPIServerPort} 102 | } 103 | joinConfig.ControlPlane = &kubeadmapi.JoinControlPlane{LocalAPIEndpoint: endpoint} 104 | 105 | joinConfig.NodeRegistration.Name = getNodenameFromResourceData(d) 106 | 107 | // ... and update the `config.join` section in the ResourceData 108 | if err := common.JoinConfigToResourceData(d, joinConfig); err != nil { 109 | return ssh.ActionError(err.Error()) 110 | } 111 | 112 | actions := ssh.ActionList{ 113 | ssh.DoRetry( 114 | ssh.Retry{Times: joinRetryTimes, Interval: joinRetryInterval}, 115 | ssh.ActionList{ 116 | doCheckLocalKubeconfigExists(d), 117 | }), 118 | ssh.DoRetry( 119 | ssh.Retry{Times: joinRetryTimes, Interval: joinRetryInterval}, 120 | ssh.ActionList{ 121 | doRefreshToken(d), 122 | }), 123 | ssh.DoRetry( 124 | ssh.Retry{Times: joinRetryTimes, Interval: joinRetryInterval}, 125 | ssh.ActionList{ 126 | ssh.DoMessageInfo("Trying to join the cluster control-plane with 'kubadm join'..."), 127 | doMaybeResetMaster(d, common.DefKubeadmJoinConfPath), 128 | doUploadCerts(d), // (we must upload certs because a "kubeadm reset" wipes them...) 129 | doKubeadm(d, common.DefKubeadmJoinConfPath, "join"), 130 | }), 131 | } 132 | return actions 133 | } 134 | 135 | // doCheckLocalKubeconfigExists checks that there is a local kubeconfig 136 | func doCheckLocalKubeconfigExists(d *schema.ResourceData) ssh.Action { 137 | kubeconfig := getKubeconfigFromResourceData(d) 138 | 139 | return ssh.ActionFunc(func(ctx context.Context) ssh.Action { 140 | return ssh.DoIf( 141 | ssh.CheckNot(ssh.CheckLocalFileExists(kubeconfig)), 142 | ssh.DoMessageWarn("no local kubeconfig found at %q", kubeconfig)) 143 | }) 144 | } 145 | -------------------------------------------------------------------------------- /pkg/provisioner/action_setup.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "fmt" 19 | "io/ioutil" 20 | 21 | "github.com/hashicorp/terraform/helper/schema" 22 | 23 | "github.com/inercia/terraform-provider-kubeadm/internal/assets" 24 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 25 | ) 26 | 27 | // doKubeadmSetup tries to install kubeadm in the remote machine 28 | // the auto-installation can be 29 | // 1) our built-in auto-installation script 30 | // 2) a user-provided script in some path 31 | // 3) an inlined user-provided script 32 | func doKubeadmSetup(d *schema.ResourceData) ssh.Action { 33 | if _, ok := d.GetOk("install"); ok { 34 | code := "" 35 | descr := "" 36 | auto := d.Get("install.0.auto").(bool) 37 | inline := d.Get("install.0.inline").(string) 38 | script := d.Get("install.0.script").(string) 39 | 40 | if auto { 41 | ssh.Debug("will upload the builtin auto-installation script") 42 | descr = "Uploading and running built-in kubeadm installation script..." 43 | code = assets.KubeadmSetupScriptCode 44 | } else if len(inline) > 0 { 45 | ssh.Debug("will upload auto-installation script from inlined script: %d bytes", len(inline)) 46 | descr = "Uploading and running inlined installation script..." 47 | code = "#!/bin/sh\n" + inline 48 | } else if len(script) > 0 { 49 | ssh.Debug("will upload auto-installation from custom script from %q", script) 50 | descr = fmt.Sprintf("Uploading and running custom kubeadm script from %s...", script) 51 | contents, err := ioutil.ReadFile(script) 52 | if err != nil { 53 | errMsg := fmt.Sprintf("when reading kubeadm setup script %q: %s", script, err.Error()) 54 | return ssh.ActionError(errMsg) 55 | } 56 | code = string(contents) 57 | } 58 | 59 | return ssh.ActionList{ 60 | ssh.DoMessage(descr), 61 | ssh.DoExecScript([]byte(code)), 62 | } 63 | } 64 | return ssh.ActionList{ 65 | ssh.DoMessageWarn("no auto-installation: assuming kubeadm is installed in the target node."), 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /pkg/provisioner/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // 16 | // The "provisioner" package provides the necessary 17 | // 18 | package provisioner 19 | -------------------------------------------------------------------------------- /pkg/provisioner/etcd_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import "testing" 18 | 19 | func TestParseEndpointsListOutput(t *testing.T) { 20 | s := "https://127.0.0.1:2379, e942f75ad6f00855, 3.3.10, 1.8 MB, true, 2, 24139" 21 | 22 | endpoint := EtcdEndpoint{} 23 | if err := endpoint.FromString(s); err != nil { 24 | t.Fatalf("Error: %v", err) 25 | } 26 | 27 | if !endpoint.IsLeader { 28 | t.Fatalf("isLeader is not set") 29 | } 30 | 31 | if endpoint.ID != "e942f75ad6f00855" { 32 | t.Fatalf("ID does not match: %s", endpoint.ID) 33 | } 34 | 35 | s = ` 36 | https://127.0.0.1:2379, e942f75ad6f00855, 3.3.10, 1.8 MB, true, 2, 24139\r 37 | https://127.0.5.1:2379, 2f75f75431008954, 3.3.10, 1.8 MB, true, 3, 24139\r 38 | https://127.0.8.1:2379, f0085f42f7f00855, 3.3.10, 1.8 MB, false, 2, 24139\r 39 | ` 40 | 41 | endpoints := EtcdEndpointsSet{} 42 | if err := endpoints.FromString(s); err != nil { 43 | t.Fatalf("Error: %v", err) 44 | } 45 | 46 | expected := map[string]struct{}{ 47 | "e942f75ad6f00855": {}, 48 | "2f75f75431008954": {}, 49 | "f0085f42f7f00855": {}, 50 | } 51 | for _, e := range endpoints { 52 | if _, ok := expected[e.ID]; !ok { 53 | t.Fatalf("%s not found in expected map", e.ID) 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /pkg/provisioner/kubernetes_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/hashicorp/terraform/helper/schema" 21 | 22 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 23 | ) 24 | 25 | func TestDoGetNodename(t *testing.T) { 26 | machineID := " bf38f8ac633e4f64a4924b0ed7b25946\r" 27 | output := ` 28 | bf38f8ac633e4f64a4924b0ed7b25946 kubeadm-master-0 29 | 0b44fe52491e401181c4ef5607b70e96 kubeadm-worker-0 30 | ` 31 | 32 | // responses from the fake remote machine 33 | responses := []string{ 34 | machineID, 35 | "CONDITION_SUCCEEDED", 36 | output, 37 | } 38 | 39 | d := schema.ResourceData{} 40 | _ = d.Set("install.0.kubectl_path", "") 41 | 42 | node := ssh.KubeNode{} 43 | ctx := ssh.NewTestingContextWithResponses(responses) 44 | actions := ssh.ActionList{ 45 | DoGetNodename(&d, &node), 46 | } 47 | res := actions.Apply(ctx) 48 | if ssh.IsError(res) { 49 | t.Fatalf("Error: %s", res.Error()) 50 | } 51 | if node.Nodename != "kubeadm-master-0" { 52 | t.Fatalf("Error: wrong nodename %q", node.Nodename) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pkg/provisioner/provisioner.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "fmt" 21 | 22 | "github.com/davecgh/go-spew/spew" 23 | "github.com/hashicorp/terraform/helper/schema" 24 | "github.com/hashicorp/terraform/terraform" 25 | 26 | "github.com/inercia/terraform-provider-kubeadm/internal/assets" 27 | "github.com/inercia/terraform-provider-kubeadm/internal/ssh" 28 | ) 29 | 30 | var ( 31 | ErrUnknownProvisioningProfile = errors.New("unknown provisioning profile") 32 | ) 33 | 34 | func init() { 35 | spew.Config.Indent = "\t" 36 | } 37 | 38 | // runActions runs the provisioner on a specific resource and returns the new 39 | // resource state along with an error. Instead of a diff, the ResourceConfig 40 | // is provided since provisioners only run after a resource has been 41 | // newly created. 42 | func applyFn(ctx context.Context) error { 43 | connData := ctx.Value(schema.ProvConnDataKey).(*schema.ResourceData) 44 | d := ctx.Value(schema.ProvConfigDataKey).(*schema.ResourceData) 45 | s := ctx.Value(schema.ProvRawStateKey).(*terraform.InstanceState) 46 | o := ctx.Value(schema.ProvOutputKey).(terraform.UIOutput) 47 | 48 | //ssh.Debug("kubeadm provisioner: configuration:\n%s\n", spew.Sdump(d)) 49 | ssh.Debug("connection:\n%s\n", spew.Sdump(connData)) 50 | ssh.Debug("instance state:\n%s\n", spew.Sdump(s)) 51 | 52 | // ensure that this is a linux machine 53 | if s.Ephemeral.ConnInfo["type"] != "ssh" { 54 | return fmt.Errorf("Unsupported connection type: %s. This provisioner currently only supports linux", s.Ephemeral.ConnInfo["type"]) 55 | } 56 | 57 | preventSudo := d.Get("prevent_sudo").(bool) 58 | useSudo := !preventSudo && s.Ephemeral.ConnInfo["user"] != "root" 59 | 60 | // build a communicator for the provisioner to use 61 | comm, err := getCommunicator(ctx, o, s) 62 | if err != nil { 63 | o.Output("Error when creating communicator") 64 | return err 65 | } 66 | 67 | // add some extra things to the context 68 | newCtx := ssh.WithValues(ctx, o, o, comm, useSudo) 69 | 70 | // 71 | // resource destruction 72 | // 73 | 74 | drain := d.Get("drain").(bool) 75 | if drain { 76 | ssh.Debug("node will be drained") 77 | action := doRemoveNode(d) 78 | return action.Apply(newCtx) 79 | } 80 | 81 | // 82 | // resource creation 83 | // 84 | 85 | actions := ssh.ActionList{} 86 | 87 | if s.Tainted { 88 | actions = append(actions, ssh.DoMessageInfo("This node will be recreated")) 89 | // TODO: maybe we should exit here 90 | } 91 | if d.IsNewResource() { 92 | actions = append(actions, ssh.DoMessageInfo("New resource: provisioning")) 93 | } 94 | 95 | // add the actions for installing kubeadm 96 | actions = append(actions, doKubeadmSetup(d)) 97 | 98 | // determine what to do (init, join or join --control-plane) depending on the argument provided 99 | join := getJoinFromResourceData(d) 100 | role := getRoleFromResourceData(d) 101 | 102 | // some common actions to do BEFORE doing initting/joining 103 | actions = append(actions, 104 | ssh.DoMessageInfo("Checking we have the required binaries..."), 105 | doCheckCommonBinaries(d), 106 | doPrepareCRI(), 107 | doUploadResolvConf(d), 108 | ssh.DoEnableService("kubelet.service"), 109 | ssh.DoUploadBytesToFile([]byte(assets.KubeletSysconfigCode), getSysconfigPathFromResourceData(d)), 110 | ssh.DoUploadBytesToFile([]byte(assets.KubeletServiceCode), getServicePathFromResourceData(d)), 111 | ssh.DoUploadBytesToFile([]byte(assets.KubeadmDropinCode), getDropinPathFromResourceData(d)), 112 | ) 113 | 114 | if len(join) == 0 { 115 | switch role { 116 | case "worker": 117 | actions = append(actions, ssh.ActionError(fmt.Sprintf("role is %q while no \"join\" argument has been provided", role))) 118 | default: 119 | actions = append(actions, doKubeadmInit(d)) 120 | } 121 | } else { 122 | switch role { 123 | case "master": 124 | actions = append(actions, doKubeadmJoinControlPlane(d)) 125 | case "worker": 126 | actions = append(actions, doKubeadmJoinWorker(d)) 127 | case "": 128 | actions = append(actions, doKubeadmJoinWorker(d)) 129 | default: 130 | actions = append(actions, ssh.ActionError(fmt.Sprintf("unknown provisioning profile: join is %q and role is %q", join, role))) 131 | } 132 | } 133 | 134 | // ... and some common actions to do AFTER initting/joining 135 | actions = append(actions, 136 | ssh.DoMessageInfo("Gathering some info about this node..."), 137 | doCheckLocalKubeconfigIsAlive(d), 138 | doPrintEtcdStatus(d), 139 | ) 140 | 141 | return ssh.ActionList{ 142 | ssh.DoWithCleanup( 143 | actions, 144 | ssh.DoCleanupLeftovers()), 145 | }.Apply(newCtx) 146 | } 147 | -------------------------------------------------------------------------------- /pkg/provisioner/provisioner_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/hashicorp/terraform/config" 21 | "github.com/hashicorp/terraform/helper/schema" 22 | "github.com/hashicorp/terraform/terraform" 23 | ) 24 | 25 | func TestResourceProvisioner_impl(t *testing.T) { 26 | var _ = Provisioner() 27 | } 28 | 29 | func TestProvisioner(t *testing.T) { 30 | if err := Provisioner().(*schema.Provisioner).InternalValidate(); err != nil { 31 | t.Fatalf("err: %s", err) 32 | } 33 | } 34 | 35 | func TestResourceProvider_Validate_good(t *testing.T) { 36 | c := testConfig(t, map[string]interface{}{ 37 | "config": (interface{})(map[string]interface{}{ 38 | "cni_plugin": "flannel", 39 | }), 40 | }) 41 | 42 | warn, errs := Provisioner().Validate(c) 43 | if len(warn) > 0 { 44 | t.Fatalf("Warnings: %v", warn) 45 | } 46 | if len(errs) > 0 { 47 | t.Fatalf("Errors: %v", errs) 48 | } 49 | } 50 | 51 | func TestResourceProvider_Validate_bad(t *testing.T) { 52 | c := testConfig(t, map[string]interface{}{ 53 | "ups": "something", 54 | }) 55 | 56 | warn, errs := Provisioner().Validate(c) 57 | if len(warn) == 0 && len(errs) == 0 { 58 | t.Fatalf("Errors: %v Warnings: %v", errs, warn) 59 | } 60 | } 61 | 62 | func testConfig(t *testing.T, c map[string]interface{}) *terraform.ResourceConfig { 63 | r, err := config.NewRawConfig(c) 64 | if err != nil { 65 | t.Fatalf("bad: %s", err) 66 | } 67 | 68 | return terraform.NewResourceConfig(r) 69 | } 70 | -------------------------------------------------------------------------------- /pkg/provisioner/ssh.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/hashicorp/terraform/communicator" 21 | "github.com/hashicorp/terraform/terraform" 22 | ) 23 | 24 | // getCommunicator gets a new communicator for the remote machine 25 | func getCommunicator(ctx context.Context, o terraform.UIOutput, s *terraform.InstanceState) (communicator.Communicator, error) { 26 | // Get a new communicator 27 | comm, err := communicator.New(s) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | retryCtx, cancel := context.WithTimeout(ctx, comm.Timeout()) 33 | defer cancel() 34 | 35 | // Wait and retry until we establish the connection 36 | err = communicator.Retry(retryCtx, func() error { 37 | return comm.Connect(o) 38 | }) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | // Wait for the context to end and then disconnect 44 | go func() { 45 | <-ctx.Done() 46 | _ = comm.Disconnect() 47 | }() 48 | 49 | return comm, err 50 | } 51 | -------------------------------------------------------------------------------- /pkg/provisioner/token_test.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Alvaro Saurin 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package provisioner 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestGetKubeadmTokensFromString(t *testing.T) { 23 | s := ` 24 | TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS\r 25 | 5befc5.a36864a4c9cc2c7d 22h 2019-07-10T15:08:31Z authentication,signing system:bootstrappers:kubeadm:default-node-token\r 26 | 9befc8.a36864a4c9cc2c7d 26h 2039-02-10T12:13:24Z authentication,signing system:bootstrappers:kubeadm:default-node-token\r 27 | \r 28 | ` 29 | 30 | testCases := map[string]struct { 31 | isExpired bool 32 | }{ 33 | "5befc5.a36864a4c9cc2c7d": { 34 | true, 35 | }, 36 | "9befc8.a36864a4c9cc2c7d": { 37 | false, 38 | }, 39 | } 40 | 41 | tokens := KubeadmTokensSet{} 42 | err := tokens.FromString(s) 43 | if err != nil { 44 | t.Fatalf("Error: %v", err) 45 | } 46 | 47 | now, _ := time.Parse(time.RFC822, "01 Jan 20 20:00 UTC") 48 | t.Logf("now: %s", now) 49 | for _, token := range tokens { 50 | testCase, ok := testCases[token.Token] 51 | t.Logf("testcase: %+v", testCase) 52 | t.Logf("token: %+v", token) 53 | 54 | if !ok { 55 | t.Fatalf("error: token %q not found in tests cases table", token.Token) 56 | } 57 | 58 | if testCase.isExpired != token.IsExpired(now) { 59 | t.Fatalf("error: token %q reports as 'expired=%t' but we expected '%t'", token.Token, token.IsExpired(now), testCase.isExpired) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /staticcheck.conf: -------------------------------------------------------------------------------- 1 | checks = [ 2 | "all", 3 | "-S1002", 4 | "-S1007", 5 | "-S1008", 6 | "-S1009", 7 | "-S1019", 8 | "-S1021", 9 | "-S1025", 10 | "-S1034", 11 | "-ST1000", 12 | "-ST1003", 13 | "-ST1005", 14 | "-ST1017", 15 | "-SA4006", 16 | "-SA4010", 17 | "-SA6000", 18 | "-SA6005" 19 | ] 20 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/00-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | TF_ARGS="" 13 | NUM_MASTERS=2 14 | NUM_WORKERS=2 15 | 16 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 17 | cd $E2E_ENV 18 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 19 | 20 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 21 | 22 | ########################################################################################### 23 | # cleanups 24 | ########################################################################################### 25 | rm -f $E2E_ENV/*.log 26 | 27 | ########################################################################################### 28 | # cluster creation 29 | ########################################################################################### 30 | 31 | section "Terraform info" 32 | terraform --version 33 | 34 | section "Docker info" 35 | docker info 36 | 37 | section "Initializing test env" 38 | terraform init 39 | [ $? -eq 0 ] || abort "could not init Terraform" 40 | 41 | section "Creating initial cluster..." 42 | tf_apply $NUM_MASTERS $NUM_WORKERS $TF_ARGS 43 | 44 | ########################################################################################### 45 | # checks 46 | ########################################################################################### 47 | 48 | check_exp_nodes $NUM_MASTERS $NUM_WORKERS 49 | 50 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/10-add-master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | NUM_MASTERS=3 13 | NUM_WORKERS=2 14 | 15 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 16 | cd $E2E_ENV 17 | 18 | TF_ARGS="" 19 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 20 | 21 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 22 | 23 | ########################################################################################### 24 | # increase one master 25 | ########################################################################################### 26 | 27 | # hack: Terraform-docker fails to restart the haproxy, so just kill it or the "apply" will fail 28 | # we need to do this before changing the number of masters 29 | docker_stop "kubeadm-haproxy" 30 | 31 | log "Adding one master..." 32 | tf_apply $NUM_MASTERS $NUM_WORKERS $TF_ARGS 33 | 34 | ########################################################################################### 35 | # checks 36 | ########################################################################################### 37 | 38 | check_exp_nodes $NUM_MASTERS $NUM_WORKERS 39 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/15-remove-all-tokens.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 13 | cd $E2E_ENV 14 | 15 | TF_ARGS="" 16 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 17 | 18 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 19 | 20 | ########################################################################################### 21 | # remove all the tokens 22 | ########################################################################################### 23 | 24 | section "Flushing all the tokens" 25 | kubeadm_token_list 26 | kubeadm_token_flush -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/20-add-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | NUM_MASTERS=3 13 | NUM_WORKERS=3 14 | 15 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 16 | cd $E2E_ENV 17 | 18 | TF_ARGS="" 19 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 20 | 21 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 22 | 23 | ########################################################################################### 24 | # increase one worker 25 | ########################################################################################### 26 | 27 | section "Adding one worker..." 28 | tf_apply $NUM_MASTERS $NUM_WORKERS $TF_ARGS 29 | 30 | ########################################################################################### 31 | # checks 32 | ########################################################################################### 33 | 34 | check_exp_nodes $NUM_MASTERS $NUM_WORKERS 35 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/25-show-tokens.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 13 | cd $E2E_ENV 14 | 15 | TF_ARGS="" 16 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 17 | 18 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 19 | 20 | ########################################################################################### 21 | # list all the tokens 22 | ########################################################################################### 23 | 24 | section "Showing all the tokens..." 25 | kubeadm_token_list 26 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/30-del-master.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | NUM_MASTERS=2 13 | NUM_WORKERS=3 14 | 15 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 16 | cd $E2E_ENV 17 | 18 | TF_ARGS="" 19 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 20 | 21 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 22 | 23 | ########################################################################################### 24 | # delete a master 25 | ########################################################################################### 26 | 27 | section "Deleting a master..." 28 | 29 | # hack: Terraform-docker fails to restart the haproxy, so just kill it or the "apply" will fail 30 | # we need to do this before changing the number of masters 31 | docker_stop "kubeadm-haproxy" 32 | 33 | tf_apply $NUM_MASTERS $NUM_WORKERS $TF_ARGS 34 | 35 | ########################################################################################### 36 | # checks 37 | ########################################################################################### 38 | 39 | check_exp_nodes $NUM_MASTERS $NUM_WORKERS 40 | 41 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/40-del-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | NUM_MASTERS=2 13 | NUM_WORKERS=2 14 | 15 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 16 | cd $E2E_ENV 17 | 18 | TF_ARGS="" 19 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 20 | 21 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 22 | 23 | ########################################################################################### 24 | # increase one worker 25 | ########################################################################################### 26 | 27 | section "Deleting a worker..." 28 | tf_apply $NUM_MASTERS $NUM_WORKERS $TF_ARGS 29 | 30 | ########################################################################################### 31 | # checks 32 | ########################################################################################### 33 | 34 | check_exp_nodes $NUM_MASTERS $NUM_WORKERS 35 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/99-teardown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ########################################################################################### 4 | # variables 5 | ########################################################################################### 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 8 | [ -f $DIR/common.bash ] && source $DIR/common.bash 9 | [ -f $DIR/dynamic.bash ] && source $DIR/dynamic.bash 10 | [ -f $DIR/local.bash ] && source $DIR/local.bash 11 | 12 | [ -d $E2E_ENV ] || abort "directory $E2E_ENV does not seem to exist" 13 | cd $E2E_ENV 14 | 15 | TF_ARGS="" 16 | [ -f "ci.tfvars" ] && [ "$IS_CI" = "true" ] && TF_ARGS="$TF_ARGS -var-file=ci.tfvars" 17 | 18 | export KUBECONFIG=$E2E_ENV/kubeconfig.local 19 | 20 | ########################################################################################### 21 | # cluster creation 22 | ########################################################################################### 23 | 24 | section "Testsuite finished: destroying cluster" 25 | 26 | if [ "$$TRAVIS_EVENT_TYPE" = "cron" ] ; then 27 | info "daily build: leaving cluster alive..." 28 | else 29 | tf_destroy $TF_ARGS 30 | fi 31 | 32 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | The goal of this test is to check that we can increase the cluster size. 4 | 5 | So the test flow is: 6 | 7 | 1. create a cluster with 2 masters and 2 workers, checking we have 2 masters and 2 workers. 8 | 2. add a master, checking we have 3 masters in total. 9 | 3. flush all the tokens. 10 | 4. add a worker, checking that a new token is created and we have 3 workers in total. 11 | 5. remove a master, checking we have 2 masters now 12 | 6. remove a worker , checking we have 2 workers now 13 | 14 | All these checks are performed by running `kubect get nodes`. 15 | 16 | -------------------------------------------------------------------------------- /tests/e2e/01-create-increase-reduce/common.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | export PATH=/opt/bin:$PATH 4 | 5 | # common Terraform arguments 6 | export TF_COMMON_ARGS="--auto-approve" 7 | 8 | export TF_LOG_FILENAME="$E2E_ENV/terraform.log" 9 | 10 | 11 | ###################################################################################### 12 | 13 | RED="\e[31m" 14 | GREEN="\e[32m" 15 | YELLOW="\e[33m" 16 | NC="\e[0m" 17 | BOLD="\e[1m" 18 | 19 | log() { echo -e >&2 ">>> $@"; } 20 | info() { log "${INFO} $@${NC}" ;} 21 | failed() { log "${RED}FAILED: $@${NC}" ; } 22 | warn() { log "${RED}WARNING: $@${NC}" ; } 23 | abort() { log "${RED}${BOLD}>>>>>>>>>> FATAL: $@ <<<<<<<<<<< <<<${NC}" ; exit 1 ; } 24 | section() { 25 | log "${GREEN} ---------------------------------------------------------------${NC}" 26 | log "${GREEN} $@${NC}" 27 | log "${GREEN} ---------------------------------------------------------------${NC}" 28 | } 29 | 30 | ###################################################################################### 31 | 32 | download_k8s_bin() { 33 | bin=$1 34 | 35 | RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)" 36 | 37 | info "Downloading $bin..." 38 | cd /tmp 39 | curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/$bin 40 | [ -f $bin ] || abort "$bin was not downloaded with curl" 41 | 42 | info "Moving $bin to /opt/bin with the right permissions ..." 43 | chmod 755 $bin 44 | mkdir -p /opt/bin 45 | sudo mv $bin /opt/bin/ 46 | [ -x /opt/bin/$bin ] || abort "$bin was not properly installed in /opt/bin" 47 | } 48 | 49 | install_kubeadm() { 50 | download_k8s_bin kubeadm 51 | } 52 | 53 | install_kubectl() { 54 | download_k8s_bin kubectl 55 | } 56 | 57 | check_exp_nodes() { 58 | local exp_num_masters=$1 59 | local exp_num_workers=$2 60 | 61 | [ -f $KUBECONFIG ] || abort "no kubeconfig found at $KUBECONFIG" 62 | 63 | info "Checking we can get cluster info with kubectl..." 64 | kubectl --kubeconfig=$KUBECONFIG get nodes 65 | [ $? -eq 0 ] || abort "could not get the nodes with kubectl" 66 | 67 | local output=$(kubectl --kubeconfig=$KUBECONFIG get nodes --show-labels) 68 | [ $? -eq 0 ] || abort "could not get the number of nodes with kubectl" 69 | 70 | exp_num_nodes=$((exp_num_masters + exp_num_workers)) 71 | info "Checking we have $exp_num_nodes nodes..." 72 | curr_num_nodes=$(echo "$output" | grep -c "kubernetes.io/hostname") 73 | if [ $curr_num_nodes -ne $exp_num_nodes ] ; then 74 | abort "current number of nodes, $curr_num_nodes, do not match $exp_num_nodes" 75 | fi 76 | info "... good, we have $exp_num_nodes nodes..." 77 | 78 | info "Checking we have $exp_num_masters masters..." 79 | curr_num_masters=$(echo "$output" | grep -c "node-role.kubernetes.io/master") 80 | if [ $curr_num_masters -ne $exp_num_masters ] ; then 81 | abort "current number of masters, $curr_num_masters, do not match $exp_num_masters" 82 | fi 83 | info "... good, we have $exp_num_masters masters..." 84 | } 85 | 86 | ###################################################################################### 87 | 88 | tf_apply() { 89 | local num_masters=$1 90 | shift 91 | local num_workers=$1 92 | shift 93 | 94 | local log_filename="$E2E_ENV/terraform.log" 95 | 96 | info "running 'terraform apply' for masters:$num_masters workers:$num_workers" 97 | exec 3>$TF_LOG_FILENAME 98 | TF_LOG=DEBUG \ 99 | TF_VAR_master_count=$num_masters TF_VAR_worker_count=$num_workers \ 100 | terraform apply $TF_COMMON_ARGS $@ 2>&3 | tee -a >(tee >&3) 101 | res=$? 102 | exec 3>&- 103 | 104 | [ $res -eq 0 ] || \ 105 | abort "could not apply Terraform script for masters:$num_masters workers:$num_workers" 106 | info "'terraform apply' was successful" 107 | } 108 | 109 | tf_destroy() { 110 | local log_filename="$E2E_ENV/terraform.log" 111 | 112 | info "running 'terraform destroy'" 113 | exec 3>$TF_LOG_FILENAME 114 | TF_LOG=DEBUG \ 115 | terraform destroy $TF_COMMON_ARGS $@ 2>&3 | tee -a >(tee >&3) 116 | res=$? 117 | exec 3>&- 118 | 119 | [ $res -eq 0 ] || abort "could not destroy Terraform cluster" 120 | info "'terraform destroy' was successful" 121 | } 122 | 123 | ###################################################################################### 124 | 125 | kubeadm_check_installation() { 126 | command -v kubeadm >/dev/null 2>&1 || { log "kubeadm is not installed: installing." ; install_kubeadm ; } 127 | } 128 | 129 | kubeadm_token_list() { 130 | kubeadm_check_installation 131 | 132 | [ -f $KUBECONFIG ] || abort "no kubeconfig found at $KUBECONFIG" 133 | 134 | info "Current list of tokens:" 135 | kubeadm token list --kubeconfig="$KUBECONFIG" 136 | } 137 | 138 | kubeadm_token_flush() { 139 | kubeadm_check_installation 140 | 141 | [ -f $KUBECONFIG ] || abort "no kubeconfig found at $KUBECONFIG" 142 | 143 | TOKENS=$(kubeadm token list --kubeconfig="$KUBECONFIG" | grep -E "[a-z0-9]{6}\.[a-z0-9]{16}" | cut -f1 -d" ") 144 | info "Removing all the tokens..." 145 | for token in $TOKENS ; do 146 | info "... removing token $token" 147 | kubeadm token delete --kubeconfig="$KUBECONFIG" $token 148 | done 149 | 150 | info "Listing tokens after 'flush':" 151 | kubeadm_token_list 152 | } 153 | 154 | ###################################################################################### 155 | 156 | docker_stop() { 157 | local name=$1 158 | 159 | docker stop $name 160 | } 161 | -------------------------------------------------------------------------------- /tests/e2e/Makefile: -------------------------------------------------------------------------------- 1 | all: ci-tests 2 | 3 | RED := \e[31m 4 | GREEN := \e[32m 5 | NC := \e[0m 6 | 7 | ci-setup: 8 | mkdir -p "$TRAVIS_BUILD_DIR/snaps-cache" 9 | 10 | @echo -e ">>>$(GREEN) Installation stuff specific to the $$E2E_ENV environment... $(NC)" 11 | [ -n "$$E2E_ENV" ] && make -C "$$E2E_ENV" ci-setup 12 | 13 | @echo -e ">>>$(GREEN) Installing other dependencies, like kubectl, terraform... $(NC)" 14 | sudo apt-get update && sudo apt-get install -y apt-transport-https 15 | curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - 16 | echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list 17 | sudo apt-get update 18 | sudo apt install -y kubectl kubeadm unzip 19 | 20 | @echo -e ">>>$(GREEN) Installing Terraform... $(NC)" 21 | wget https://releases.hashicorp.com/terraform/0.11.13/terraform_0.11.13_linux_amd64.zip 22 | unzip terraform_0.11.13_linux_amd64.zip 23 | sudo mv terraform /usr/local/bin/ 24 | 25 | @echo -e ">>>$(GREEN) Checking if we must setup things for nightly tests... $(NC)" 26 | @if [ "$$TRAVIS_EVENT_TYPE" = "cron" ] ; then \ 27 | make -C "nightly" ci-setup ; \ 28 | else \ 29 | echo -e ">>>$(GREEN) ... no nightly tests. $(NC)" ; \ 30 | fi 31 | 32 | ci-cleanup: 33 | @[ -n "$$E2E_CLEANUP" ] && \ 34 | echo -e ">>>$(RED) Destroying cluster... $(NC)\n" && \ 35 | make -C $$E2E_ENV ci-cleanup || /bin/true 36 | 37 | ci-logs: 38 | @[ -f $$E2E_ENV/terraform.log ] || echo -e ">>>$(RED) No logs stored at $$E2E_ENV/terraform.log $(NC)" 39 | @[ -f $$E2E_ENV/terraform.log -a -n "$$IS_CI" ] && \ 40 | echo -e "\n\n\n\n\n\n" && \ 41 | cat $$E2E_ENV/terraform.log && \ 42 | echo -e "\n\n\n\n\n\n" || \ 43 | /bin/true 44 | @[ -f $$E2E_ENV/terraform.log ] && \ 45 | echo -e ">>>$(RED) Logs available at $$E2E_ENV/terraform.log $(NC)" || \ 46 | /bin/true 47 | 48 | ci-nightly: 49 | @echo -e ">>>$(GREEN) Checking if we must run the nightly tests... $(NC)" 50 | @if [ "$$TRAVIS_EVENT_TYPE" = "cron" ] ; then \ 51 | make -C "nightly" ci-tests ; \ 52 | else \ 53 | echo -e ">>>$(GREEN) ... no nightly tests. $(NC)" ; \ 54 | fi 55 | 56 | ci-tests-testsuites: 57 | @echo -e ">>>$(GREEN) Starting tests suites: $(NC)" 58 | @echo 59 | @rm -f $$E2E_ENV/terraform.log 60 | @for d in `pwd`/[0-9]* ; do \ 61 | echo -e ">>>$(GREEN) Running testsuite `basename $$d` $(NC)" ; \ 62 | for i in $$d/[0-9]*-*.sh ; do \ 63 | echo -e ">>>$(GREEN) Running step `basename $$d`/`basename $$i .sh`$(NC)" ; \ 64 | $$i ; \ 65 | if [ $$? -ne 0 ] ; then \ 66 | echo -e ">>>$(RED) FAILED: $$i $(NC)" ; \ 67 | echo -e ">>>" ; \ 68 | echo -e ">>>" ; \ 69 | make ci-cleanup ; \ 70 | echo -e ">>>" ; \ 71 | echo -e ">>>" ; \ 72 | make ci-logs ; \ 73 | exit 1 ; \ 74 | fi ; \ 75 | done ; \ 76 | done 77 | @echo -e ">>>$(GREEN) All e2e tests have completed$(NC)" 78 | 79 | 80 | ci-tests: ci-tests-testsuites ci-nightly 81 | 82 | -------------------------------------------------------------------------------- /tests/e2e/README.md: -------------------------------------------------------------------------------- 1 | ## End-to-end (e2e) tests 2 | 3 | The _e2e_ tests here are simple scripts based on invoking `terraform` 4 | in a testing environment with different variables that will 5 | drive the test. 6 | 7 | The environment is specified in the `$E2E_ENV` variable, and 8 | is currently one of the directories in [docs/examples](../docs/examples). 9 | By default, we will use the `DnD` environment. You can run the `e2e` test in 10 | a different environment with something like 11 | 12 | ```console 13 | $ make e2e E2E_ENV=`pwd`/docs/examples/aws 14 | ``` 15 | 16 | The Terraform cluster is left intact in case of an error for 17 | forensics purposes. You can destroy it with: 18 | 19 | ```console 20 | $ make e2e-cleanup E2E_ENV=`pwd`/docs/examples/aws 21 | ``` 22 | 23 | Checkout the subdirectories for more details on the current tests suites... 24 | 25 | ## Variables 26 | 27 | The variables that drive these tests will be things like: 28 | 29 | * `TF_VAR_master_count`: number of masters 30 | * `TF_VAR_worker_count`: number of workers 31 | * `TF_VAR_cni`: CNI driver 32 | 33 | Some other vars: 34 | 35 | * `E2E_ENV`: an absolute directory with the tests environment 36 | * `E2E_CLEANUP`: clean up the environment (ie, `terraform destroy`) after failing a tests suite. 37 | 38 | and, when running in Travis, we will add any variables defined in `$E2E_ENV/ci.tfvars`. 39 | 40 | ## Logs 41 | 42 | A debug log from Terraform is left at `$E2E_ENV/terraform.log`. 43 | -------------------------------------------------------------------------------- /tests/e2e/nightly/Makefile: -------------------------------------------------------------------------------- 1 | 2 | # see https://github.com/heptio/sonobuoy 3 | 4 | ci-setup: 5 | @echo ">>> Installing additional stuff for the nightly build..." 6 | GO111MODULE="off" go get -u -v github.com/heptio/sonobuoy 7 | 8 | ci-tests: 9 | @echo ">>> Running nightly job: sonobuoy tests..." 10 | KUBECONFIG=$$E2E_ENV/kubeconfig.local sonobuoy run --wait 11 | 12 | 13 | -------------------------------------------------------------------------------- /utils/errcheck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Check gofmt 4 | echo ">>> Checking for unchecked errors..." 5 | 6 | if ! which errcheck > /dev/null; then 7 | echo ">>> Installing errcheck..." 8 | go get -u github.com/kisielk/errcheck 9 | fi 10 | 11 | err_files=$(errcheck -ignoretests \ 12 | -ignore 'github.com/hashicorp/terraform/helper/schema:Set' \ 13 | -ignore 'bytes:.*' \ 14 | -ignore 'io:Close|Write' \ 15 | $(go list ./...| grep -v /vendor/)) 16 | 17 | if [[ -n ${err_files} ]]; then 18 | echo 'ERROR: Unchecked errors found in the following places:' 19 | echo "${err_files}" 20 | echo "ERROR: Please handle returned errors. You can check directly with \`make errcheck\`" 21 | exit 1 22 | fi 23 | 24 | exit 0 25 | -------------------------------------------------------------------------------- /utils/generate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # a simple script for generating a golang file where we store 3 | # the contents of the files in a constant 4 | # 5 | 6 | log() { echo ">>> $1" ; } 7 | warn() { log "WARNING: $1" ; } 8 | abort() { log "FATAL: $1" ; exit 1 ; } 9 | 10 | IN_FILES= 11 | OUT_FILE="generated.go" 12 | OUT_PACKAGE="main" 13 | OUT_VAR="text" 14 | 15 | while [ $# -gt 0 ] ; do 16 | case $1 in 17 | --out-file) 18 | OUT_FILE=$2 19 | shift 20 | ;; 21 | --out-package) 22 | OUT_PACKAGE=$2 23 | shift 24 | ;; 25 | --out-var) 26 | OUT_VAR=$2 27 | shift 28 | ;; 29 | *) 30 | IN_FILES="$IN_FILES $1" 31 | ;; 32 | esac 33 | shift 34 | done 35 | 36 | [ -z "$IN_FILES" ] && abort "no input files provided" 37 | [ -z "$OUT_FILE" ] && abort "no output files provided" 38 | [ -z "$OUT_PACKAGE" ] && abort "no output package provided" 39 | [ -z "$OUT_VAR" ] && abort "no output variable provided" 40 | 41 | rm -f $OUT_FILE 42 | 43 | grep \` $IN_FILES && abort "input file cannot contain character \`" 44 | 45 | echo "// Code generated automatically with go generate; DO NOT EDIT." > $OUT_FILE 46 | echo >> $OUT_FILE 47 | echo "package $OUT_PACKAGE" >> $OUT_FILE 48 | echo >> $OUT_FILE 49 | echo -n "const $OUT_VAR=\`" >> $OUT_FILE 50 | cat $IN_FILES | sed -e 's|`|\\`|g' >> $OUT_FILE 51 | echo "\`" >> $OUT_FILE 52 | -------------------------------------------------------------------------------- /utils/gofmtcheck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SRC_DIRS="pkg internal" 4 | 5 | # Check gofmt 6 | echo ">>> Checking that code complies with gofmt requirements..." 7 | gofmt_files=$(gofmt -l `find $SRC_DIRS -name '*.go' | grep -v vendor`) 8 | if [[ -n ${gofmt_files} ]]; then 9 | echo 'ERROR: gofmt needs running on the following files:' 10 | echo "${gofmt_files}" 11 | echo "ERROR: You can use the command: \`make fmt\` to reformat code." 12 | exit 1 13 | fi 14 | 15 | exit 0 16 | -------------------------------------------------------------------------------- /utils/travis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | echo ">>> Running travis in /src" 4 | cd /src 5 | su - travis 6 | 7 | # TODO 8 | 9 | echo ">>> Done." 10 | exit 0 11 | --------------------------------------------------------------------------------