├── modules ├── k3s_worker_profile │ ├── variables.tf │ ├── outputs.tf │ └── main.tf ├── k3s_master │ ├── outputs.tf │ ├── variables.tf │ ├── vpc.tf │ ├── master_role.tf │ └── master_instance.tf ├── k3s_workers │ ├── variables.tf │ └── main.tf ├── ocean_workers │ ├── variables.tf │ └── main.tf ├── k3s_aws │ └── main.tf └── ocean_k3s_aws │ └── main.tf ├── .gitignore ├── manifests ├── echo_server.yaml ├── echo_server-local.yaml ├── echo_server-hpa.yaml └── cloud-provider-aws.yaml └── README.md /modules/k3s_worker_profile/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | } 4 | -------------------------------------------------------------------------------- /modules/k3s_worker_profile/outputs.tf: -------------------------------------------------------------------------------- 1 | output "worker_profile_id" { 2 | value = "${aws_iam_instance_profile.k3s_worker_profile.id}" 3 | } 4 | -------------------------------------------------------------------------------- /modules/k3s_master/outputs.tf: -------------------------------------------------------------------------------- 1 | output "security_group_id" { 2 | value = "${module.sg.this_security_group_id}" 3 | } 4 | 5 | output "subnet_ids" { 6 | value = module.vpc.public_subnets 7 | } 8 | 9 | output "master_dns_name" { 10 | value = "${module.k3s_master.private_dns[0]}" 11 | } 12 | 13 | output "master_token" { 14 | value = "${random_uuid.token.result}" 15 | } 16 | 17 | -------------------------------------------------------------------------------- /modules/k3s_master/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | default = "ocean" 4 | } 5 | 6 | variable "ssh_key_name" { 7 | type = string 8 | } 9 | 10 | variable "region" { 11 | type = string 12 | } 13 | 14 | variable "install_ocean_controller" { 15 | default = false 16 | } 17 | 18 | variable "ocean_controller_token" { 19 | type = string 20 | default = "" 21 | } 22 | 23 | variable "ocean_account" { 24 | type = string 25 | default = "" 26 | } 27 | -------------------------------------------------------------------------------- /modules/k3s_workers/variables.tf: -------------------------------------------------------------------------------- 1 | variable "worker_count" { 2 | type = number 3 | default = 1 4 | } 5 | 6 | variable "cluster_name" { 7 | type = string 8 | } 9 | 10 | variable "ssh_key_name" { 11 | type = string 12 | } 13 | 14 | variable "security_group_id" { 15 | type = string 16 | } 17 | 18 | variable "subnet_ids" { 19 | type = list(string) 20 | } 21 | 22 | variable "master_dns_name" { 23 | type = string 24 | } 25 | 26 | variable "master_token" { 27 | type = string 28 | } 29 | 30 | variable "region" { 31 | type = string 32 | } 33 | -------------------------------------------------------------------------------- /modules/ocean_workers/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | default = "k3s_ocean" 4 | description = "Name for the k3s cluster" 5 | } 6 | 7 | variable "ssh_key_name" { 8 | type = string 9 | description = "Name of the AWS SSH Key to use" 10 | } 11 | 12 | variable "security_group_id" { 13 | type = string 14 | description = "AWS SecurityGroup the workers will be associated with" 15 | } 16 | 17 | variable "subnet_ids" { 18 | type = list(string) 19 | description = "AWS Subnet IDs the workers will be launched in" 20 | } 21 | 22 | variable "master_dns_name" { 23 | type = string 24 | description = "The k3s master DNS name" 25 | } 26 | 27 | variable "master_token" { 28 | type = string 29 | description = "k3s master token" 30 | } 31 | 32 | variable "region" { 33 | type = string 34 | description = "AWS Region" 35 | } 36 | 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # Crash log files 9 | crash.log 10 | 11 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 12 | # .tfvars files are managed as part of configuration and so should be included in 13 | # version control. 14 | # 15 | # example.tfvars 16 | 17 | # Ignore override files as they are usually used to override resources locally and so 18 | # are not checked in 19 | override.tf 20 | override.tf.json 21 | *_override.tf 22 | *_override.tf.json 23 | 24 | # Include override files you do wish to add to version control using negated pattern 25 | # 26 | # !example_override.tf 27 | 28 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 29 | # example: *tfplan* 30 | 31 | # Ignore CLI configuration files 32 | .terraformrc 33 | terraform.rc 34 | -------------------------------------------------------------------------------- /modules/k3s_aws/main.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | default = "aws" 4 | } 5 | 6 | variable "ssh_key_name" { 7 | type = string 8 | } 9 | 10 | variable "worker_count" { 11 | type = number 12 | } 13 | 14 | variable "region" { 15 | type = string 16 | } 17 | 18 | module "k3s_master" { 19 | source = "../k3s_master" 20 | 21 | region = var.region 22 | cluster_name = var.cluster_name 23 | ssh_key_name = var.ssh_key_name 24 | } 25 | 26 | module "k3s_workers" { 27 | source = "../k3s_workers" 28 | 29 | region = var.region 30 | cluster_name = var.cluster_name 31 | ssh_key_name = var.ssh_key_name 32 | 33 | worker_count = var.worker_count 34 | security_group_id = module.k3s_master.security_group_id 35 | subnet_ids = module.k3s_master.subnet_ids 36 | master_dns_name = module.k3s_master.master_dns_name 37 | master_token = module.k3s_master.master_token 38 | 39 | } 40 | 41 | -------------------------------------------------------------------------------- /manifests/echo_server.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: echo-server 5 | spec: 6 | replicas: 9 7 | selector: 8 | matchLabels: 9 | app: echo-server 10 | template: 11 | metadata: 12 | labels: 13 | app: echo-server 14 | spec: 15 | containers: 16 | - name: echo-server 17 | image: jmalloc/echo-server 18 | ports: 19 | - name: http-port 20 | containerPort: 8080 21 | resources: 22 | limits: 23 | cpu: "1" 24 | requests: 25 | cpu: "0.5" 26 | dnsConfig: 27 | options: 28 | - name: ndots 29 | value: "1" 30 | --- 31 | apiVersion: v1 32 | kind: Service 33 | metadata: 34 | name: echo-server 35 | spec: 36 | ports: 37 | - name: http-port 38 | port: 80 39 | targetPort: http-port 40 | protocol: TCP 41 | selector: 42 | app: echo-server 43 | type: LoadBalancer 44 | -------------------------------------------------------------------------------- /manifests/echo_server-local.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: echo-server 5 | spec: 6 | replicas: 9 7 | selector: 8 | matchLabels: 9 | app: echo-server 10 | template: 11 | metadata: 12 | labels: 13 | app: echo-server 14 | spec: 15 | containers: 16 | - name: echo-server 17 | image: jmalloc/echo-server 18 | ports: 19 | - name: http-port 20 | containerPort: 8080 21 | resources: 22 | limits: 23 | cpu: "1" 24 | requests: 25 | cpu: "0.5" 26 | dnsConfig: 27 | options: 28 | - name: ndots 29 | value: "1" 30 | --- 31 | apiVersion: v1 32 | kind: Service 33 | metadata: 34 | name: echo-server 35 | spec: 36 | ports: 37 | - name: http-port 38 | port: 80 39 | targetPort: http-port 40 | protocol: TCP 41 | selector: 42 | app: echo-server 43 | externalTrafficPolicy: Local 44 | type: LoadBalancer 45 | -------------------------------------------------------------------------------- /modules/ocean_k3s_aws/main.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | default = "ocean" 4 | } 5 | 6 | variable "ssh_key_name" { 7 | type = string 8 | } 9 | 10 | variable "region" { 11 | type = string 12 | } 13 | 14 | variable "ocean_controller_token" { 15 | type = string 16 | } 17 | 18 | variable "ocean_account" { 19 | type = string 20 | } 21 | 22 | module "k3s_master" { 23 | source = "../k3s_master" 24 | 25 | region = var.region 26 | cluster_name = var.cluster_name 27 | ssh_key_name = var.ssh_key_name 28 | install_ocean_controller = true 29 | ocean_controller_token = var.ocean_controller_token 30 | ocean_account = var.ocean_account 31 | } 32 | 33 | module "k3s_workers" { 34 | source = "../ocean_workers" 35 | 36 | cluster_name = var.cluster_name 37 | ssh_key_name = var.ssh_key_name 38 | 39 | region = var.region 40 | security_group_id = module.k3s_master.security_group_id 41 | subnet_ids = module.k3s_master.subnet_ids 42 | master_dns_name = module.k3s_master.master_dns_name 43 | master_token = module.k3s_master.master_token 44 | 45 | } 46 | 47 | -------------------------------------------------------------------------------- /manifests/echo_server-hpa.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: echo-server 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: echo-server 10 | template: 11 | metadata: 12 | labels: 13 | app: echo-server 14 | spec: 15 | containers: 16 | - name: echo-server 17 | image: jmalloc/echo-server 18 | ports: 19 | - name: http-port 20 | containerPort: 8080 21 | resources: 22 | limits: 23 | cpu: 250m 24 | requests: 25 | cpu: 100m 26 | --- 27 | apiVersion: v1 28 | kind: Service 29 | metadata: 30 | name: echo-server 31 | spec: 32 | ports: 33 | - name: http-port 34 | port: 80 35 | targetPort: http-port 36 | protocol: TCP 37 | selector: 38 | app: echo-server 39 | externalTrafficPolicy: Local 40 | type: LoadBalancer 41 | --- 42 | apiVersion: autoscaling/v1 43 | kind: HorizontalPodAutoscaler 44 | metadata: 45 | name: echo-server 46 | namespace: default 47 | spec: 48 | scaleTargetRef: 49 | apiVersion: apps/v1 50 | kind: Deployment 51 | name: echo-server 52 | minReplicas: 1 53 | maxReplicas: 100 54 | targetCPUUtilizationPercentage: 10 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # k3s-terraform-modules 2 | 3 | Modules for testing and development of k3s with Ocean by Spot 4 | 5 | 6 | ## Ocean k3s 7 | 8 | * VPC with 2 subnets in region of choice 9 | * k3s master 10 | * [AWS k8s Cloud Provider][cloud-provider-aws-url] 11 | * [AWS EBS CSI Driver][ebs-csi-driver-url] 12 | * Ocean by Spot worker nodes 13 | 14 | Provision master and Ocean workers 15 | 16 | ``` 17 | module "ocean_k3s_aws" { 18 | source = "github.com/kmcgrath/k3s-terraform-modules//modules/ocean_k3s_aws" 19 | 20 | ocean_account = "act-XXXXXX" 21 | ocean_controller_token = "SECRET" 22 | region = "us-east-1" 23 | ssh_key_name = "my-key" 24 | cluster_name = "ocean-k3s" 25 | } 26 | ``` 27 | 28 | 29 | ## k3s only 30 | 31 | * VPC with 2 subnets in region of choice 32 | * k3s master 33 | * [AWS k8s Cloud Provider][cloud-provider-aws-url] 34 | * [AWS EBS CSI Driver][ebs-csi-driver-url] 35 | * Worker nodes 36 | 37 | Provision master and Ocean workers 38 | 39 | ``` 40 | module "k3s_aws" { 41 | source = "github.com/kmcgrath/k3s-terraform-modules//modules/k3s_aws" 42 | 43 | region = "us-east-1" 44 | ssh_key_name = "my-key" 45 | worker_count = 1 46 | } 47 | ``` 48 | 49 | 50 | [cloud-provider-aws-url]: https://github.com/kubernetes/cloud-provider-aws 51 | [ebs-csi-driver-url]: https://github.com/kubernetes-sigs/aws-ebs-csi-driver 52 | -------------------------------------------------------------------------------- /modules/k3s_workers/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = var.region 3 | } 4 | 5 | data "aws_ami" "latest_amzn" { 6 | most_recent = true 7 | owners = ["amazon"] 8 | 9 | filter { 10 | name = "name" 11 | values = ["amzn2-ami-hvm-2.0.????????.?-x86_64-gp2"] 12 | } 13 | 14 | filter { 15 | name = "state" 16 | values = ["available"] 17 | } 18 | } 19 | 20 | 21 | module "k3s_worker_profile" { 22 | source = "../k3s_worker_profile" 23 | 24 | cluster_name = "${var.cluster_name}" 25 | } 26 | 27 | module "k3s_workers" { 28 | source = "terraform-aws-modules/ec2-instance/aws" 29 | 30 | name = "k3s-worker-${var.cluster_name}" 31 | instance_count = "${var.worker_count}" 32 | 33 | ami = data.aws_ami.latest_amzn.id 34 | instance_type = "t2.medium" 35 | key_name = "${var.ssh_key_name}" 36 | vpc_security_group_ids = ["${var.security_group_id}"] 37 | subnet_id = var.subnet_ids[0] 38 | user_data = "${local.worker_userdata}" 39 | iam_instance_profile = "${module.k3s_worker_profile.worker_profile_id}" 40 | associate_public_ip_address = true 41 | 42 | tags = { 43 | KubernetesCluster = "${var.cluster_name}" 44 | } 45 | } 46 | 47 | 48 | 49 | locals { 50 | 51 | worker_userdata = <