├── .gitignore ├── terraform-eks ├── sg.tf ├── vars.tf ├── karpenter.tf ├── outputs.tf ├── vpc.tf ├── provider.tf ├── eks.tf └── iam.tf ├── karpenter ├── amd64-deployment.yaml ├── arm64-deployment.yaml ├── spot-deployment.yaml ├── spot-provisioner.yaml ├── amd64-provisioner.yaml └── arm64-provisioner.yaml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .terraform* 2 | terraform.tfstate* 3 | kubeconfig* -------------------------------------------------------------------------------- /terraform-eks/sg.tf: -------------------------------------------------------------------------------- 1 | resource "aws_security_group" "worker_group_mgmt" { 2 | name_prefix = "worker_group_mgmt" 3 | vpc_id = module.vpc.vpc_id 4 | 5 | ingress { 6 | from_port = 22 7 | to_port = 22 8 | protocol = "tcp" 9 | 10 | cidr_blocks = [ "10.0.0.0/16" ] 11 | } 12 | } -------------------------------------------------------------------------------- /terraform-eks/vars.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | default = "eu-west-1" 3 | description = "AWS region" 4 | } 5 | 6 | variable "cluster_name" { 7 | default = "karpenter-cluster" 8 | } 9 | 10 | variable "cluster_version" { 11 | default = "1.21" 12 | } 13 | 14 | variable "private_subnets" { 15 | description = "List of private subnets" 16 | type = list(string) 17 | default = ["10.0.0.0/24", "10.0.1.0/24", "10.0.2.0/24"] 18 | } 19 | 20 | variable "public_subnets" { 21 | description = "List of public subnets" 22 | type = list(string) 23 | default = ["10.0.3.0/24", "10.0.4.0/24", "10.0.5.0/24"] 24 | } -------------------------------------------------------------------------------- /karpenter/amd64-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: inflate-amd64 5 | spec: 6 | replicas: 0 #You can scale up the deployment 7 | selector: 8 | matchLabels: 9 | app: inflate-amd64 10 | template: 11 | metadata: 12 | labels: 13 | app: inflate-amd64 14 | spec: 15 | terminationGracePeriodSeconds: 0 16 | containers: 17 | - name: inflate-amd64 18 | image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 19 | resources: 20 | requests: 21 | cpu: 1 22 | memory: 256M 23 | nodeSelector: 24 | kubernetes.io/arch: amd64 -------------------------------------------------------------------------------- /karpenter/arm64-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: inflate-arm64 5 | spec: 6 | replicas: 0 #You can scale up the deployment 7 | selector: 8 | matchLabels: 9 | app: inflate-arm64 10 | template: 11 | metadata: 12 | labels: 13 | app: inflate-arm64 14 | spec: 15 | terminationGracePeriodSeconds: 0 16 | containers: 17 | - name: inflate-arm64 18 | image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 19 | resources: 20 | requests: 21 | cpu: 1 22 | memory: 256M 23 | nodeSelector: 24 | kubernetes.io/arch: arm64 -------------------------------------------------------------------------------- /karpenter/spot-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: spot 5 | spec: 6 | replicas: 0 #You can scale up the deployment 7 | selector: 8 | matchLabels: 9 | app: inflate-spot 10 | template: 11 | metadata: 12 | labels: 13 | app: inflate-spot 14 | spec: 15 | terminationGracePeriodSeconds: 0 16 | containers: 17 | - name: inflate-spot 18 | image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 19 | resources: 20 | requests: 21 | cpu: 1 22 | memory: 256M 23 | nodeSelector: 24 | kubernetes.io/arch: amd64 25 | karpenter.sh/capacity-type: spot -------------------------------------------------------------------------------- /terraform-eks/karpenter.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "karpenter" { 2 | depends_on = [module.eks.kubeconfig] 3 | namespace = "karpenter" 4 | create_namespace = true 5 | 6 | name = "karpenter" 7 | repository = "https://charts.karpenter.sh" 8 | chart = "karpenter" 9 | version = "0.5.2" 10 | 11 | set { 12 | name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" 13 | value = module.iam_assumable_role_karpenter.iam_role_arn 14 | } 15 | 16 | set { 17 | name = "controller.clusterName" 18 | value = var.cluster_name 19 | } 20 | 21 | set { 22 | name = "controller.clusterEndpoint" 23 | value = module.eks.cluster_endpoint 24 | } 25 | } -------------------------------------------------------------------------------- /karpenter/spot-provisioner.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: karpenter.sh/v1alpha5 2 | kind: Provisioner 3 | metadata: 4 | name: spot 5 | spec: 6 | requirements: 7 | - key: karpenter.sh/capacity-type 8 | operator: In 9 | values: ["spot"] 10 | - key: "topology.kubernetes.io/zone" 11 | operator: In 12 | values: ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 13 | - key: "kubernetes.io/arch" 14 | operator: In 15 | values: ["amd64"] 16 | limits: 17 | resources: 18 | cpu: 1000 19 | provider: 20 | instanceProfile: KarpenterNodeInstanceProfile-karpenter-cluster 21 | securityGroupSelector: 22 | kubernetes.io/cluster/karpenter-cluster: '*' 23 | ttlSecondsAfterEmpty: 30 -------------------------------------------------------------------------------- /karpenter/amd64-provisioner.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: karpenter.sh/v1alpha5 2 | kind: Provisioner 3 | metadata: 4 | name: amd64 5 | spec: 6 | requirements: 7 | - key: karpenter.sh/capacity-type 8 | operator: In 9 | values: ["on-demand"] 10 | - key: "topology.kubernetes.io/zone" 11 | operator: In 12 | values: ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 13 | - key: "kubernetes.io/arch" 14 | operator: In 15 | values: ["amd64"] 16 | limits: 17 | resources: 18 | cpu: 1000 19 | provider: 20 | instanceProfile: KarpenterNodeInstanceProfile-karpenter-cluster 21 | securityGroupSelector: 22 | kubernetes.io/cluster/karpenter-cluster: '*' 23 | ttlSecondsAfterEmpty: 30 -------------------------------------------------------------------------------- /karpenter/arm64-provisioner.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: karpenter.sh/v1alpha5 2 | kind: Provisioner 3 | metadata: 4 | name: arm64 5 | spec: 6 | requirements: 7 | - key: karpenter.sh/capacity-type 8 | operator: In 9 | values: ["on-demand"] 10 | - key: "topology.kubernetes.io/zone" 11 | operator: In 12 | values: ["eu-west-1a", "eu-west-1b", "eu-west-1c"] 13 | - key: "kubernetes.io/arch" 14 | operator: In 15 | values: ["arm64"] 16 | limits: 17 | resources: 18 | cpu: 1000 19 | provider: 20 | instanceProfile: KarpenterNodeInstanceProfile-karpenter-cluster 21 | securityGroupSelector: 22 | kubernetes.io/cluster/karpenter-cluster: '*' 23 | ttlSecondsAfterEmpty: 30 -------------------------------------------------------------------------------- /terraform-eks/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cluster_id" { 2 | description = "EKS cluster ID" 3 | value = module.eks.cluster_id 4 | } 5 | 6 | output "cluster_endpoint" { 7 | description = "Endpoint for EKS control plane" 8 | value = module.eks.cluster_endpoint 9 | } 10 | 11 | output "cluster_security_group_id" { 12 | description = "Security group ids attached to the cluster control plane" 13 | value = module.eks.cluster_security_group_id 14 | } 15 | 16 | output "kubectl_config" { 17 | description = "kubectl config as generated by the module" 18 | value = module.eks.kubeconfig 19 | } 20 | 21 | output "config_map_aws_auth" { 22 | description = "A kubernetes configuration to authenticate to this EKS cluster" 23 | value = module.eks.config_map_aws_auth 24 | } -------------------------------------------------------------------------------- /terraform-eks/vpc.tf: -------------------------------------------------------------------------------- 1 | module "vpc" { 2 | source = "terraform-aws-modules/vpc/aws" 3 | version = "2.77.0" 4 | 5 | name = "karpenter-vpc" 6 | cidr = "10.0.0.0/16" 7 | azs = ["${var.region}a", "${var.region}b", "${var.region}c",] 8 | private_subnets = var.private_subnets 9 | public_subnets = var.public_subnets 10 | enable_nat_gateway = true 11 | single_nat_gateway = true 12 | enable_dns_hostnames = true 13 | enable_dns_support = true 14 | 15 | public_subnet_tags = { 16 | Name = "karpenter-public-subnets" 17 | } 18 | 19 | private_subnet_tags = { 20 | "kubernetes.io/cluster/${var.cluster_name}" = "owned" 21 | Name = "karpenter-private-subnets" 22 | } 23 | 24 | tags = { 25 | Name = "karpenter-vpc" 26 | } 27 | } -------------------------------------------------------------------------------- /terraform-eks/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | version = "3.69.0" 6 | } 7 | kubernetes = { 8 | source = "hashicorp/kubernetes" 9 | version = "2.7.1" 10 | } 11 | helm = { 12 | source = "hashicorp/helm" 13 | version = "2.4.1" 14 | } 15 | } 16 | } 17 | 18 | provider "helm" { 19 | kubernetes { 20 | config_path = "" #Path to Kubeconfig file 21 | } 22 | } 23 | 24 | provider "aws" { 25 | region = var.region 26 | } 27 | 28 | provider "kubernetes" { 29 | host = data.aws_eks_cluster.cluster.endpoint 30 | cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) 31 | token = data.aws_eks_cluster_auth.cluster.token 32 | } -------------------------------------------------------------------------------- /terraform-eks/eks.tf: -------------------------------------------------------------------------------- 1 | data "aws_eks_cluster" "cluster" { 2 | name = module.eks.cluster_id 3 | } 4 | 5 | data "aws_eks_cluster_auth" "cluster" { 6 | name = module.eks.cluster_id 7 | } 8 | 9 | module "eks" { 10 | source = "terraform-aws-modules/eks/aws" 11 | version = "17.24.0" 12 | cluster_name = var.cluster_name 13 | cluster_version = var.cluster_version 14 | vpc_id = module.vpc.vpc_id 15 | subnets = [for sb in module.vpc.private_subnets : sb] 16 | cluster_endpoint_private_access = true 17 | cluster_endpoint_public_access = true 18 | cluster_create_timeout = "1h" 19 | enable_irsa = true 20 | 21 | 22 | worker_additional_security_group_ids = [aws_security_group.worker_group_mgmt.id] 23 | node_groups_defaults = { 24 | disk_size = 50 25 | } 26 | 27 | node_groups = { 28 | node_group = { 29 | desired_capacity = 1 30 | max_capacity = 1 31 | min_capacity = 1 32 | 33 | instance_types = ["t3.large"] 34 | capacity_type = "ON_DEMAND" 35 | k8s_labels = { 36 | Environment = "karpenter" 37 | } 38 | update_config = { 39 | max_unavailable_percentage = 50 40 | } 41 | } 42 | } 43 | 44 | tags = { 45 | Name = "karpenter-cluster" 46 | } 47 | } -------------------------------------------------------------------------------- /terraform-eks/iam.tf: -------------------------------------------------------------------------------- 1 | data "aws_iam_policy" "ssm_managed_instance" { 2 | arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" 3 | } 4 | 5 | resource "aws_iam_role_policy_attachment" "karpenter_ssm_policy" { 6 | role = module.eks.worker_iam_role_name 7 | policy_arn = data.aws_iam_policy.ssm_managed_instance.arn 8 | } 9 | 10 | resource "aws_iam_instance_profile" "karpenter" { 11 | name = "KarpenterNodeInstanceProfile-${var.cluster_name}" 12 | role = module.eks.worker_iam_role_name 13 | } 14 | 15 | module "iam_assumable_role_karpenter" { 16 | source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" 17 | version = "4.7.0" 18 | create_role = true 19 | role_name = "karpenter-controller-${var.cluster_name}" 20 | provider_url = module.eks.cluster_oidc_issuer_url 21 | oidc_fully_qualified_subjects = ["system:serviceaccount:karpenter:karpenter"] 22 | } 23 | 24 | resource "aws_iam_role_policy" "karpenter_contoller" { 25 | name = "karpenter-policy-${var.cluster_name}" 26 | role = module.iam_assumable_role_karpenter.iam_role_name 27 | 28 | policy = jsonencode({ 29 | Version = "2012-10-17" 30 | Statement = [ 31 | { 32 | Action = [ 33 | "ec2:CreateLaunchTemplate", 34 | "ec2:CreateFleet", 35 | "ec2:RunInstances", 36 | "ec2:CreateTags", 37 | "iam:PassRole", 38 | "ec2:TerminateInstances", 39 | "ec2:DescribeLaunchTemplates", 40 | "ec2:DescribeInstances", 41 | "ec2:DescribeSecurityGroups", 42 | "ec2:DescribeSubnets", 43 | "ec2:DescribeInstanceTypes", 44 | "ec2:DescribeInstanceTypeOfferings", 45 | "ec2:DescribeAvailabilityZones", 46 | "ssm:GetParameter" 47 | ] 48 | Effect = "Allow" 49 | Resource = "*" 50 | }, 51 | ] 52 | }) 53 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Karpenter 2 | This repository consists example codes for **Karpenter Cluster Autoscaler**. 3 | 4 | ![](https://github.com/aws/karpenter/blob/main/website/static/full_logo.png) 5 | 6 | **Karpenter** is an Open Source Cluster Autoscaler tool developed and published by **AWS**. 7 | 8 | ## Folder Structure 9 | ``` 10 | ├── README.md 11 | ├── karpenter 12 | │ ├── amd64-deployment.yaml 13 | │ ├── amd64-provisioner.yaml 14 | │ ├── arm64-deployment.yaml 15 | │ ├── arm64-provisioner.yaml 16 | │ ├── spot-deployment.yaml 17 | │ └── spot-provisioner.yaml 18 | └── terraform-eks 19 | ├── eks.tf 20 | ├── iam.tf 21 | ├── karpenter.tf 22 | ├── outputs.tf 23 | ├── provider.tf 24 | ├── sg.tf 25 | ├── vars.tf 26 | └── vpc.tf 27 | ``` 28 | In the [Terraform folder](https://github.com/eminalemdar/karpenter/tree/master/terraform-eks), you can find the necessary configuration files for deploying a **VPC**, an **EKS Cluster** and **Karpenter Helm Chart** on top of that EKS cluster. Terraform configurations also creates the **IAM** Roles and Instance Profiles. You can change the configurations according to your needs. 29 | 30 | In the [Karpenter folder](https://github.com/eminalemdar/karpenter/tree/master/karpenter), you can find the **Provisioner** and **Kubernetes Deployment** yaml files. 31 | 32 | ## Usage 33 | 34 | - First, create the environment with Terraform. 35 | ```bash 36 | terraform init 37 | ``` 38 | ```bash 39 | terraform plan 40 | ``` 41 | ```bash 42 | terraform apply 43 | ``` 44 | - You must update the `config_path` parameter in the [Provider Configuration](https://github.com/eminalemdar/karpenter/tree/master/terraform-eks/provider.tf) file for Helm provider's connection to the EKS cluster. 45 | - Then you need to create the Provisioners and Deployments. 46 | - Finally you can scale the deployments using ``kubectl scale deployment --replicas=10`` 47 | - You can set the log level of the Karpenter to **DEBUG** to see more information using ``kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}`` 48 | - You can see the logs of the Karpenter Controller using ``kubectl logs -f -n karpenter $(kubectl get pods -n karpenter -l karpenter=controller -o name)`` 49 | 50 | ## Cleanup 51 | 52 | - First, you can delete the Deployments with ``kubectl delete deployment `` 53 | - You can delete the Karpenter installation with ``helm uninstall karpenter -n karpenter`` 54 | - Finally, you can delete the whole environment with 55 | ```bash 56 | terraform destroy 57 | ``` 58 | > PS: Do not forget to delete Launch Templates created by Karpenter from your AWS Account. --------------------------------------------------------------------------------