├── locals.tf
├── architecture.jpg
├── .gitignore
├── CODE_OF_CONDUCT.md
├── test.yaml
├── providers.tf
├── sample-sg.tf
├── scripts
    ├── sg-policy.sh
    └── network.sh
├── README.md
├── LICENSE
├── iam.tf
├── CONTRIBUTING.md
└── main.tf


/locals.tf:
--------------------------------------------------------------------------------
1 | locals {
2 |   cluster_name = "CNICustomNetworkDemoEKS"
3 |   region       = "ap-southeast-2"
4 | }
5 | 


--------------------------------------------------------------------------------
/architecture.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aws-samples/terraform-cni-custom-network-sample/HEAD/architecture.jpg


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Local .terraform directories
2 | **/.terraform/*
3 | 
4 | # .tfstate files
5 | *.tfstate
6 | *.tfstate.*
7 | .terraform.lock.hcl
8 | 


--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | ## Code of Conduct
2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
4 | opensource-codeofconduct@amazon.com with any additional questions or comments.
5 | 


--------------------------------------------------------------------------------
/test.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: apps/v1
 2 | kind: Deployment
 3 | metadata:
 4 |   name: nginx-deployment
 5 |   labels:
 6 |     app: nginx
 7 | spec:
 8 |   replicas: 1
 9 |   selector:
10 |     matchLabels:
11 |       app: nginx
12 |   template:
13 |     metadata:
14 |       labels:
15 |         app: nginx
16 |     spec:
17 |       containers:
18 |       - name: nginx
19 |         image: nginx:latest
20 |         ports:
21 |         - containerPort: 80
22 | 


--------------------------------------------------------------------------------
/providers.tf:
--------------------------------------------------------------------------------
 1 | provider "aws" {
 2 |   region = "${local.region}"
 3 | }
 4 | 
 5 | data "aws_eks_cluster" "cluster" {
 6 |   name = module.eks.cluster_id
 7 | }
 8 | 
 9 | data "aws_eks_cluster_auth" "cluster" {
10 |   name = module.eks.cluster_id
11 | }
12 | 
13 | provider "kubernetes" {
14 |   host                   = data.aws_eks_cluster.cluster.endpoint
15 |   cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
16 |   token                  = data.aws_eks_cluster_auth.cluster.token
17 | #  load_config_file       = false
18 | #  version                = "~> 1.9"
19 | }
20 | 


--------------------------------------------------------------------------------
/sample-sg.tf:
--------------------------------------------------------------------------------
 1 | # Create example security group for pod
 2 | resource "aws_security_group" "example_sg" {
 3 |   name        = "example_sg"
 4 |   vpc_id      = module.vpc.vpc_id
 5 | 
 6 |   ingress {
 7 |     description      = "allow http"
 8 |     from_port        = 80
 9 |     to_port          = 80
10 |     protocol         = "tcp"
11 |    # cidr_blocks      = module.vpc.private_subnets_cidr_blocks
12 |     cidr_blocks      = ["10.0.0.0/16"]
13 |   }
14 | 
15 |   egress {
16 |     from_port        = 0
17 |     to_port          = 0
18 |     protocol         = "-1"
19 |     cidr_blocks      = ["0.0.0.0/0"]
20 |     ipv6_cidr_blocks = ["::/0"]
21 |   }
22 | }
23 | 


--------------------------------------------------------------------------------
/scripts/sg-policy.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # kubectl auth
 4 | KUBE_CONFIG="$(mktemp)"
 5 | aws eks update-kubeconfig --name "${CLUSTER_NAME}" --kubeconfig "${KUBE_CONFIG}"
 6 | 
 7 | 
 8 | # Create security group policy
 9 | cat <<EOF | kubectl --kubeconfig "${KUBE_CONFIG}" apply -f -
10 | apiVersion: vpcresources.k8s.aws/v1beta1
11 | kind: SecurityGroupPolicy
12 | metadata:
13 |   name: my-security-group-policy
14 |   namespace: test-namespace
15 | spec:
16 |   podSelector:
17 |     matchLabels:
18 |       role: test-role
19 |   securityGroups:
20 |     groupIds:
21 |       - ${SECURITY_GROUP}
22 | EOF
23 | 
24 | # remobe creds
25 | rm "${KUBE_CONFIG}"
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Sample EKS CNI Custom Network with Security Group for Pods
 2 | 
 3 | ## What does this do?
 4 | 
 5 | This repo hosts sample terraform code to create an EKS cluster with CNI custom networking alongside security group for pods.
 6 | 
 7 | Resources created are highlighted in following diagram:
 8 | 
 9 | ![Diagram](/architecture.jpg)
10 | 
11 | ## Steps to run
12 | 
13 | Checkout this repo and trigger terraform build through:
14 | 
15 | ``` bash
16 | 
17 | terraform init
18 | 
19 | terraform apply
20 | ```
21 | 
22 | Type "yes" and the build will start.
23 | 
24 | ### Note
25 | 
26 | By default, the resources are created in `ap-southeast-2` region.
27 | 
28 | # Security
29 | 
30 | See CONTRIBUTING for more information.
31 | 
32 | # License
33 | 
34 | This library is licensed under the MIT-0 License. See the LICENSE file.
35 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 4 | this software and associated documentation files (the "Software"), to deal in
 5 | the Software without restriction, including without limitation the rights to
 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 7 | the Software, and to permit persons to whom the Software is furnished to do so.
 8 | 
 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
15 | 


--------------------------------------------------------------------------------
/scripts/network.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | # kubectl auth
 4 | KUBE_CONFIG="$(mktemp)"
 5 | aws eks update-kubeconfig --name "${CLUSTER_NAME}" --kubeconfig "${KUBE_CONFIG}"
 6 | 
 7 | # CNI env configurations
 8 | kubectl --kubeconfig "${KUBE_CONFIG}" set env daemonset aws-node -n kube-system AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true
 9 | kubectl --kubeconfig "${KUBE_CONFIG}" set env daemonset aws-node -n kube-system ENI_CONFIG_LABEL_DEF=topology.kubernetes.io/zone
10 | 
11 | # Security group for pod env configurations
12 | kubectl --kubeconfig "${KUBE_CONFIG}" set env daemonset aws-node -n kube-system ENABLE_POD_ENI=true
13 | kubectl --kubeconfig "${KUBE_CONFIG}" patch daemonset aws-node \
14 |   -n kube-system \
15 |   -p '{"spec": {"template": {"spec": {"initContainers": [{"env":[{"name":"DISABLE_TCP_EARLY_DEMUX","value":"true"}],"name":"aws-vpc-cni-init"}]}}}}'
16 | 
17 | # set up ENIConfig 
18 | subnet_a=$(echo ${SUBNETS} |awk -F"," '{print $1}')
19 | subnet_b=$(echo ${SUBNETS} |awk -F"," '{print $2}')
20 | subnet_c=$(echo ${SUBNETS} |awk -F"," '{print $3}')
21 | 
22 | cat <<EOF | kubectl --kubeconfig "${KUBE_CONFIG}" apply -f -
23 | apiVersion: crd.k8s.amazonaws.com/v1alpha1
24 | kind: ENIConfig
25 | metadata:
26 |  name: "ap-southeast-2a"
27 | spec:
28 |  subnet: "${subnet_a}"
29 |  securityGroups:
30 |  - ${NODE_SG}
31 | EOF
32 | 
33 | cat <<EOF | kubectl --kubeconfig "${KUBE_CONFIG}" apply -f -
34 | apiVersion: crd.k8s.amazonaws.com/v1alpha1
35 | kind: ENIConfig
36 | metadata:
37 |  name: "ap-southeast-2b"
38 | spec:
39 |  subnet: "${subnet_b}"
40 |  securityGroups:
41 |  - ${NODE_SG}
42 | EOF
43 | 
44 | cat <<EOF | kubectl --kubeconfig "${KUBE_CONFIG}" apply -f -
45 | apiVersion: crd.k8s.amazonaws.com/v1alpha1
46 | kind: ENIConfig
47 | metadata:
48 |  name: "ap-southeast-2c"
49 | spec:
50 |  subnet: "${subnet_c}"
51 |  securityGroups:
52 |  - ${NODE_SG}
53 | EOF
54 | 
55 | # remobe creds
56 | rm "${KUBE_CONFIG}"
57 | 


--------------------------------------------------------------------------------
/iam.tf:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 3 |  * SPDX-License-Identifier: MIT-0
 4 |  *
 5 |  * Permission is hereby granted, free of charge, to any person obtaining a copy of this
 6 |  * software and associated documentation files (the "Software"), to deal in the Software
 7 |  * without restriction, including without limitation the rights to use, copy, modify,
 8 |  * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
 9 |  * permit persons to whom the Software is furnished to do so.
10 |  *
11 |  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
12 |  * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
13 |  * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
14 |  * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
15 |  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
16 |  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
17 |  */
18 | 
19 | # Create IAM role for EKS worker nodes
20 | resource "aws_iam_role" "eks_cluster_node_role" {
21 |   name = "eks-cluster-node-${local.cluster_name}"
22 | 
23 |   assume_role_policy = jsonencode({
24 |     Statement = [{
25 |       Action = "sts:AssumeRole"
26 |       Effect = "Allow"
27 |       Principal = {
28 |         Service = "ec2.amazonaws.com"
29 |       }
30 |     }]
31 |     Version = "2012-10-17"
32 |   })
33 | }
34 | 
35 | resource "aws_iam_role_policy_attachment" "AmazonEKSWorkerNodePolicy" {
36 |   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
37 |   role       = aws_iam_role.eks_cluster_node_role.name
38 | }
39 | 
40 | resource "aws_iam_role_policy_attachment" "AmazonEKS_CNI_Policy" {
41 |   policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
42 |   role       = aws_iam_role.eks_cluster_node_role.name
43 | }
44 | 
45 | resource "aws_iam_role_policy_attachment" "AmazonEC2ContainerRegistryReadOnly" {
46 |   policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
47 |   role       = aws_iam_role.eks_cluster_node_role.name
48 | }
49 | 


--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
 1 | # Contributing Guidelines
 2 | 
 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional
 4 | documentation, we greatly value feedback and contributions from our community.
 5 | 
 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary
 7 | information to effectively respond to your bug report or contribution.
 8 | 
 9 | 
10 | ## Reporting Bugs/Feature Requests
11 | 
12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features.
13 | 
14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already
15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
16 | 
17 | * A reproducible test case or series of steps
18 | * The version of our code being used
19 | * Any modifications you've made relevant to the bug
20 | * Anything unusual about your environment or deployment
21 | 
22 | 
23 | ## Contributing via Pull Requests
24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
25 | 
26 | 1. You are working against the latest source on the *main* branch.
27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
29 | 
30 | To send us a pull request, please:
31 | 
32 | 1. Fork the repository.
33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
34 | 3. Ensure local tests pass.
35 | 4. Commit to your fork using clear commit messages.
36 | 5. Send us a pull request, answering any default questions in the pull request interface.
37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
38 | 
39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and
40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
41 | 
42 | 
43 | ## Finding contributions to work on
44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start.
45 | 
46 | 
47 | ## Code of Conduct
48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
50 | opensource-codeofconduct@amazon.com with any additional questions or comments.
51 | 
52 | 
53 | ## Security issue notifications
54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.
55 | 
56 | 
57 | ## Licensing
58 | 
59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
60 | 


--------------------------------------------------------------------------------
/main.tf:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  3 |  * SPDX-License-Identifier: MIT-0
  4 |  *
  5 |  * Permission is hereby granted, free of charge, to any person obtaining a copy of this
  6 |  * software and associated documentation files (the "Software"), to deal in the Software
  7 |  * without restriction, including without limitation the rights to use, copy, modify,
  8 |  * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
  9 |  * permit persons to whom the Software is furnished to do so.
 10 |  *
 11 |  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 12 |  * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
 13 |  * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 14 |  * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 15 |  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 16 |  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 17 |  */
 18 | 
 19 | # STEP 1: Create VPC with public, private and intra subnets
 20 | module "vpc" {
 21 |   source  = "terraform-aws-modules/vpc/aws"
 22 |   version = "3.7.0"
 23 |   name                  = "terraform-demo-vpc"
 24 |   cidr                  = "10.0.0.0/16"
 25 |   secondary_cidr_blocks = ["100.64.0.0/16"]
 26 |   azs                   = ["${local.region}a", "${local.region}b", "${local.region}c"]
 27 |   private_subnets       = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
 28 |   intra_subnets         = [ "100.64.1.0/24", "100.64.2.0/24", "100.64.3.0/24"]
 29 |   public_subnets        = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
 30 |   enable_nat_gateway    = true
 31 |   single_nat_gateway    = true
 32 |   enable_dns_hostnames  = true
 33 | }
 34 | # private nat gateway
 35 | resource "aws_nat_gateway" "private_nat" {
 36 |   connectivity_type = "private"
 37 |   subnet_id         = module.vpc.private_subnets[0]
 38 | }
 39 | resource "aws_route" "intra_subnets_default_gateway" {
 40 |   route_table_id            = module.vpc.intra_route_table_ids[0]
 41 |   destination_cidr_block    = "0.0.0.0/0"
 42 |   nat_gateway_id            = aws_nat_gateway.private_nat.id
 43 |   depends_on                = [aws_nat_gateway.private_nat]
 44 | }
 45 | 
 46 | # STEP 2: Create EKS cluster 
 47 | module "eks" {
 48 |   source          = "terraform-aws-modules/eks/aws"
 49 |   cluster_name    = local.cluster_name
 50 |   cluster_version = "1.21"
 51 |   subnet_ids      = module.vpc.private_subnets
 52 |   vpc_id          = module.vpc.vpc_id
 53 |   cluster_enabled_log_types = [ "audit", "api", "authenticator", "controllerManager", "scheduler" ]
 54 | }
 55 | 
 56 | # STEP 3: Configure CNI custom network 
 57 | resource "null_resource" "cni_patch" {
 58 |   triggers = {
 59 |     cluster_name  = local.cluster_name
 60 |     node_sg       = module.eks.node_security_group_id
 61 |     intra_subnets = join(",", module.vpc.intra_subnets)
 62 |     content       = file("${path.module}/scripts/network.sh")
 63 |   }
 64 |   provisioner "local-exec" {
 65 |     environment = {
 66 |       CLUSTER_NAME = self.triggers.cluster_name
 67 |       NODE_SG      = self.triggers.node_sg
 68 |       SUBNETS      = self.triggers.intra_subnets
 69 |     }
 70 |     command     = "${path.cwd}/scripts/network.sh"
 71 |     interpreter = ["bash"]
 72 |   }
 73 |   depends_on = [
 74 |     module.eks
 75 |   ]
 76 | }
 77 | 
 78 | # STEP 4: Create managed node group
 79 | resource "aws_eks_node_group" "default" {
 80 |   cluster_name  = local.cluster_name
 81 |   node_group_name = "${local.cluster_name}-node-group-default"
 82 |   node_role_arn   = aws_iam_role.eks_cluster_node_role.arn
 83 |   subnet_ids      = module.vpc.private_subnets
 84 |   instance_types  = [ "c5.large" ]
 85 |   scaling_config {
 86 |     desired_size = 1
 87 |     max_size     = 2
 88 |     min_size     = 1
 89 |   }
 90 |   depends_on = [
 91 |     null_resource.cni_patch,
 92 |     module.vpc
 93 |   ]
 94 | }
 95 | 
 96 | # STEP 5: Configure security group policy
 97 | resource "null_resource" "sg_policy" {
 98 |   triggers = {
 99 |     node_group    = aws_eks_node_group.default.id
100 |     cluster_name  = local.cluster_name
101 |     sg       = aws_security_group.example_sg.id
102 |     content       = file("${path.module}/scripts/network.sh")
103 |   }
104 |   provisioner "local-exec" {
105 |     environment = {
106 |       CLUSTER_NAME   = self.triggers.cluster_name
107 |       SECURITY_GROUP = self.triggers.sg
108 |     }
109 |     command     = "${path.cwd}/scripts/sg-policy.sh"
110 |     interpreter = ["bash"]
111 |   }
112 |   depends_on = [
113 |     aws_eks_node_group.default,
114 |     aws_security_group.example_sg,
115 |     kubernetes_namespace.example,
116 |     module.vpc
117 |   ]
118 | }
119 | 
120 | # STEP 6: Launch an example deployment
121 | resource "kubernetes_namespace" "example" {
122 |   metadata {
123 |     name = "test-namespace"
124 |   }
125 | }
126 | resource "kubernetes_deployment" "example" {
127 |   metadata {
128 |     name = "deployment-example"
129 |     namespace = "test-namespace"
130 |     labels = {
131 |       test = "MyExampleApp"
132 |     }
133 |   }
134 |   spec {
135 |     replicas = 2
136 |     selector {
137 |       match_labels = {
138 |         test = "MyExampleApp"
139 |       }
140 |     }
141 |     template {
142 |       metadata {
143 |         labels = {
144 |           test = "MyExampleApp"
145 |           role = "test-role"
146 |         }
147 |       }
148 |       spec {
149 |         container {
150 |           image = "nginx:latest"
151 |           name  = "example"
152 |         }
153 |       }
154 |     }
155 |   }
156 |   depends_on = [
157 |     null_resource.sg_policy
158 |   ]
159 | }
160 | 


--------------------------------------------------------------------------------