├── .eslintrc.js ├── .gitignore ├── .npmignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── THIRD-PARTY-LICENSES ├── bin └── eks-cdk-js.ts ├── cdk.json ├── images ├── bastion_connect.png ├── bastion_ssm_connect.png ├── console_error.png └── container_insights_1.png ├── lib ├── custom-resources │ └── eksloggingOnEvent.py ├── eks-cdk-js-stack.ts ├── k8s-baseline.ts ├── k8s-nodegroup.ts ├── policies │ └── policies.ts └── vpc-stack.ts ├── manifests ├── awsSecretsManifest.yaml ├── consoleViewOnlyGroup.yaml ├── fluentBitSetup.yaml ├── metricServerManifest.yaml └── otelContainerInsights.yaml ├── package-lock.json ├── package.json ├── test ├── manifests │ ├── alb_example.yaml │ ├── cluster_autoscaling_example.yaml │ ├── ebs_pod_example.yaml │ ├── efs_storageclass_example.yaml │ ├── hpa_example.yaml │ ├── network_policy_example.yaml │ └── secrets_providerdeployment_example.yaml └── test-deployment.sh └── tsconfig.json /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | browser: true, 4 | }, 5 | extends: ["standard", "plugin:import/typescript", "prettier"], 6 | parser: "@typescript-eslint/parser", 7 | parserOptions: { 8 | ecmaVersion: "2018", 9 | sourceType: "module", 10 | project: "./tsconfig.json", 11 | }, 12 | plugins: ["@typescript-eslint", "import", "prettier"], 13 | settings: { 14 | "import/parsers": { 15 | "@typescript-eslint/parser": [".ts", ".tsx"], 16 | }, 17 | "import/resolver": { 18 | node: {}, 19 | typescript: { 20 | directory: "./tsconfig.json", 21 | }, 22 | }, 23 | }, 24 | ignorePatterns: ["*cdk.out/*", "node_modules/", "*.js"], 25 | rules: { 26 | "@typescript-eslint/no-require-imports": ["error"], 27 | "@typescript-eslint/indent": ["error", 2], 28 | "comma-dangle": ["error", "always-multiline"], 29 | "comma-spacing": ["error", { before: false, after: true }], 30 | "array-bracket-newline": ["error", "consistent"], 31 | curly: ["error", "multi-line", "consistent"], 32 | "import/no-extraneous-dependencies": ["error"], 33 | "import/no-unresolved": ["error"], 34 | "import/order": [ 35 | "error", 36 | { 37 | groups: ["builtin", "external"], 38 | alphabetize: { order: "asc", caseInsensitive: true }, 39 | }, 40 | ], 41 | "no-duplicate-imports": ["error"], 42 | "@typescript-eslint/no-shadow": ["error"], 43 | semi: ["error", "always"], 44 | "quote-props": ["error", "consistent-as-needed"], 45 | "max-len": [ 46 | "error", 47 | { 48 | code: 150, 49 | ignoreUrls: true, 50 | ignoreStrings: true, 51 | ignoreTemplateLiterals: true, 52 | ignoreComments: true, 53 | ignoreRegExpLiterals: true, 54 | }, 55 | ], 56 | "@typescript-eslint/no-floating-promises": ["error"], 57 | "no-return-await": "off", 58 | "@typescript-eslint/return-await": "error", 59 | "no-console": ["error"], 60 | "no-bitwise": ["error"], 61 | }, 62 | }; 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | .vscode 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | # Other files 10 | test/eks-cdk-js.test.ts 11 | cdk.context.json 12 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 10 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 11 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 12 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 13 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 14 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 15 | 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!NOTE] 2 | > Recommend to use the newer released EKS Blueprints for https://aws-quickstart.github.io/cdk-eks-blueprints/getting-started/ 3 | 4 | 5 | # Amazon EKS using AWS CDK with Typescript ! 6 | 7 | A sample project that deploys an EKS Cluster following a set of best practices with options to install additional addons. Easy deployment of the EBS CSI Driver, EFS CSI Driver, FluentBit Centralized Logging using Cloudwatch, Cluster Autoscaler, ALB Ingress Controller, Secrets CSI Driver and Network Policy Engine. 8 | 9 | ## Pre-requisites 10 | 11 | - [x] AWS CDK >= 2.3.0 check [Getting Started with AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/getting_started.html#getting_started_install) to setup your CDK environment. Run `cdk --version` to check the CLI version. 12 | 13 | ## Usage 14 | 15 | ### Quick Start 16 | 17 | ```sh 18 | git clone https://github.com/aws-samples/amazon-eks-using-cdk-typescript.git 19 | # install dependant packages 20 | npm install 21 | # If you have not used cdk before, you may be advised to create cdk resources 22 | cdk bootstrap aws://ACCOUNT_ID/REGION 23 | # check the diff before deployment to understand any changes, on first run all resources will created 24 | cdk diff 25 | # Deploy the stack, you will be prompted for confirmation for creation of IAM and Security Group resources 26 | cdk -c cluster_name=myfirstcluster deploy --all 27 | 28 | ``` 29 | 30 | ### Logging into Bastion Host 31 | 32 | Go to the EC2 Console and look for an instance `Name: EKSBastionHost`. Right click on the instance and select connect 33 | 34 | ![](images/bastion_connect.png) 35 | 36 | Select Session Manager and click the `Connect` button 37 | 38 | ![](images/bastion_ssm_connect.png) 39 | 40 | In the new window run the following command to configure kubectl with EKS Cluster. Replace CLUSTER_NAME and REGION_NAME with the proper values. 41 | 42 | ```sh 43 | aws eks update-kubeconfig --name CLUSTER_NAME --region REGION_NAME 44 | ``` 45 | 46 | To destroy the cluster run the command below 47 | 48 | ```sh 49 | cdk destroy -c cluster_name=myfirstcluster --all 50 | ``` 51 | 52 | ### Advanced Deployment 53 | 54 | ##### Default Deploy 55 | 56 | ``` 57 | cdk deploy -c cluster_name=mycluster --all 58 | ``` 59 | 60 | ##### Deploy EKS Cluster Only 61 | 62 | ``` 63 | cdk deploy -c cluster_name=mycluster EKSStack 64 | ``` 65 | 66 | ##### Default Deploy with custom prefix for Cloudformation Stack 67 | 68 | ``` 69 | cdk deploy -c cluster_name=mycluster -c stack_prefix="dev-" --all 70 | ``` 71 | 72 | _All cdk stacks will contain prefix ex. dev-EKSStack dev-EKSK8sBaseline_ 73 | 74 | ##### Existing VPC 75 | 76 | Tag existing VPC with proper EKS Tags for ALB/NLB [auto discovery](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/deploy/subnet_discovery/) 77 | 78 | ``` 79 | cdk deploy -c use_vpc_id=vpc-1234567 -c cluster_name=mycluster --all 80 | ``` 81 | 82 | _Requires VPC to have private subnets_ 83 | 84 | ##### Custom Deployment 85 | 86 | ``` 87 | cdk deploy -c cluster_name=mycluster --all --parameter EKSK8sBaseline:ebsDriverParameter=true --parameter EKSK8sBaseline:albDriverParameter=false 88 | ``` 89 | 90 | _If using stack_prefix context, add prefix to parameters. ex dev-EKSK8sBaseline:ebsDriverParameter=true_ 91 | 92 | ### Configuration 93 | 94 | #### Available Parameters 95 | 96 | Parameters can be configured via Cloudformation [console](https://docs.aws.amazon.com/cdk/latest/guide/parameters.html#parameters_deploy) or via `cdk deploy --parameter STACK_NAME:PARAMETER_NAME=PARAMETER_VALUE` 97 | | Stack | Parameter | Default | Values | Description | 98 | | ------------- | ------------- | ------------- | ------------- | ------------- | 99 | | EKSStack | k8sVersion | 1.21 | [Valid Values](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html) | Kubernetes Version to deploy 100 | | EKSStack | eksLoggingOpts | api,audit,authenticator,
controllerManager,scheduler | [Valid Logging Types](https://docs.aws.amazon.com/eks/latest/APIReference/API_LogSetup.html#API_LogSetup_Contents) | List of options to enabled, "" to disable control plane logs | 101 | | EKSNodeGroups | nodegroupMax | 10 | Number | Max number of worker node to scale up for nodegroup 102 | | EKSNodeGroups | nodegroupCount | 2 | Number | Desired number of worker nodes for nodegroup | 103 | | EKSNodeGroups | nodegroupMin | 2 | Number | Min number of worker node to scale down for nodegroup 104 | | EKSNodeGroups | nodegroupInstanceType| t2.medium | String | Instance Type to be used with EKS Managed Nodegroup ng-1 105 | | EKSNodeGroups | nodeAMIVersion | 1.21.2-20210722 | [Valid Values](https://docs.aws.amazon.com/eks/latest/userguide/eks-linux-ami-versions.html#eks-al2-ami-versions) | AMI Release Version 106 | | EKSK8sBaseline | ebsDriver | false | true,false | Deploy EBS CSI Driver | 107 | | EKSK8sBaseline | albDriver | true | true,false | Deploy ALB Ingress Controller |Driver | 108 | | EKSK8sBaseline | efsDriver | false | true,false | Deploy EFS CSI Driver | 109 | | EKSK8sBaseline | fluentBit | true | true,false | Deploy FluentBit driver to output logs to centralized location 110 | | EKSK8sBaseline | secretsDriver | false | true,false | Deploy AWS Secret/Parameter CSI Driver | 111 | | EKSK8sBaseline | networkPolicyEngine | false | true,false | Deploy Calico Network Policy Engine | 112 | | EKSK8sBaseline | clusterAutoscaler | true | true,false | Deploy Cluster Autoscaler 113 | | EKSK8sBaseline | containerInsights | false | true,false | Deploy Container Insights using OpenTelemetry 114 | | EKSK8sBaseline | metricServer | true | true,false | Deploys Metric Server | 115 | 116 | #### Available Context 117 | 118 | Context can only be configured at synthesis time via `cdk deploy -c CONTEXT_NAME=CONTEXT_VALUE` or editing cdk.json file. 119 | 120 | | Context | Default | Values | Description | 121 | | ----------------------------------------- | ------------------ | --------------------------------- | ------------------------------------------------------------------- | 122 | | eks-addon-vpc-cni-version | v1.9.0-eksbuild.1 | AWS CLI\* | Amazon VPC Container Network Interface (CNI) plugin | 123 | | eks-addon-kube-proxy-version | v1.21.2-eksbuild.2 | AWS CLI\* | K8s Network Proxy | 124 | | eks-addon-coredns-version | v1.8.4-eksbuild.1 | AWS CLI\* | K8s Cluster DNS | 125 | | cluster-autoscaler-helm-version | 9.10.3 | [Values](#cluster-autoscaler) | Management of k8s worker node scaling | 126 | | aws-load-balancer-controller-helm-version | 1.2.3 | [Values](#alb-ingress-controller) | AWS Ingress Controller (ALB/NLB) | 127 | | aws-ebs-csi-driver-helm-version | 2.0.1 | [Values](#ebs-csi-driver) | EBS CSI Driver | 128 | | aws-efs-csi-driver-helm-version | 2.1.4 | [Values](#efs-csi-driver) | EFS CSI Driver | 129 | | secrets-store-csi-helm-version | 0.1.0 | [Values](#secrets-csi-driver) | Secrets CSI Driver | 130 | | aws-calico-helm-version | 0.3.5 | [Values](#network-policy-engine) | Calico Network Policy Engine | 131 | | cluster_name | myfirstcluster | String | Name of EKS Cluster | 132 | | use_vpc_id | | String | (Optional) Use existing VPC to deploy resources | 133 | | stack_prefix | | String | (Optional) Prefixes to add to underlying Cloudformation stack names | 134 | 135 | \*_Run `aws eks describe-addon-versions --kubernetes-version 1.21` to see full compatible list_ 136 | 137 | ### Best Practices 138 | 139 | Provides an example of EKS best practices in Infrastructure as Code 140 | 141 | - [x] [Make the EKS Cluster Endpoint private](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#make-the-eks-cluster-endpoint-private) 142 | - [x] [IAM Roles for Service Accounts(IRSA) for deployable addons (ex. ALB Ingress Controller)](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#kubernetes-service-accounts) 143 | - [x] [Update the aws-node daemonset to use IRSA via EKS AddOns](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#update-the-aws-node-daemonset-to-use-irsa) 144 | - [x] [Option to enable Audit Logs](https://aws.github.io/aws-eks-best-practices/security/docs/detective/#enable-audit-logs) 145 | - [x] [Option to enable network policies using Calico engine](https://aws.github.io/aws-eks-best-practices/security/docs/network/#network-policy) 146 | - [x] [Use AWS KMS for envelope encryption of kubernetes secrets](https://aws.github.io/aws-eks-best-practices/security/docs/data/#use-aws-kms-for-envelope-encryption-of-kubernetes-secrets) 147 | - [x] [Minimize access to worker node use SSM Manager](https://aws.github.io/aws-eks-best-practices/security/docs/hosts/#minimize-access-to-worker-nodes) 148 | - [x] [Deploy workers onto private subnets\*](https://aws.github.io/aws-eks-best-practices/security/docs/hosts/#deploy-workers-onto-private-subnets) 149 | - [x] [ECR Private Endpoints\*](https://aws.github.io/aws-eks-best-practices/security/docs/image/#consider-using-ecr-private-endpoints) 150 | - [x] [Option to enable Cluster Autoscaler](https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/#operating-the-cluster-autoscaler) 151 | - [x] [EKS Console Read Only Group called eks-console-dashboard-full-access-group](manifests/consoleViewOnlyGroup.yaml) 152 | - [x] [EBS Encryption of Worker and Bastion Instances](https://aws.github.io/aws-eks-best-practices/security/docs/network/#encryption-in-transit_1) 153 | 154 | \*_Use of vpc created by stack_ 155 | 156 | ### Advanced Configurations 157 | 158 | Helm Chart versions can be configured by editing the cdk.json before deployment. The versions can be listed by using helm commands. 159 | Example for Cluster Autoscaler: 160 | 161 | ``` 162 | helm repo add autoscaler https://kubernetes.github.io/autoscaler 163 | helm search repo autoscaler --versions 164 | ``` 165 | 166 | Another way to identity Helm Chart versions is looking at that index.yaml of Chart repositories for version information. 167 | 168 | #### Cluster Autoscaler - [Scalability](https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/) 169 | 170 | Deploys Cluster Autoscaler chart version 9.10.3 with app version 1.21.0 using [Helm chart](https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler). A full list of chart versions with app versions can be found [here](https://github.com/kubernetes/autoscaler/blob/gh-pages/index.yaml) 171 | 172 | Default configuration below, additional configurations can be edited in [k8s-baseline.ts](lib/k8s-baseline.ts): 173 | 174 | ``` 175 | extraArgs: { 176 | // https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca 177 | 'skip-nodes-with-system-pods': false, 178 | 'skip-nodes-with-local-storage': false, 179 | 'balance-similar-node-groups': true, 180 | //How long a node should be unneeded before it is eligible for scale down 181 | 'scale-down-unneeded-time' : '300s', 182 | //How long after scale up that scale down evaluation resumes 183 | 'scale-down-delay-after-add':'300s' 184 | } 185 | ``` 186 | 187 | #### ALB Ingress Controller - [Scalability](https://aws.github.io/aws-eks-best-practices/security/docs/network/#ingress-controllers-and-load-balancers) 188 | 189 | Deploys ALB Ingress Controller chart version 1.2.3 with app version 2.2.1 using [Helm](https://github.com/aws/eks-charts/tree/master/stable/aws-load-balancer-controller). A full list of chart versions with app versions can be found [here](https://github.com/aws/eks-charts/blob/gh-pages/index.yaml) 190 | 191 | #### EBS CSI Driver - [Persistent Data](https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/#ebs-volumes) 192 | 193 | Deploys EBS CSI Driver Chart version 2.0.1 with app version 1.1.1 using [Helm](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/charts/aws-ebs-csi-driver). A full list of chart versions with app versions can be found [here](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/gh-pages/index.yaml) 194 | 195 | Needs user to deploy EBS StorageClass for use with pods. Example steps in EBS CSI Driver [docs](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/examples/kubernetes/dynamic-provisioning) 196 | 197 | #### EFS CSI Driver - [Persistent Data](https://aws.github.io/aws-eks-best-practices/security/docs/data/#use-efs-access-points-to-simplify-access-to-shared-datasets) 198 | 199 | Deploys EFS CSI Driver chart version 2.1.4 with app version 1.3.2 using [Helm](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/charts/aws-efs-csi-driver). A full list of chart versions with app versions can be found [here](https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/gh-pages/index.yaml) 200 | 201 | Needs user to create EFS Volume and provision a storageclass for use with pods. Example steps in EFS CSI Driver [docs](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/examples/kubernetes/dynamic_provisioning) 202 | 203 | #### Secrets CSI Driver - [Security](https://aws.github.io/aws-eks-best-practices/security/docs/data/#secrets-management) 204 | 205 | Deploys the Secrets CSI Driver chart version 0.1.0 with app version 0.1.0 using [Helm](https://github.com/kubernetes-sigs/secrets-store-csi-driver/tree/master/charts/secrets-store-csi-driver).A full list of chart versions with app versions can be found [here](https://github.com/kubernetes-sigs/secrets-store-csi-driver/blob/master/charts/index.yaml) 206 | 207 | Requires pods to have ServiceAccount with proper permissions to get values from AWS Secrets Manager or AWS Parameter Store. Example steps in AWS Blog [post](https://aws.amazon.com/blogs/security/how-to-use-aws-secrets-configuration-provider-with-kubernetes-secrets-store-csi-driver/), continue from step 4. 208 | 209 | Automated rotation for the driver using the rotation reconciler is turned off by default. Uncomment the below lines in `secretsCsiHelmChart` [k8s-baseline.ts](lib/k8s-baseline.ts) 210 | 211 | ```js 212 | values: { 213 | grpcSupportedProviders: 'aws', 214 | // alpha feature reconciler feature 215 | rotationPollInterval: 3600 216 | enableSecretRotation: true 217 | }, 218 | ``` 219 | 220 | #### Network Policy Engine - [Security](https://aws.github.io/aws-eks-best-practices/security/docs/network/#network-policy) 221 | 222 | Deploys Calico Network Policy Engine chart version 0.3.5 with app version 1.15.1 using [Helm](https://github.com/aws/eks-charts/tree/master/stable/aws-calico). 223 | 224 | Increases default cpu/memory limits to be 225 | 226 | ```js 227 | resources: { 228 | limits: { 229 | memory: '256Mi', 230 | cpu: '500m', 231 | }, 232 | }, 233 | ``` 234 | 235 | #### Container Insights - [Observability](https://aws.github.io/aws-eks-best-practices/reliability/docs/application/#observability) 236 | 237 | Deploys Container Insights on Amazon EKS Cluster by using the AWS Distro for OpenTelemetry (collectoramazon/aws-otel-collector:[v0.11.0](https://hub.docker.com/r/amazon/aws-otel-collector)). The visualized metrics can be found on [Cloudwatch -> Insights -> Container Insights](https://console.aws.amazon.com/cloudwatch/home#container-insights:infrastructure). The logs can be found at /aws/containerinsights/CLUSTER_NAME/performance. List of available metrics that can be collected available [here](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Container-Insights-metrics-EKS.html) 238 | 239 | EKS Pod Metric Example: 240 | ![](images/container_insights_1.png) 241 | 242 | Configurations are applied via [manifest](manifests/otelContainerInsights.yaml), edits can be made following the [documentation](https://aws-otel.github.io/docs/getting-started/container-insights/eks-infra#advanced-usage) 243 | 244 | #### FluentBit - [Observability](https://aws.github.io/aws-eks-best-practices/reliability/docs/application/#use-centralized-logging-tools-to-collect-and-persist-logs) 245 | 246 | Deploys FluentBit daemonset using manifests. The configuration is found in the [manifest](manifests/fluentBitSetup.yaml). The FluentBit conf files are stored in ConfigMap k8s definition. Additional configuration like log_retention_days, log_format, role_arn(cross-account) can be [configured](https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch#configuration-parameters) by editing conf files. 247 | 248 | ##### Service Account IAM Policy 249 | 250 | IAM permission to read/write permission to log group: `/aws/containerinsights/${CLUSTER_NAME}`. Editable in [k8s-baseline.ts](lib/k8s-baseline.ts) 251 | 252 | ``` 253 | { 254 | "Version": "2012-10-17", 255 | "Statement": [ 256 | { 257 | "Action": [ 258 | "logs:CreateLogStream", 259 | "logs:CreateLogGroup", 260 | "logs:DescribeLogStreams", 261 | "logs:PutLogEvents" 262 | ], 263 | "Resource": [ 264 | "arn:aws:logs:${REGION}:${ACCOUNT_ID}:log-group:/aws/containerinsights/${CLUSTER_NAME}/*", 265 | "arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/containerinsights/${CLUSTER_NAME}/*:log-stream:*" 266 | ], 267 | "Effect": "Allow" 268 | } 269 | ] 270 | } 271 | ``` 272 | 273 | ##### application-log.conf 274 | 275 | Include Logs from: 276 | 277 | ``` 278 | /var/log/containers/*.log 279 | ``` 280 | 281 | Exclude Logs from: 282 | 283 | ``` 284 | /var/log/containers/aws-node* 285 | /var/log/containers/kube-proxy* 286 | ``` 287 | 288 | Output to: 289 | 290 | ``` 291 | Cloudwatch Log group /aws/containerinsights/${CLUSTER_NAME}/application 292 | ``` 293 | 294 | ##### dataplane-log.conf 295 | 296 | Include Logs from: 297 | 298 | ``` 299 | systemd:docker.service 300 | systemd:kubelet.service 301 | /var/log/containers/aws-node* 302 | /var/log/containers/kube-proxy* 303 | ``` 304 | 305 | Exclude Logs from: 306 | 307 | ``` 308 | N/A 309 | ``` 310 | 311 | Output to: 312 | 313 | ``` 314 | Cloudwatch Log group /aws/containerinsights/${CLUSTER_NAME}/dataplane 315 | ``` 316 | 317 | ##### host-log.conf 318 | 319 | Include Logs from: 320 | 321 | ``` 322 | /var/log/secure 323 | /var/log/messages 324 | /var/log/dmesg 325 | ``` 326 | 327 | Exclude Logs from: 328 | 329 | ``` 330 | N/A 331 | ``` 332 | 333 | Output to: 334 | 335 | ``` 336 | Cloudwatch Log group /aws/containerinsights/${CLUSTER_NAME}/host 337 | ``` 338 | 339 | ### Next Steps 340 | 341 | A list of recommendations to further enhance the environment. 342 | 343 | ##### Least Privileged Access - [Security](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#employ-least-privileged-access-when-creating-rolebindings-and-clusterrolebindings) 344 | 345 | ###### ReadOnly K8s Access 346 | 347 | A readonly K8s Group is created called `eks-console-dashboard-full-access-group`. Adding the below code snippet to the sample will add an existing Role to view `eks-console-dashboard-full-access-group` RBAC group. See [eks-cdk-js-stack.ts](lib/eks-cdk-js-stack.ts) for example. 348 | 349 | ```js 350 | // Add existing IAM Role to Custom Group 351 | this.awsauth.addRoleMapping( 352 | Role.fromRoleArn( 353 | this, 354 | "Role_Admin", 355 | `arn:aws:iam::${this.account}:role/Admin` 356 | ), 357 | { 358 | groups: ["eks-console-dashboard-full-access-group"], 359 | username: `arn:aws:iam::${this.account}:role/Admin/{{SessionName}}`, 360 | } 361 | ); 362 | ``` 363 | 364 | ```js 365 | // Add existing IAM User to Custom Group 366 | this.awsauth.addUserMapping( 367 | User.fromUserArn( 368 | this, 369 | "Role_Admin", 370 | `arn:aws:iam::${this.account}:user/Admin` 371 | ), 372 | { 373 | groups: ["eks-console-dashboard-full-access-group"], 374 | username: `arn:aws:iam::${this.account}:user/Admin`, 375 | } 376 | ); 377 | ``` 378 | 379 | The error below can be resolved by adding the iam user/role to `eks-console-dashboard-full-access-group` k8s group. 380 | 381 | ![](images/console_error.png) 382 | 383 | ###### Bastion Host Permissions - [Security](https://aws.github.io/aws-eks-best-practices/security/docs/hosts/#minimize-access-to-worker-nodes) 384 | 385 | The sample deploys a bastion host with outbound to `0.0.0.0` with outbound access on port `443`. The IAM role associated with the ec2 instance is registered as Admin role for EKS Cluster. SSM is able to log session data to Cloudwatch Logs or S3, see [docs](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-logging.html) for more details. The end user permissions can be scoped to specific instance [ids](https://docs.aws.amazon.com/systems-manager/latest/userguide/getting-started-restrict-access-examples.html#restrict-access-example-instances) or [tags](https://docs.aws.amazon.com/systems-manager/latest/userguide/getting-started-restrict-access-examples.html#restrict-access-example-instance-tags). 386 | 387 | ###### Secrets CSI Driver - [Security]() 388 | 389 | Each pods requires a Service Account with IAM Role to access AWS Secrets Manager or AWS Parameter Store resources. 390 | 391 | Sample permissions scoped to Secrets/Parameters with tags ekscluster:myclustername for dynamic access permissions. 392 | 393 | ```js 394 | const secretsManagerPolicyStatement = new PolicyStatement({ 395 | resources: ["*"], 396 | actions: ["secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret"], 397 | conditions: { 398 | StringEquals: { 399 | "secretsmanager:ResourceTag/ekscluster": `${props.eksCluster.clusterName}`, 400 | }, 401 | }, 402 | }); 403 | const parameterManagerPolicyStatement = new PolicyStatement({ 404 | resources: ["*"], 405 | actions: ["ssm:GetParameters"], 406 | conditions: { 407 | StringEquals: { 408 | "ssm:ResourceTag/ekscluster": `${props.eksCluster.clusterName}`, 409 | }, 410 | }, 411 | }); 412 | const awsSA = new ServiceAccount(this, "awsSA", { 413 | cluster: props.eksCluster, 414 | name: "my-deployment", 415 | namespace: "default", 416 | }); 417 | const secretsManagerPolicy = new Policy(this, "secrets-manager-policy", { 418 | statements: [parameterManagerPolicyStatement, secretsManagerPolicyStatement], 419 | policyName: `${props.eksCluster.clusterName}-secrets`, 420 | roles: [awsSA.role], 421 | }); 422 | ``` 423 | 424 | ###### IMDSv2 Configuration - [Security](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#when-your-application-needs-access-to-idms-use-imdsv2-and-increase-the-hop-limit-on-ec2-instances-to-2) 425 | 426 | The sample deploys Managed Node group with IMDSv1 and IMDSv2 enabled with hop count: 2 to ensure broadly levels of compatibility. 427 | 428 | [Current config](lib/k8s-nodegroup.ts): 429 | 430 | ```js 431 | // Not all components are IMDSv2 aware. Ex. Fluentbit 432 | metadataOptions: { 433 | httpTokens: 'optional', 434 | httpPutResponseHopLimit: 2, 435 | 436 | }, 437 | ``` 438 | 439 | To enforce IMDSv2 only, change to below code. 440 | 441 | ```js 442 | metadataOptions: { 443 | httpTokens: 'required', 444 | httpPutResponseHopLimit: 1, 445 | }, 446 | ``` 447 | 448 | ###### FluentBit Log Retention - Cost Optimization 449 | 450 | The configuration for the FluentBit is set to create LogGroup and LogStreams with log retention=forever. The full set of options can be found [here](https://docs.fluentbit.io/manual/pipeline/outputs/cloudwatch). 451 | 452 | A sample output snippet for cross-account and log retention days (30 days). 453 | 454 | ```ApacheConf 455 | [OUTPUT] 456 | Name cloudwatch_logs 457 | Match application.* 458 | region ${AWS_REGION} 459 | log_group_name /aws/containerinsights/${CLUSTER_NAME}/application 460 | log_stream_prefix ${HOST_NAME}- 461 | auto_create_group true 462 | extra_user_agent container-insights 463 | log_retention_days 30 464 | role_arn arn:aws:iam::123456789:role/role-name 465 | ``` 466 | 467 | ## Security 468 | 469 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 470 | 471 | ## License 472 | 473 | This library is licensed under the MIT-0 License. See the LICENSE file. 474 | -------------------------------------------------------------------------------- /THIRD-PARTY-LICENSES: -------------------------------------------------------------------------------- 1 | ** aws-secrets-store https://github.com/aws/secrets-store-csi-driver-provider-aws 2 | ** otel-collector https://github.com/aws-observability/aws-otel-collector/ 3 | ** metric-server https://github.com/kubernetes-sigs/metrics-server/ 4 | Apache License 5 | Version 2.0, January 2004 6 | http://www.apache.org/licenses/ 7 | 8 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 9 | 10 | 1. Definitions. 11 | 12 | "License" shall mean the terms and conditions for use, reproduction, 13 | and distribution as defined by Sections 1 through 9 of this document. 14 | 15 | "Licensor" shall mean the copyright owner or entity authorized by 16 | the copyright owner that is granting the License. 17 | 18 | "Legal Entity" shall mean the union of the acting entity and all 19 | other entities that control, are controlled by, or are under common 20 | control with that entity. For the purposes of this definition, 21 | "control" means (i) the power, direct or indirect, to cause the 22 | direction or management of such entity, whether by contract or 23 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 24 | outstanding shares, or (iii) beneficial ownership of such entity. 25 | 26 | "You" (or "Your") shall mean an individual or Legal Entity 27 | exercising permissions granted by this License. 28 | 29 | "Source" form shall mean the preferred form for making modifications, 30 | including but not limited to software source code, documentation 31 | source, and configuration files. 32 | 33 | "Object" form shall mean any form resulting from mechanical 34 | transformation or translation of a Source form, including but 35 | not limited to compiled object code, generated documentation, 36 | and conversions to other media types. 37 | 38 | "Work" shall mean the work of authorship, whether in Source or 39 | Object form, made available under the License, as indicated by a 40 | copyright notice that is included in or attached to the work 41 | (an example is provided in the Appendix below). 42 | 43 | "Derivative Works" shall mean any work, whether in Source or Object 44 | form, that is based on (or derived from) the Work and for which the 45 | editorial revisions, annotations, elaborations, or other modifications 46 | represent, as a whole, an original work of authorship. For the purposes 47 | of this License, Derivative Works shall not include works that remain 48 | separable from, or merely link (or bind by name) to the interfaces of, 49 | the Work and Derivative Works thereof. 50 | 51 | "Contribution" shall mean any work of authorship, including 52 | the original version of the Work and any modifications or additions 53 | to that Work or Derivative Works thereof, that is intentionally 54 | submitted to Licensor for inclusion in the Work by the copyright owner 55 | or by an individual or Legal Entity authorized to submit on behalf of 56 | the copyright owner. For the purposes of this definition, "submitted" 57 | means any form of electronic, verbal, or written communication sent 58 | to the Licensor or its representatives, including but not limited to 59 | communication on electronic mailing lists, source code control systems, 60 | and issue tracking systems that are managed by, or on behalf of, the 61 | Licensor for the purpose of discussing and improving the Work, but 62 | excluding communication that is conspicuously marked or otherwise 63 | designated in writing by the copyright owner as "Not a Contribution." 64 | 65 | "Contributor" shall mean Licensor and any individual or Legal Entity 66 | on behalf of whom a Contribution has been received by Licensor and 67 | subsequently incorporated within the Work. 68 | 69 | 2. Grant of Copyright License. Subject to the terms and conditions of 70 | this License, each Contributor hereby grants to You a perpetual, 71 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 72 | copyright license to reproduce, prepare Derivative Works of, 73 | publicly display, publicly perform, sublicense, and distribute the 74 | Work and such Derivative Works in Source or Object form. 75 | 76 | 3. Grant of Patent License. Subject to the terms and conditions of 77 | this License, each Contributor hereby grants to You a perpetual, 78 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 79 | (except as stated in this section) patent license to make, have made, 80 | use, offer to sell, sell, import, and otherwise transfer the Work, 81 | where such license applies only to those patent claims licensable 82 | by such Contributor that are necessarily infringed by their 83 | Contribution(s) alone or by combination of their Contribution(s) 84 | with the Work to which such Contribution(s) was submitted. If You 85 | institute patent litigation against any entity (including a 86 | cross-claim or counterclaim in a lawsuit) alleging that the Work 87 | or a Contribution incorporated within the Work constitutes direct 88 | or contributory patent infringement, then any patent licenses 89 | granted to You under this License for that Work shall terminate 90 | as of the date such litigation is filed. 91 | 92 | 4. Redistribution. You may reproduce and distribute copies of the 93 | Work or Derivative Works thereof in any medium, with or without 94 | modifications, and in Source or Object form, provided that You 95 | meet the following conditions: 96 | 97 | (a) You must give any other recipients of the Work or 98 | Derivative Works a copy of this License; and 99 | 100 | (b) You must cause any modified files to carry prominent notices 101 | stating that You changed the files; and 102 | 103 | (c) You must retain, in the Source form of any Derivative Works 104 | that You distribute, all copyright, patent, trademark, and 105 | attribution notices from the Source form of the Work, 106 | excluding those notices that do not pertain to any part of 107 | the Derivative Works; and 108 | 109 | (d) If the Work includes a "NOTICE" text file as part of its 110 | distribution, then any Derivative Works that You distribute must 111 | include a readable copy of the attribution notices contained 112 | within such NOTICE file, excluding those notices that do not 113 | pertain to any part of the Derivative Works, in at least one 114 | of the following places: within a NOTICE text file distributed 115 | as part of the Derivative Works; within the Source form or 116 | documentation, if provided along with the Derivative Works; or, 117 | within a display generated by the Derivative Works, if and 118 | wherever such third-party notices normally appear. The contents 119 | of the NOTICE file are for informational purposes only and 120 | do not modify the License. You may add Your own attribution 121 | notices within Derivative Works that You distribute, alongside 122 | or as an addendum to the NOTICE text from the Work, provided 123 | that such additional attribution notices cannot be construed 124 | as modifying the License. 125 | 126 | You may add Your own copyright statement to Your modifications and 127 | may provide additional or different license terms and conditions 128 | for use, reproduction, or distribution of Your modifications, or 129 | for any such Derivative Works as a whole, provided Your use, 130 | reproduction, and distribution of the Work otherwise complies with 131 | the conditions stated in this License. 132 | 133 | 5. Submission of Contributions. Unless You explicitly state otherwise, 134 | any Contribution intentionally submitted for inclusion in the Work 135 | by You to the Licensor shall be under the terms and conditions of 136 | this License, without any additional terms or conditions. 137 | Notwithstanding the above, nothing herein shall supersede or modify 138 | the terms of any separate license agreement you may have executed 139 | with Licensor regarding such Contributions. 140 | 141 | 6. Trademarks. This License does not grant permission to use the trade 142 | names, trademarks, service marks, or product names of the Licensor, 143 | except as required for reasonable and customary use in describing the 144 | origin of the Work and reproducing the content of the NOTICE file. 145 | 146 | 7. Disclaimer of Warranty. Unless required by applicable law or 147 | agreed to in writing, Licensor provides the Work (and each 148 | Contributor provides its Contributions) on an "AS IS" BASIS, 149 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 150 | implied, including, without limitation, any warranties or conditions 151 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 152 | PARTICULAR PURPOSE. You are solely responsible for determining the 153 | appropriateness of using or redistributing the Work and assume any 154 | risks associated with Your exercise of permissions under this License. 155 | 156 | 8. Limitation of Liability. In no event and under no legal theory, 157 | whether in tort (including negligence), contract, or otherwise, 158 | unless required by applicable law (such as deliberate and grossly 159 | negligent acts) or agreed to in writing, shall any Contributor be 160 | liable to You for damages, including any direct, indirect, special, 161 | incidental, or consequential damages of any character arising as a 162 | result of this License or out of the use or inability to use the 163 | Work (including but not limited to damages for loss of goodwill, 164 | work stoppage, computer failure or malfunction, or any and all 165 | other commercial damages or losses), even if such Contributor 166 | has been advised of the possibility of such damages. 167 | 168 | 9. Accepting Warranty or Additional Liability. While redistributing 169 | the Work or Derivative Works thereof, You may choose to offer, 170 | and charge a fee for, acceptance of support, warranty, indemnity, 171 | or other liability obligations and/or rights consistent with this 172 | License. However, in accepting such obligations, You may act only 173 | on Your own behalf and on Your sole responsibility, not on behalf 174 | of any other Contributor, and only if You agree to indemnify, 175 | defend, and hold each Contributor harmless for any liability 176 | incurred by, or claims asserted against, such Contributor by reason 177 | of your accepting any such warranty or additional liability. 178 | 179 | ** eks-user-guide https://github.com/awsdocs/amazon-eks-user-guide 180 | Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 181 | ** amazon cloudwatch insights https://github.com/aws-samples/amazon-cloudwatch-container-insights 182 | Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. 183 | 184 | Permission is hereby granted, free of charge, to any person obtaining a copy of this 185 | software and associated documentation files (the "Software"), to deal in the Software 186 | without restriction, including without limitation the rights to use, copy, modify, 187 | merge, publish, distribute, sublicense, and/or sell copies of the Software, and to 188 | permit persons to whom the Software is furnished to do so. 189 | 190 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, 191 | INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 192 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 193 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 194 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 195 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 196 | -------------------------------------------------------------------------------- /bin/eks-cdk-js.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { App } from 'aws-cdk-lib'; 3 | import { Construct } from 'constructs'; 4 | import { Ekstack } from '../lib/eks-cdk-js-stack'; 5 | 6 | import { K8sBaselineStack } from '../lib/k8s-baseline'; 7 | import { K8snodegroups } from '../lib/k8s-nodegroup'; 8 | 9 | const DEFAULT_CONFIG = { 10 | env: { 11 | account: process.env.CDK_DEFAULT_ACCOUNT, 12 | region: process.env.CDK_DEFAULT_REGION, 13 | }, 14 | }; 15 | 16 | const app = new App(); 17 | const prefix = stackPrefix(app); 18 | const eks = new Ekstack(app, 'EKSStack', ({ 19 | env: DEFAULT_CONFIG.env, 20 | stackName: `${prefix}EKSStack`, 21 | })); 22 | 23 | const nodegroups = new K8snodegroups(app, 'EKSNodeGroups', ({ 24 | env: DEFAULT_CONFIG.env, 25 | stackName: `${prefix}EKSNodeGroups`, 26 | eksCluster: eks.cluster, 27 | nodeGroupRole: eks.createNodegroupRole('nodeGroup1'), 28 | })); 29 | 30 | const k8sbase = new K8sBaselineStack(app, 'EKSK8sBaseline', ({ 31 | env: DEFAULT_CONFIG.env, 32 | stackName: `${prefix}EKSK8sBaseline`, 33 | eksCluster: eks.cluster, 34 | 35 | })); 36 | 37 | k8sbase.addDependency(nodegroups); 38 | nodegroups.addDependency(eks); 39 | 40 | function stackPrefix (stack: Construct): string { 41 | const prefixValue = stack.node.tryGetContext('stack_prefix'); 42 | 43 | if (prefixValue !== undefined) { 44 | return prefixValue.trim(); 45 | } 46 | // if no stack_prefix return empty string 47 | return ''; 48 | } 49 | app.synth(); 50 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/eks-cdk-js.ts", 3 | "context": { 4 | "eks-addon-vpc-cni-version": "v1.7.5-eksbuild.2", 5 | "eks-addon-kube-proxy-version": "v1.21.2-eksbuild.2", 6 | "eks-addon-coredns-version": "v1.8.4-eksbuild.1", 7 | "cluster-autoscaler-helm-version": "9.10.3", 8 | "aws-load-balancer-controller-helm-version": "1.2.3", 9 | "aws-ebs-csi-driver-helm-version": "2.0.4", 10 | "aws-efs-csi-driver-helm-version": "2.1.4", 11 | "secrets-store-csi-helm-version": "0.1.0", 12 | "aws-calico-helm-version": "0.3.5", 13 | "cluster_name" : "myfirstcluster" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /images/bastion_connect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-eks-using-cdk-typescript/c5bef24686c5d7dad0e975dbbe4febc40ed29b50/images/bastion_connect.png -------------------------------------------------------------------------------- /images/bastion_ssm_connect.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-eks-using-cdk-typescript/c5bef24686c5d7dad0e975dbbe4febc40ed29b50/images/bastion_ssm_connect.png -------------------------------------------------------------------------------- /images/console_error.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-eks-using-cdk-typescript/c5bef24686c5d7dad0e975dbbe4febc40ed29b50/images/console_error.png -------------------------------------------------------------------------------- /images/container_insights_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/amazon-eks-using-cdk-typescript/c5bef24686c5d7dad0e975dbbe4febc40ed29b50/images/container_insights_1.png -------------------------------------------------------------------------------- /lib/custom-resources/eksloggingOnEvent.py: -------------------------------------------------------------------------------- 1 | import time 2 | import boto3,botocore 3 | import json 4 | import logging 5 | 6 | import botocore 7 | boto3.set_stream_logger('botocore.waiter', logging.DEBUG) 8 | 9 | client = boto3.client('eks') 10 | 11 | ALL_LOGGING_OPTS = [ 'api','audit','authenticator','controllerManager','scheduler'] 12 | 13 | def on_event(event, context): 14 | print(json.dumps(event)) 15 | request_type = event['RequestType'] 16 | if request_type == 'Create': return on_create(event) 17 | if request_type == 'Update': return on_update(event) 18 | if request_type == 'Delete': return on_delete(event) 19 | raise Exception("Invalid request type: %s" % request_type) 20 | 21 | def on_create(event): 22 | props = event["ResourceProperties"] 23 | print("Update %s with logging configuration %s" % (props['eksCluster'], props['loggingOpts'])) 24 | try: 25 | currentLoggingOpts = { 26 | 'enabled': [], 27 | 'disabled': [] 28 | } 29 | eks_info = client.describe_cluster( 30 | name=props['eksCluster'] 31 | ) 32 | for item in eks_info['cluster']['logging']['clusterLogging']: 33 | if (item['enabled']== True): 34 | currentLoggingOpts['enabled'] = item 35 | elif (item['enabled'] == False): 36 | currentLoggingOpts['disabled'] = item 37 | else: 38 | print("Unable to parse current eks logging configuration %s" % eks_info['cluster']['logging']['clusterLogging']) 39 | raise ValueError() 40 | updateLogOpts=log_config_diff(currentLoggingOpts,props['loggingOpts']) 41 | print(updateLogOpts) 42 | eks=client.update_cluster_config( 43 | name= props['eksCluster'], 44 | logging={ 45 | 'clusterLogging': updateLogOpts 46 | }) 47 | print("%s" % eks) 48 | except client.exceptions.InvalidParameterException as i: 49 | if ('No changes needed for the logging config provided' == i.response['Error']['Message']): 50 | print('No changes needed for the logging config provided, current config is valid, skipping action returning success') 51 | pass 52 | except Exception as e: 53 | print(e) 54 | raise 55 | return {'Status': 'SUCCESS', 'PhysicalResourceId': props['eksCluster'], 'StackId': event['StackId'], 'LogicalResourceId': event['LogicalResourceId']} 56 | 57 | def on_update(event): 58 | physical_id = event["PhysicalResourceId"] 59 | props = event["ResourceProperties"] 60 | print("update resource %s with props %s" % (physical_id, props)) 61 | on_create(event) 62 | 63 | def on_delete(event): 64 | physical_id = event["PhysicalResourceId"] 65 | print("Skip delete since eks cluster will be deleted %s" % physical_id) 66 | return {'Status': 'SUCCESS', 'PhysicalResourceId': physical_id, 'StackId': event['StackId'], 'LogicalResourceId': event['LogicalResourceId']} 67 | 68 | 69 | 70 | def is_complete(event,context): 71 | print(event) 72 | props = event["ResourceProperties"] 73 | print("Starting Waiter") 74 | # Sleep for 30 seconds before checking for cluster status to change 75 | time.sleep(30) 76 | print("Waited 30 seconds") 77 | try: 78 | waiter = client.get_waiter('cluster_active') 79 | waiter.wait( 80 | name=props['eksCluster'], 81 | WaiterConfig={ 82 | 'Delay': 10, 83 | 'MaxAttempts' : 40 84 | } 85 | ) 86 | eks_info = client.describe_cluster( 87 | name=props['eksCluster'] 88 | ) 89 | print(eks_info['cluster']['logging']) 90 | print("Completed wait") 91 | except botocore.exceptions.WaiterError as e: 92 | print(e) 93 | if "Max attempts exceeded" in e.message: 94 | return {'IsComplete': False} 95 | except Exception as e: 96 | print(e) 97 | raise 98 | 99 | return { 'IsComplete': True } 100 | 101 | 102 | def log_config_diff(currentOpts,newOpts): 103 | print ("Evaluate for updated logging to be enabled") 104 | if "" in newOpts: 105 | newOpts.remove("") 106 | if 'types' not in currentOpts['enabled']: 107 | currentOpts['enabled'] = {'types': []} 108 | 109 | enabledOpts = set(newOpts) - set(currentOpts['enabled']['types']) 110 | print("Need \"%s\" logging to be enabled" % list(enabledOpts)) 111 | disabledOpts = set(ALL_LOGGING_OPTS) - set(newOpts) 112 | print ("Need \"%s\" logging to be disabled" % list(disabledOpts)) 113 | return [{ 114 | 'types': list(enabledOpts), 115 | 'enabled': True 116 | }, 117 | { 118 | 'types': list(disabledOpts), 119 | 'enabled': False 120 | }] 121 | 122 | if __name__ == "__main__": 123 | jsonObj = """{ 124 | "RequestType": "Create", 125 | "ServiceToken": "EXAMPLE", 126 | "ResponseURL": "EXAMPLE", 127 | "StackId": "arn:aws:cloudformation:us-west-2:ACCOUNT_ID:stack/EKS_STACK/EXAMPLE", 128 | "RequestId": "EXAMPLE", 129 | "LogicalResourceId": "eksLoggingCustomResource", 130 | "ResourceType": "AWS::CloudFormation::CustomResource", 131 | "ResourceProperties": { 132 | "ServiceToken": "test", 133 | "eksCluster": "myekscluster", 134 | "enable": "true", 135 | "loggingOpts": [ "" 136 | ] 137 | } 138 | }""" 139 | print(on_event(json.loads(jsonObj),'')) 140 | print(is_complete(json.loads(jsonObj),'')) -------------------------------------------------------------------------------- /lib/eks-cdk-js-stack.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | import * as path from 'path'; 3 | import * as cdk from 'aws-cdk-lib'; 4 | import { CfnParameter, CustomResource, Duration, CfnJson } from 'aws-cdk-lib'; 5 | import { Vpc, IVpc, InstanceType, Port, BlockDeviceVolume, EbsDeviceVolumeType, Instance, MachineImage, AmazonLinuxGeneration, SecurityGroup, Peer } from 'aws-cdk-lib/aws-ec2'; 6 | import { AwsAuth, Cluster, EndpointAccess, KubernetesVersion, KubernetesManifest, CfnAddon } from 'aws-cdk-lib/aws-eks'; 7 | import { PolicyStatement, Effect, Role, ManagedPolicy, ServicePrincipal, OpenIdConnectPrincipal } from 'aws-cdk-lib/aws-iam'; 8 | import { Key } from 'aws-cdk-lib/aws-kms'; 9 | import { Runtime, SingletonFunction, Code } from 'aws-cdk-lib/aws-lambda'; 10 | import { RetentionDays } from 'aws-cdk-lib/aws-logs'; 11 | import { Provider } from 'aws-cdk-lib/custom-resources'; 12 | import { Construct } from 'constructs'; 13 | import * as yaml from 'js-yaml'; 14 | 15 | import { eksVpc, addEndpoint } from '../lib/vpc-stack'; 16 | 17 | interface ekstackprops extends cdk.StackProps { 18 | } 19 | 20 | export class Ekstack extends cdk.Stack { 21 | public readonly cluster: Cluster 22 | public readonly awsauth: AwsAuth 23 | 24 | constructor(scope: Construct, id: string, props: ekstackprops) { 25 | super(scope, id, props); 26 | // Clusters can only be upgraded and cannot be downgraded. Nodegroups are updated separately, refer to nodeAMIVersion parameter in README.md 27 | // https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html 28 | const k8sversion = new CfnParameter(this, 'k8sVersion', { 29 | type: 'String', 30 | description: 'K8s Version', 31 | default: '1.21', 32 | }); 33 | // https://github.com/aws/aws-cdk/issues/4159 34 | // https://aws.github.io/aws-eks-best-practices/security/docs/detective/#enable-audit-logs 35 | const eksLoggingOpts = new CfnParameter(this, 'eksLoggingOpts', { 36 | type: 'CommaDelimitedList', 37 | default: 'api,audit,authenticator,controllerManager,scheduler', 38 | description: 'EKS Logging values,leave empty to disable, options https://docs.aws.amazon.com/eks/latest/APIReference/API_LogSetup.html#AmazonEKS-Type-LogSetup-types,', 39 | }); 40 | 41 | const vpc = this.getOrCreateVpc(this); 42 | // Locked Down Bastion Host Security Group to only allow outbound access to port 443. 43 | const bastionHostLinuxSecurityGroup = new SecurityGroup(this, 'bastionHostSecurityGroup', { 44 | allowAllOutbound: false, 45 | securityGroupName: this.getOrCreateEksName(this) + '-bastionSecurityGroup', 46 | vpc: vpc, 47 | }); 48 | // Recommended to use connections to manage ingress/egress for security groups 49 | bastionHostLinuxSecurityGroup.connections.allowTo(Peer.anyIpv4(), Port.tcp(443), 'Outbound to 443 only'); 50 | // Create Custom IAM Role and Policies for Bastion Host 51 | // https://docs.aws.amazon.com/eks/latest/userguide/security_iam_id-based-policy-examples.html#policy_example3 52 | const bastionHostPolicy = new ManagedPolicy(this, 'bastionHostManagedPolicy'); 53 | bastionHostPolicy.addStatements(new PolicyStatement({ 54 | resources: ['*'], 55 | actions: [ 56 | 'eks:DescribeNodegroup', 57 | 'eks:ListNodegroups', 58 | 'eks:DescribeCluster', 59 | 'eks:ListClusters', 60 | 'eks:AccessKubernetesApi', 61 | 'eks:ListUpdates', 62 | 'eks:ListFargateProfiles', 63 | ], 64 | effect: Effect.ALLOW, 65 | sid: 'EKSReadonly', 66 | })); 67 | const bastionHostRole = new Role(this, 'bastionHostRole', { 68 | roleName: this.getOrCreateEksName(this) + '-bastion-host', 69 | assumedBy: new ServicePrincipal('ec2.amazonaws.com'), 70 | managedPolicies: [ 71 | // SSM Manager Permissions 72 | ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), 73 | // Read only EKS Permissions 74 | bastionHostPolicy, 75 | ], 76 | }); 77 | // Create Bastion Host, connect using Session Manager 78 | const bastionHostLinux = new Instance(this, 'BastionEKSHost', { 79 | // Defaults to private subnets https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-ec2.Instance.html#vpcsubnets 80 | vpc: vpc, 81 | instanceName: this.getOrCreateEksName(this) + '-EKSBastionHost', 82 | instanceType: new InstanceType('t3.small'), 83 | // Always use Latest Amazon Linux 2 instance, if new AMI is released will replace instance to keep it patched 84 | // If replaced with specific AMI, ensure SSM Agent is installed and running 85 | machineImage: MachineImage.latestAmazonLinux({ 86 | generation: AmazonLinuxGeneration.AMAZON_LINUX_2, 87 | }), 88 | securityGroup: bastionHostLinuxSecurityGroup, 89 | role: bastionHostRole, 90 | // Ensure Bastion host EBS volume is encrypted 91 | blockDevices: [{ 92 | deviceName: '/dev/xvda', 93 | volume: BlockDeviceVolume.ebs(30, { 94 | volumeType: EbsDeviceVolumeType.GP3, 95 | encrypted: true, 96 | }), 97 | }], 98 | }); 99 | 100 | // Need KMS Key for EKS Envelope Encryption, if deleted, KMS will wait default (30 days) time before removal. 101 | const clusterKmsKey = new Key(this, 'ekskmskey', { 102 | enableKeyRotation: true, 103 | alias: cdk.Fn.join('', ['alias/', 'eks/', this.getOrCreateEksName(this)]), 104 | }); 105 | this.cluster = new Cluster(this, 'EKSCluster', { 106 | version: KubernetesVersion.of(k8sversion.valueAsString), 107 | defaultCapacity: 0, 108 | // https://aws.github.io/aws-eks-best-practices/security/docs/iam/#make-the-eks-cluster-endpoint-private 109 | endpointAccess: EndpointAccess.PRIVATE, 110 | vpc: vpc, 111 | secretsEncryptionKey: clusterKmsKey, 112 | mastersRole: bastionHostLinux.role, 113 | clusterName: this.getOrCreateEksName(this), 114 | // Ensure EKS helper lambadas are in private subnets 115 | placeClusterHandlerInVpc: true, 116 | }); 117 | // Allow BastionHost security group access to EKS Control Plane 118 | bastionHostLinux.connections.allowTo(this.cluster, Port.tcp(443), 'Allow between BastionHost and EKS '); 119 | // Install kubectl version similar to EKS k8s version 120 | bastionHostLinux.userData.addCommands( 121 | `VERSION=$(aws --region ${this.region} eks describe-cluster --name ${this.cluster.clusterName} --query 'cluster.version' --output text)`, 122 | 'echo \'K8s version is $VERSION\'', 123 | 'curl -LO https://dl.k8s.io/release/v$VERSION.0/bin/linux/amd64/kubectl', 124 | 'install -o root -g root -m 0755 kubectl /bin/kubectl', 125 | ); 126 | this.awsauth = new AwsAuth(this, 'EKS_AWSAUTH', { 127 | cluster: this.cluster, 128 | }); 129 | 130 | // deploy Custom k8s RBAC group to provide EKS Web Console read only permissions https://docs.aws.amazon.com/eks/latest/userguide/add-user-role.html 131 | // https://aws.github.io/aws-eks-best-practices/security/docs/iam.html#employ-least-privileged-access-when-creating-rolebindings-and-clusterrolebindings 132 | const manifestConsoleViewGroup = yaml.loadAll(fs.readFileSync('manifests/consoleViewOnlyGroup.yaml', 'utf-8')); 133 | const manifestConsoleViewGroupDeploy = new KubernetesManifest(this, 'eks-group-view-only', { 134 | cluster: this.cluster, 135 | manifest: manifestConsoleViewGroup, 136 | }); 137 | this.awsauth.node.addDependency(manifestConsoleViewGroupDeploy); 138 | this.awsauth.addMastersRole(bastionHostLinux.role, `${bastionHostLinux.role.roleArn}/{{SessionName}}`); 139 | // Patch aws-node daemonset to use IRSA via EKS Addons, do before nodes are created 140 | // https://aws.github.io/aws-eks-best-practices/security/docs/iam/#update-the-aws-node-daemonset-to-use-irsa 141 | const awsNodeconditionsPolicy = new CfnJson(this, 'awsVpcCniconditionPolicy', { 142 | value: { 143 | [`${this.cluster.openIdConnectProvider.openIdConnectProviderIssuer}:aud`]: 'sts.amazonaws.com', 144 | [`${this.cluster.openIdConnectProvider.openIdConnectProviderIssuer}:sub`]: 'system:serviceaccount:kube-system:aws-node', 145 | }, 146 | }); 147 | const awsNodePrincipal = new OpenIdConnectPrincipal(this.cluster.openIdConnectProvider).withConditions({ 148 | StringEquals: awsNodeconditionsPolicy, 149 | }); 150 | const awsVpcCniRole = new Role(this, 'awsVpcCniRole', { 151 | assumedBy: awsNodePrincipal, 152 | }); 153 | 154 | awsVpcCniRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonEKS_CNI_Policy')); 155 | (() => new CfnAddon(this, 'vpc-cni', { 156 | addonName: 'vpc-cni', 157 | resolveConflicts: 'OVERWRITE', 158 | serviceAccountRoleArn: awsVpcCniRole.roleArn, 159 | clusterName: this.cluster.clusterName, 160 | addonVersion: this.node.tryGetContext('eks-addon-vpc-cni-version'), 161 | }))(); 162 | 163 | (() => new CfnAddon(this, 'kube-proxy', { 164 | addonName: 'kube-proxy', 165 | resolveConflicts: 'OVERWRITE', 166 | clusterName: this.cluster.clusterName, 167 | addonVersion: this.node.tryGetContext('eks-addon-kube-proxy-version'), 168 | }))(); 169 | (() => new CfnAddon(this, 'core-dns', { 170 | addonName: 'coredns', 171 | resolveConflicts: 'OVERWRITE', 172 | clusterName: this.cluster.clusterName, 173 | addonVersion: this.node.tryGetContext('eks-addon-coredns-version'), 174 | }))(); 175 | 176 | // Add existing IAM Role to Custom Group 177 | // https://aws.github.io/aws-eks-best-practices/security/docs/iam/#use-iam-roles-when-multiple-users-need-identical-access-to-the-cluster 178 | // this.awsauth.addRoleMapping(Role.fromRoleArn(this, 'Role_Admin', `arn:aws:iam::${this.account}:role/Admin`), { 179 | // groups: ['eks-console-dashboard-full-access-group'], 180 | // username: `arn:aws:iam::${this.account}:role/Admin/{{SessionName}}`, 181 | // }); 182 | 183 | // Enable EKS Cluster Logging using Custom Resources 184 | // https://github.com/aws/aws-cdk/issues/4159 185 | // https://aws.github.io/aws-eks-best-practices/security/docs/detective/#enable-audit-logs 186 | const eksLoggingCustomLambdaPolicy = new PolicyStatement( 187 | { 188 | resources: [ 189 | this.cluster.clusterArn, 190 | `${this.cluster.clusterArn}/update-config`, 191 | ], 192 | actions: [ 193 | 'eks:UpdateClusterConfig', 194 | 'eks:DescribeCluster', 195 | ], 196 | }); 197 | // Lambda function to update EKS Cluster with logging options see custom-resources/eksloggingOnEvent.py for code 198 | const eksLoggingCustomLambda = new SingletonFunction(this, 'eksLoggingLambdaFunctiononEvent', { 199 | uuid: this.stackName + 'eksLoggingLambdaFunctiononEvent', 200 | functionName: this.stackName + 'eksLoggingLambdaFunctiononEvent', 201 | runtime: Runtime.PYTHON_3_8, 202 | code: Code.fromAsset(path.join(__dirname, 'custom-resources')), 203 | handler: 'eksloggingOnEvent.on_event', 204 | initialPolicy: [eksLoggingCustomLambdaPolicy], 205 | timeout: Duration.seconds(300), 206 | }); 207 | // Lambda function to check if EKS Cluster logging options are updated see custom-resources/eksloggingOnEvent.py for code 208 | const eksLoggingCustomLambdaisComplete = new SingletonFunction(this, 'eksLoggingLambdaFunctionisComplete', { 209 | uuid: this.stackName + 'eksLoggingLambdaFunctionisComplete', 210 | functionName: this.stackName + 'eksLoggingLambdaFunctionisComplete', 211 | runtime: Runtime.PYTHON_3_8, 212 | code: Code.fromAsset(path.join(__dirname, 'custom-resources')), 213 | handler: 'eksloggingOnEvent.is_complete', 214 | initialPolicy: [eksLoggingCustomLambdaPolicy], 215 | timeout: Duration.seconds(300), 216 | }); 217 | // Custom Resource to update/check EKS logging using lambdas 218 | const eksLoggingProvider = new Provider(this, 'eksLoggingProvider', { 219 | onEventHandler: eksLoggingCustomLambda, 220 | isCompleteHandler: eksLoggingCustomLambdaisComplete, 221 | logRetention: RetentionDays.ONE_WEEK, 222 | totalTimeout: Duration.minutes(10), 223 | queryInterval: Duration.seconds(300), 224 | }); 225 | // Invoke Customer Resource with logging options from parameters 226 | (() => new CustomResource(this, 'eksLoggingCustomResource', { 227 | serviceToken: eksLoggingProvider.serviceToken, 228 | properties: { 229 | eksCluster: this.getOrCreateEksName(this), 230 | loggingOpts: eksLoggingOpts.valueAsList, 231 | }, 232 | }))(); 233 | } 234 | 235 | // Create nodegroup IAM role in same stack as eks cluster to ensure there is not a circular dependency 236 | public createNodegroupRole(id: string): Role { 237 | const role = new Role(this, id, { 238 | assumedBy: new ServicePrincipal('ec2.amazonaws.com'), 239 | }); 240 | role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonEKSWorkerNodePolicy')); 241 | role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ContainerRegistryReadOnly')); 242 | this.awsauth.addRoleMapping(role, { 243 | username: 'system:node:{{EC2PrivateDNSName}}', 244 | groups: [ 245 | 'system:bootstrappers', 246 | 'system:nodes', 247 | ], 248 | }); 249 | return role; 250 | } 251 | 252 | private getOrCreateVpc(scope: Construct): IVpc { 253 | // use an existing vpc or create a new one using cdk context 254 | const stack = cdk.Stack.of(scope); 255 | // Able to choose default vpc but will error if EKS Cluster endpointAccess is set to be Private, need private subnets 256 | if (stack.node.tryGetContext('use_default_vpc') === '1') { 257 | return Vpc.fromLookup(stack, 'EKSNetworking', { isDefault: true }); 258 | } 259 | if (stack.node.tryGetContext('use_vpc_id') !== undefined) { 260 | return Vpc.fromLookup(stack, 'EKSNetworking', { vpcId: stack.node.tryGetContext('use_vpc_id') }); 261 | } 262 | const vpc = new Vpc(stack, stack.stackName + '-EKSNetworking', eksVpc); 263 | addEndpoint(stack, vpc); 264 | return vpc; 265 | } 266 | 267 | private getOrCreateEksName(scope: Construct): string { 268 | // use an existing vpc or create a new one using cdk context 269 | const stack = cdk.Stack.of(scope); 270 | if (stack.node.tryGetContext('cluster_name') !== undefined) { 271 | return stack.node.tryGetContext('cluster_name'); 272 | } 273 | return 'myekscluster'; 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /lib/k8s-baseline.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs'; 2 | import { CfnCondition, CfnParameter, CfnResource, Fn } from 'aws-cdk-lib'; 3 | import { Cluster, HelmChart, KubernetesManifest, ServiceAccount } from 'aws-cdk-lib/aws-eks'; 4 | import { ManagedPolicy } from 'aws-cdk-lib/aws-iam'; 5 | import * as cdk from 'aws-cdk-lib'; 6 | import { Construct, IConstruct } from 'constructs'; 7 | import * as yaml from 'js-yaml'; 8 | import * as genPolicy from './policies/policies'; 9 | 10 | interface k8sBaselineProps extends cdk.StackProps { 11 | eksCluster: Cluster, 12 | } 13 | 14 | export class K8sBaselineStack extends cdk.Stack { 15 | constructor(scope: Construct, 16 | id: string, 17 | props: k8sBaselineProps) { 18 | super(scope, id, props); 19 | // ============================================================================================================================================ 20 | // Parameters 21 | // ============================================================================================================================================ 22 | const ebsDriver = new CfnParameter(this, 'ebsDriver', { 23 | type: 'String', 24 | default: 'false', 25 | description: 'Deploy EBS CSI Driver', 26 | allowedValues: ['true', 'false'], 27 | }); 28 | const albDriver = new CfnParameter(this, 'albDriver', { 29 | type: 'String', 30 | default: 'true', 31 | description: 'Deploy Application Load Balancer Ingress ', 32 | allowedValues: ['true', 'false'], 33 | }); 34 | const efsDriver = new CfnParameter(this, 'efsDriver', { 35 | type: 'String', 36 | default: 'false', 37 | description: 'Deploy EFS CSI Driver', 38 | allowedValues: ['true', 'false'], 39 | }); 40 | // FluentBit needs to implement IMDSv2 41 | // https://github.com/fluent/fluent-bit/issues/2840#issuecomment-774393238 42 | const fluentBitDriver = new CfnParameter(this, 'fluentBit', { 43 | type: 'String', 44 | default: 'true', 45 | description: 'Deploy FluentBit Log Collection Driver', 46 | allowedValues: ['true', 'false'], 47 | }); 48 | const secretsDriver = new CfnParameter(this, 'secretsDriver', { 49 | type: 'String', 50 | default: 'false', 51 | description: 'Deploy AWS Secrets CSI Driver', 52 | allowedValues: ['true', 'false'], 53 | }); 54 | 55 | const networkPolicyEngineDriver = new CfnParameter(this, 'networkPolicyEngine', { 56 | type: 'String', 57 | default: 'false', 58 | description: 'Deploy Calico Network Policy Engine Driver', 59 | allowedValues: ['true', 'false'], 60 | }); 61 | 62 | const clusterAutoscalerDriver = new CfnParameter(this, 'clusterAutoscaler', { 63 | type: 'String', 64 | default: 'true', 65 | description: 'Deploy Cluster Autoscaler', 66 | allowedValues: ['true', 'false'], 67 | }); 68 | const containerInsightsDriver = new CfnParameter(this, 'containerInsights', { 69 | type: 'String', 70 | default: 'false', 71 | description: 'Deploy Container Insights', 72 | allowedValues: ['true', 'false'], 73 | }); 74 | const metricServerDriver = new CfnParameter(this, 'metricServer', { 75 | type: 'String', 76 | default: 'true', 77 | description: 'Deploy Metric Server', 78 | allowedValues: ['true', 'false'], 79 | }); 80 | 81 | // ============================================================================================================================================ 82 | // Conditions 83 | // ============================================================================================================================================ 84 | 85 | const ebsDriverCondition = new CfnCondition(this, 'ebsDriverCondition', { 86 | expression: Fn.conditionEquals(ebsDriver.valueAsString, 'true'), 87 | }); 88 | 89 | const albDriverCondition = new CfnCondition(this, 'albDriverCondition', { 90 | expression: Fn.conditionEquals(albDriver.valueAsString, 'true'), 91 | }); 92 | const efsDriverCondition = new CfnCondition(this, 'efsDriverCondition', { 93 | expression: Fn.conditionEquals(efsDriver.valueAsString, 'true'), 94 | }); 95 | const fluentBitDriverCondition = new CfnCondition(this, 'fluentBitDriverCondition', { 96 | expression: Fn.conditionEquals(fluentBitDriver.valueAsString, 'true'), 97 | }); 98 | const secretsDriverCondition = new CfnCondition(this, 'secretsDriverCondition', { 99 | expression: Fn.conditionEquals(secretsDriver.valueAsString, 'true'), 100 | }); 101 | const networkPolicyDriverCondition = new CfnCondition(this, 'networkPolicyEngineDriverCondition', { 102 | expression: Fn.conditionEquals(networkPolicyEngineDriver.valueAsString, 'true'), 103 | }); 104 | const clusterAutoscalerDriverCondition = new CfnCondition(this, 'clusterAutoscalerDriverCondition', { 105 | expression: Fn.conditionEquals(clusterAutoscalerDriver.valueAsString, 'true'), 106 | }); 107 | const containerInsightsDriverCondition = new CfnCondition(this, 'containerInsightsDriverCondition', { 108 | expression: Fn.conditionEquals(containerInsightsDriver.valueAsString, 'true'), 109 | }); 110 | const metricServerDriverCondition = new CfnCondition(this, 'metricServerDriverCondition', { 111 | expression: Fn.conditionEquals(metricServerDriver.valueAsString, 'true'), 112 | }); 113 | 114 | // ============================================================================================================================================ 115 | // Resource Creation 116 | // ============================================================================================================================================ 117 | /* 118 | Service Account Resources will be created in CDK to ensure proper IAM to K8s RBAC Mapping 119 | Helm Chart Version are taken from cdk.json file or from command line parameter -c 120 | Helm Chart full version list can be found via helm repo list or viewing yaml file on github directly, see README. 121 | */ 122 | 123 | /* 124 | Resources needed to create Cluster Autoscaler 125 | Service Account Role 126 | IAM Policy 127 | Helm Chart 128 | */ 129 | const clusterAutoscalerSA = new ServiceAccount(this, 'clusterAutoscalerSA', { 130 | name: 'cluster-autoscaler-sa', 131 | cluster: props.eksCluster, 132 | namespace: 'kube-system', 133 | }); 134 | this.addConditions(clusterAutoscalerSA, clusterAutoscalerDriverCondition); 135 | const clusterAutoscalerDeploy = new HelmChart(this, 'clusterautoscaler-deploy', { 136 | repository: 'https://kubernetes.github.io/autoscaler', 137 | release: 'cluster-autoscaler', 138 | cluster: props.eksCluster, 139 | chart: 'cluster-autoscaler', 140 | namespace: 'kube-system', 141 | wait: true, 142 | // https://github.com/kubernetes/autoscaler/blob/gh-pages/index.yaml 143 | version: this.node.tryGetContext('cluster-autoscaler-helm-version'), 144 | // https://github.com/kubernetes/autoscaler/tree/master/charts/cluster-autoscaler#values 145 | values: { 146 | cloudProvider: 'aws', 147 | awsRegion: this.region, 148 | autoDiscovery: { 149 | clusterName: props.eksCluster.clusterName, 150 | }, 151 | rbac: { 152 | serviceAccount: { 153 | create: false, 154 | name: clusterAutoscalerSA.serviceAccountName, 155 | }, 156 | }, 157 | extraArgs: { 158 | // https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-are-the-parameters-to-ca 159 | 'skip-nodes-with-system-pods': false, 160 | 'skip-nodes-with-local-storage': false, 161 | 'balance-similar-node-groups': true, 162 | // How long a node should be unneeded before it is eligible for scale down 163 | 'scale-down-unneeded-time': '300s', 164 | // How long after scale up that scale down evaluation resumes 165 | 'scale-down-delay-after-add': '300s', 166 | }, 167 | 168 | }, 169 | }); 170 | // Generate IAM Policy with scoped permissions 171 | const clusterAutoscalerPolicy = genPolicy.createClusterAutoscalerPolicy(this, props.eksCluster.clusterName, clusterAutoscalerSA.role); 172 | // Add condition to deploy clusterAutoscaler Resources only if condition is true 173 | this.addConditions(clusterAutoscalerDeploy, clusterAutoscalerDriverCondition); 174 | this.addConditions(clusterAutoscalerPolicy, clusterAutoscalerDriverCondition); 175 | /* 176 | Resources needed to create Fluent Bit DaemonSet 177 | Namespace 178 | Service Account Role 179 | IAM Policy 180 | K8s Manifest 181 | 182 | Current Config pushes to Cloudwatch , other outputs found here https://docs.fluentbit.io/manual/pipeline/outputs 183 | Fluentbit does not support IMDSv2 184 | https://github.com/fluent/fluent-bit/issues/2840#issuecomment-774393238 185 | */ 186 | 187 | // YAML contains fluentbit parser configurations, remove namespace and serviceaccount from yaml to properly annotate with IAM Role 188 | const manifestFluentBitSetup = this.cleanManifest('manifests/fluentBitSetup.yaml'); 189 | const fluentBitNamespace = new KubernetesManifest(this, 'amazon-cloudwatch-namespace', { 190 | cluster: props.eksCluster, 191 | manifest: [{ 192 | apiVersion: 'v1', 193 | kind: 'Namespace', 194 | metadata: { 195 | name: 'amazon-cloudwatch', 196 | labels: { 197 | name: 'amazon-cloudwatch', 198 | }, 199 | }, 200 | }], 201 | }); 202 | this.addConditions(fluentBitNamespace, fluentBitDriverCondition); 203 | const fluentBitSA = new ServiceAccount(this, 'fluentbit-sa', { 204 | name: 'fluent-bit', 205 | namespace: 'amazon-cloudwatch', 206 | cluster: props.eksCluster, 207 | }); 208 | // Ensure Namespace is created first before fluentBitSA resource 209 | fluentBitSA.node.addDependency(fluentBitNamespace); 210 | const fluentbitPolicy = genPolicy.createFluentbitPolicy(this, props.eksCluster.clusterName, fluentBitSA.role); 211 | this.addConditions(fluentbitPolicy, fluentBitDriverCondition); 212 | this.addConditions(fluentBitSA, fluentBitDriverCondition); 213 | // Configurable variables for manifests/fluentBitSetup.yaml 214 | const fluentBitClusterInfo = new KubernetesManifest(this, 'fluentbit-cluster-info', { 215 | cluster: props.eksCluster, 216 | manifest: [{ 217 | apiVersion: 'v1', 218 | kind: 'ConfigMap', 219 | metadata: { 220 | name: 'fluent-bit-cluster-info', 221 | namespace: 'amazon-cloudwatch', 222 | labels: { 223 | name: 'fluent-bit-cluster-info', 224 | }, 225 | }, 226 | data: { 227 | 'cluster.name': props.eksCluster.clusterName, 228 | 'http.port': '2020', 229 | 'http.server': 'On', 230 | 'logs.region': this.region, 231 | 'read.head': 'Off', 232 | 'read.tail': 'On', 233 | }, 234 | 235 | }], 236 | 237 | }); 238 | fluentBitClusterInfo.node.addDependency(fluentBitNamespace); 239 | this.addConditions(fluentBitClusterInfo, fluentBitDriverCondition); 240 | const fluentBitResource = new KubernetesManifest(this, 'fluentbit-resource', { 241 | cluster: props.eksCluster, 242 | manifest: manifestFluentBitSetup, 243 | }); 244 | fluentBitResource.node.addDependency(fluentBitSA); 245 | fluentBitResource.node.addDependency(fluentBitClusterInfo); 246 | this.addConditions(fluentBitResource, fluentBitDriverCondition); 247 | 248 | /* 249 | Resources needed to create ALB Ingress Controller 250 | Namespace 251 | Service Account Role 252 | IAM Policy 253 | Helm Chart 254 | AddOn: https://github.com/aws/containers-roadmap/issues/1162 255 | */ 256 | 257 | // Create Namespace and Service Account for ALB Ingress 258 | const albNamespace = new KubernetesManifest(this, 'alb-ingress-controller-namespace', { 259 | cluster: props.eksCluster, 260 | manifest: [{ 261 | apiVersion: 'v1', 262 | kind: 'Namespace', 263 | metadata: { 264 | name: 'alb-ingress-controller', 265 | labels: { 266 | name: 'alb-ingress-controller', 267 | }, 268 | }, 269 | }], 270 | }); 271 | const albSA = new ServiceAccount(this, 'alb-ingress-controller-sa', { 272 | name: 'alb-ingress-controller-sa', 273 | namespace: 'alb-ingress-controller', 274 | cluster: props.eksCluster, 275 | }); 276 | this.addConditions(albNamespace, albDriverCondition); 277 | albSA.node.addDependency(albNamespace); 278 | this.addConditions(albSA, albDriverCondition); 279 | 280 | // ALB Controller IAMPolicy 281 | const albIamTest = genPolicy.createAlbIngressPolicy(this, props.eksCluster.clusterName, albSA.role); 282 | this.addConditions(albIamTest, albDriverCondition); 283 | // https://github.com/aws/eks-charts/blob/master/stable/aws-load-balancer-controller/values.yaml 284 | const albIngressHelmChart = new HelmChart(this, 'alb-ingress-controller-chart', { 285 | chart: 'aws-load-balancer-controller', 286 | cluster: props.eksCluster, 287 | repository: 'https://aws.github.io/eks-charts', 288 | wait: true, 289 | release: 'aws-load-balancer-controller', 290 | createNamespace: true, 291 | namespace: 'alb-ingress-controller', 292 | // https://github.com/aws/eks-charts/blob/gh-pages/index.yaml 293 | version: this.node.tryGetContext('aws-load-balancer-controller-helm-version'), 294 | values: { 295 | clusterName: props.eksCluster.clusterName, 296 | defaultTags: { 297 | 'eks:cluster-name': props.eksCluster.clusterName, 298 | }, 299 | // Start - values needed if ec2metadata endpoint is unavailable - https://github.com/aws/eks-charts/tree/master/stable/aws-load-balancer-controller#configuration 300 | region: this.region, 301 | vpcId: props.eksCluster.vpc.vpcId, 302 | // End - values needed if ec2metadata endpoint is unavailable 303 | serviceAccount: { 304 | create: false, 305 | name: albSA.serviceAccountName, 306 | }, 307 | }, 308 | }); 309 | albIngressHelmChart.node.addDependency(albSA); 310 | this.addConditions(albIngressHelmChart, albDriverCondition); 311 | /* 312 | Resources needed to create EBS CSI Driver 313 | Service Account Role 314 | IAM Policy 315 | Helm Chart 316 | Add On: https://github.com/aws/containers-roadmap/issues/247 317 | */ 318 | 319 | // Create Service Account (Pod IAM Role Mapping) for EBS Controller 320 | const ebsSA = new ServiceAccount(this, 'ebs-csi-controller-sa', { 321 | name: 'ebs-csi-controller-sa', 322 | namespace: 'kube-system', 323 | cluster: props.eksCluster, 324 | }); 325 | this.addConditions(ebsSA, ebsDriverCondition); 326 | 327 | // EBS Controller IAMPolicyDoc 328 | const ebsIamPolicyTest = genPolicy.createEBSPolicy(this, props.eksCluster.clusterName, ebsSA.role); 329 | this.addConditions(ebsIamPolicyTest, ebsDriverCondition); 330 | // Helm Chart Values: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/charts/aws-ebs-csi-driver/values.yaml 331 | const ebsCsiHelmChart = new HelmChart(this, 'ebs-csi-helm-chart', { 332 | chart: 'aws-ebs-csi-driver', 333 | cluster: props.eksCluster, 334 | createNamespace: true, 335 | repository: 'https://kubernetes-sigs.github.io/aws-ebs-csi-driver', 336 | release: 'aws-ebs-csi-driver', 337 | namespace: 'kube-system', 338 | wait: true, 339 | // Helm Chart Versions: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/gh-pages/index.yaml 340 | version: this.node.tryGetContext('aws-ebs-csi-driver-helm-version'), 341 | values: { 342 | controller: { 343 | serviceAccount: { 344 | create: false, 345 | name: ebsSA.serviceAccountName, 346 | }, 347 | extraVolumeTags: { 348 | 'eks:cluster-name': props.eksCluster.clusterName, 349 | }, 350 | }, 351 | 352 | }, 353 | 354 | }); 355 | 356 | ebsCsiHelmChart.node.addDependency(ebsSA); 357 | this.addConditions(ebsCsiHelmChart, ebsDriverCondition); 358 | 359 | /* 360 | Resources needed to create EFS Controller 361 | Service Account Role 362 | IAM Policy 363 | Helm Chart 364 | */ 365 | 366 | const efsSA = new ServiceAccount(this, 'efs-csi-controller-sa', { 367 | name: 'efs-csi-controller-sa', 368 | namespace: 'kube-system', 369 | cluster: props.eksCluster, 370 | }); 371 | this.addConditions(efsSA, efsDriverCondition); 372 | 373 | // Make sure to allow traffic on port 2049 on the security group associated to your EFS file system from the CIDR assigned to your EKS cluster. 374 | // https://docs.aws.amazon.com/efs/latest/ug/network-access.html 375 | // Ensure EFS connections are using TLS when provisioning EFS PersistentVolume 376 | // https://docs.aws.amazon.com/eks/latest/userguide/efs-csi.html#efs-install-driver 377 | const efsPolicy = genPolicy.createEFSPolicy(this, props.eksCluster.clusterName, efsSA.role); 378 | this.addConditions(efsPolicy, efsDriverCondition); 379 | // Helm Chart Values: https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/charts/aws-efs-csi-driver/values.yaml 380 | const efsCsiHelmChart = new HelmChart(this, 'efs-csi-helm-chart', { 381 | chart: 'aws-efs-csi-driver', 382 | cluster: props.eksCluster, 383 | createNamespace: true, 384 | repository: 'https://kubernetes-sigs.github.io/aws-efs-csi-driver', 385 | release: 'aws-efs-csi-driver', 386 | namespace: 'kube-system', 387 | wait: true, 388 | // Helm Chart Versions: https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/gh-pages/index.yaml 389 | version: this.node.tryGetContext('aws-efs-csi-driver-helm-version'), 390 | values: { 391 | controller: { 392 | logLevel: 10, 393 | serviceAccount: { 394 | create: false, 395 | name: efsSA.serviceAccountName, 396 | }, 397 | tags: { 398 | // Unable to use ":" in tags due to EFS CSI Driver splitting string by ":" eks:cluster-name: myclustername -> eks: cluster-name 399 | // https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/388c3f90f21f9c550935815bc8af25ddce4c7f32/pkg/driver/driver.go#L145 400 | 'eks/cluster-name': props.eksCluster.clusterName, 401 | }, 402 | }, 403 | 404 | }, 405 | 406 | }); 407 | 408 | efsCsiHelmChart.node.addDependency(efsSA); 409 | this.addConditions(efsCsiHelmChart, efsDriverCondition); 410 | 411 | /* 412 | Resources needed to create Secrets Manager 413 | Service Account Role 414 | Helm Chart 415 | Kubernetes Manifest 416 | */ 417 | 418 | const awsSecretSa = new ServiceAccount(this, 'aws-secrets-sa', { 419 | cluster: props.eksCluster, 420 | name: 'csi-secrets-store-provider-aws', 421 | namespace: 'kube-system', 422 | }); 423 | 424 | this.addConditions(awsSecretSa, secretsDriverCondition); 425 | 426 | // https://github.com/kubernetes-sigs/secrets-store-csi-driver/blob/master/charts/secrets-store-csi-driver/values.yaml 427 | // Deploys Secrets Store CSI Driver 428 | const secretsCsiHelmChart = new HelmChart(this, 'secrets-csi-helm-chart', { 429 | chart: 'secrets-store-csi-driver', 430 | cluster: props.eksCluster, 431 | createNamespace: true, 432 | repository: 'https://raw.githubusercontent.com/kubernetes-sigs/secrets-store-csi-driver/master/charts', 433 | release: 'csi-secrets-store', 434 | namespace: 'kube-system', 435 | wait: true, 436 | // Helm Chart Values: https://github.com/kubernetes-sigs/secrets-store-csi-driver/blob/master/charts/index.yaml 437 | version: this.node.tryGetContext('secrets-store-csi-helm-version'), 438 | values: { 439 | grpcSupportedProviders: 'aws', 440 | // alpha feature reconciler feature 441 | // rotationPollInterval: 3600 442 | // enableSecretRotation: true 443 | }, 444 | 445 | }); 446 | 447 | this.addConditions(secretsCsiHelmChart, secretsDriverCondition); 448 | // Deploys AWS Secrets and Configuration Provider (ASCP) 449 | const awsSecretsManifest = this.cleanManifest('manifests/awsSecretsManifest.yaml'); 450 | const awsSecretsManifestDeploy = new KubernetesManifest(this, 'aws-secrets-manifest', { 451 | cluster: props.eksCluster, 452 | manifest: awsSecretsManifest, 453 | }); 454 | 455 | awsSecretsManifestDeploy.node.addDependency(secretsCsiHelmChart); 456 | this.addConditions(awsSecretsManifestDeploy, secretsDriverCondition); 457 | 458 | /* 459 | Resources needed to create Calico Policy Engine 460 | Helm Chart 461 | */ 462 | // https://github.com/aws/eks-charts/blob/master/stable/aws-calico/values.yaml 463 | // https://github.com/aws/amazon-vpc-cni-k8s/issues/1517 464 | const calicoPolicyEngine = new HelmChart(this, 'calico-policy-engine', { 465 | chart: 'aws-calico', 466 | cluster: props.eksCluster, 467 | createNamespace: true, 468 | repository: 'https://aws.github.io/eks-charts', 469 | release: 'aws-calico', 470 | namespace: 'kube-system', 471 | wait: true, 472 | // https://github.com/aws/eks-charts/blob/gh-pages/index.yaml 473 | version: this.node.tryGetContext('aws-calico-helm-version'), 474 | // increase limits for calico pods, more needed as number of nodes/pods increase. Uses default Requests values in values.yaml in above comment 475 | values: { 476 | calico: { 477 | node: { 478 | logseverity: 'Debug', 479 | resources: { 480 | limits: { 481 | memory: '256Mi', 482 | cpu: '500m', 483 | }, 484 | }, 485 | }, 486 | }, 487 | }, 488 | 489 | }); 490 | this.addConditions(calicoPolicyEngine, networkPolicyDriverCondition); 491 | 492 | /* 493 | Resources needed to create Container Insights using OpenTelemetry 494 | Service Account Role 495 | IAM Policy 496 | Kubernetes Manifest 497 | 498 | OpenTelemetry is configured to emit all public available metrics for ContainerInsights, to reduce CloudWatch Metric cost customize the ADOT collector 499 | https://aws-otel.github.io/docs/getting-started/container-insights/eks-infra#advanced-usage 500 | */ 501 | // Create Namespace for Container Insights components 502 | const containerInsightsNamespace = new KubernetesManifest(this, 'container-insights-namespace', { 503 | cluster: props.eksCluster, 504 | manifest: [{ 505 | apiVersion: 'v1', 506 | kind: 'Namespace', 507 | metadata: { 508 | name: 'aws-otel-eks', 509 | labels: { 510 | name: 'aws-otel-eks', 511 | }, 512 | }, 513 | }], 514 | }); 515 | this.addConditions(containerInsightsNamespace, containerInsightsDriverCondition); 516 | 517 | const containerInsightsSA = new ServiceAccount(this, 'container-insights-sa', { 518 | name: 'aws-otel-sa', 519 | namespace: 'aws-otel-eks', 520 | cluster: props.eksCluster, 521 | }); 522 | containerInsightsSA.role.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, 'CloudWatchAgentServerPolicyManaged', 'arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy')); 523 | this.addConditions(containerInsightsSA, containerInsightsDriverCondition); 524 | 525 | containerInsightsSA.node.addDependency(containerInsightsNamespace); 526 | const manifestcontainerInsightsSetup = this.cleanManifest('manifests/otelContainerInsights.yaml'); 527 | const containerInsightsDeploy = new KubernetesManifest(this, 'container-insights-deploy', { 528 | cluster: props.eksCluster, 529 | manifest: [ 530 | ...manifestcontainerInsightsSetup, 531 | ], 532 | }); 533 | containerInsightsDeploy.node.addDependency(containerInsightsSA); 534 | this.addConditions(containerInsightsDeploy, containerInsightsDriverCondition); 535 | /* Resources needed to create Metric Server 536 | Manifest 537 | 538 | Metric Server Scaling Requirements -> https://github.com/kubernetes-sigs/metrics-server#scaling 539 | AddOn: https://github.com/aws/containers-roadmap/issues/261 540 | */ 541 | // version is based on default metric server manifest see file for origin, does not require clean up function as namespace service account does not reply on IAM Role creation dependency. 542 | const manifestMetricServer = yaml.loadAll(fs.readFileSync('manifests/metricServerManifest.yaml', 'utf-8'), null, { schema: yaml.JSON_SCHEMA }); 543 | const metricServerManifestDeploy = new KubernetesManifest(this, 'metric-server', { 544 | cluster: props.eksCluster, 545 | manifest: [ 546 | ...manifestMetricServer, 547 | ], 548 | }); 549 | this.addConditions(metricServerManifestDeploy, metricServerDriverCondition); 550 | } 551 | 552 | // Takes a CDK Abstract Resource and adds CFN Conditions to the underlying CFN Resources to ensure proper resource creation/deletion 553 | addConditions(resource: IConstruct, cond: CfnCondition) { 554 | // Add Conditions to Cfn type resources only, which map directly to Cloudformation resource type AWS::TYPE::RESOURCE 555 | // https://docs.aws.amazon.com/cdk/api/latest/docs/core-readme.html#intrinsic-functions-and-condition-expressions 556 | if (resource.node.defaultChild !== undefined && resource.node.defaultChild.constructor.name.match(/^Cfn+/)) { 557 | (resource.node.defaultChild as CfnResource).cfnOptions.condition = cond; 558 | } else { 559 | resource.node.children.forEach(node => { 560 | this.addConditions(node, cond); 561 | }); 562 | } 563 | } 564 | 565 | // Removes namespace and ServiceAccount objects from manifests, performing this in code to keep original manifest files. 566 | cleanManifest(file: string) { 567 | const manifest = yaml.loadAll(fs.readFileSync(file, 'utf-8'), null, { schema: yaml.JSON_SCHEMA }); 568 | return manifest.filter(element => (element.kind !== 'Namespace' && element.kind !== 'ServiceAccount')); 569 | } 570 | } 571 | -------------------------------------------------------------------------------- /lib/k8s-nodegroup.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from 'aws-cdk-lib'; 2 | import { CfnParameter, Fn } from 'aws-cdk-lib'; 3 | import { CfnLaunchTemplate, MultipartBody, MultipartUserData, UserData } from 'aws-cdk-lib/aws-ec2'; 4 | import { Cluster, Nodegroup } from 'aws-cdk-lib/aws-eks'; 5 | import { Role, ManagedPolicy } from 'aws-cdk-lib/aws-iam'; 6 | import { Construct } from 'constructs'; 7 | 8 | interface k8snodegroupsProps extends cdk.StackProps { 9 | eksCluster: Cluster, 10 | nodeGroupRole: Role 11 | } 12 | 13 | export class K8snodegroups extends cdk.Stack { 14 | constructor (scope: Construct, 15 | id: string, 16 | props: k8snodegroupsProps) { 17 | super(scope, id, props); 18 | const nodegroupMax = new CfnParameter(this, 'nodegroupMax', { 19 | type: 'Number', 20 | description: 'Max number of EKS worker nodes to scale up to', 21 | default: 10, 22 | }); 23 | const nodegroupCount = new CfnParameter(this, 'nodegroupCount', { 24 | type: 'Number', 25 | description: 'Desired Count of EKS Worker Nodes to launch', 26 | default: 2, 27 | }); 28 | const nodegroupMin = new CfnParameter(this, 'nodegroupMin', { 29 | type: 'Number', 30 | description: 'Min number of EKS worker nodes to scale down to', 31 | default: 2, 32 | }); 33 | const nodeType = new CfnParameter(this, 'nodegroupInstanceType', { 34 | type: 'String', 35 | description: 'Instance Type to be used with nodegroup ng-1', 36 | default: 't3.medium', 37 | }); 38 | const nodeAMIVersion = new CfnParameter(this, 'nodeAMIVersion', { 39 | type: 'String', 40 | default: '1.21.2-20210722', 41 | description: 'AMI version used for EKS Worker nodes https://docs.aws.amazon.com/eks/latest/userguide/eks-linux-ami-versions.html', 42 | }); 43 | 44 | const userdataCommands = UserData.forLinux(); 45 | // SSH only allowed via SSM Session Manager - https://aws.github.io/aws-eks-best-practices/security/docs/hosts/#minimize-access-to-worker-nodes 46 | userdataCommands.addCommands( 47 | `sudo yum install -y https://s3.${this.region}.amazonaws.com/amazon-ssm-${this.region}/latest/linux_amd64/amazon-ssm-agent.rpm`, 48 | ); 49 | const multipart = new MultipartUserData(); 50 | // const part = MultipartBody 51 | multipart.addPart( 52 | MultipartBody.fromUserData(userdataCommands), 53 | ); 54 | 55 | const launchtemplate = new CfnLaunchTemplate(this, 'LaunchTemplate', { 56 | launchTemplateData: { 57 | instanceType: nodeType.valueAsString, 58 | userData: Fn.base64(multipart.render()), 59 | // Ensure Managed Nodes Instances EBS Volumes are encrypted 60 | blockDeviceMappings: [ 61 | { 62 | deviceName: '/dev/xvda', 63 | ebs: { 64 | encrypted: true, 65 | volumeType: 'gp3', 66 | }, 67 | }, 68 | ], 69 | // Restrict access to the instance profile assigned to the worker node (not enabled) 70 | // Not all components are IMDSv2 aware. Ex. Fluentbit 71 | // https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node 72 | // https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-launchtemplate-launchtemplatedata-metadataoptions.html#aws-properties-ec2-launchtemplate-launchtemplatedata-metadataoptions-properties 73 | // https://aws.github.io/aws-eks-best-practices/security/docs/iam/#when-your-application-needs-access-to-idms-use-imdsv2-and-increase-the-hop-limit-on-ec2-instances-to-2 74 | metadataOptions: { 75 | httpTokens: 'optional', 76 | httpPutResponseHopLimit: 2, 77 | 78 | }, 79 | tagSpecifications: [{ 80 | resourceType: 'instance', 81 | tags: [ 82 | { 83 | key: 'Name', 84 | value: Fn.join('-', [props.eksCluster.clusterName, 'WorkerNodes']), 85 | }, 86 | ], 87 | }], 88 | }, 89 | launchTemplateName: Fn.join('-', ['ng-1', props.eksCluster.clusterName]), 90 | 91 | }); 92 | props.nodeGroupRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')); 93 | 94 | (() => new Nodegroup(this, 'ng-1', { 95 | cluster: props.eksCluster, 96 | // https://docs.aws.amazon.com/eks/latest/userguide/eks-linux-ami-versions.html 97 | releaseVersion: nodeAMIVersion.valueAsString, 98 | nodegroupName: 'ng-1', 99 | // Require specific order of max,desired,min or generated CDK Tokens fail desired>min check 100 | // https://github.com/aws/aws-cdk/issues/15485 101 | nodeRole: props.nodeGroupRole, 102 | maxSize: nodegroupMax.valueAsNumber, 103 | desiredSize: nodegroupCount.valueAsNumber, 104 | minSize: nodegroupMin.valueAsNumber, 105 | // LaunchTemplate for custom userdata to install SSM Agent 106 | launchTemplateSpec: { 107 | id: launchtemplate.ref, 108 | version: launchtemplate.attrLatestVersionNumber, 109 | }, 110 | tags: { 111 | Name: Fn.join('-', [props.eksCluster.clusterName, 'WorkerNodes']), 112 | }, 113 | }))(); 114 | // Permissions for SSM Manager for core functionality 115 | props.nodeGroupRole.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /lib/policies/policies.ts: -------------------------------------------------------------------------------- 1 | import { Policy, PolicyStatement, IRole } from 'aws-cdk-lib/aws-iam'; 2 | import { CfnJson, Stack } from 'aws-cdk-lib'; 3 | import { Construct } from 'constructs'; 4 | 5 | // Scope fluentbit to push logs to log-group /aws/containerinsights/CLUSTER_NAME/ only 6 | export function createFluentbitPolicy(stack: Stack, clusterName: string, roleSA: IRole): Policy { 7 | const fluentBitSaRoleStatementPolicy = new PolicyStatement({ 8 | resources: [ 9 | `arn:aws:logs:${stack.region}:${stack.account}:log-group:/aws/containerinsights/${clusterName}/*`, 10 | `arn:aws:logs:${stack.region}:${stack.account}:log-group:/aws/containerinsights/${clusterName}/*:log-stream:*`, 11 | ], 12 | actions: [ 13 | 'logs:CreateLogStream', 14 | 'logs:CreateLogGroup', 15 | 'logs:DescribeLogStreams', 16 | 'logs:PutLogEvents', 17 | ], 18 | }); 19 | return new Policy(stack, 'fluentBitSaRolePolicy', { 20 | roles: [ 21 | roleSA, 22 | ], 23 | statements: [ 24 | fluentBitSaRoleStatementPolicy, 25 | ], 26 | }); 27 | } 28 | // Scope ClusterAutoscaler to read/write to tags with cluster-name 29 | export function createClusterAutoscalerPolicy(stack: Construct, clusterName: string, roleSA: IRole): Policy { 30 | const clusterAutoscalerSAPolicyStatementDescribe = new PolicyStatement({ 31 | // https://docs.aws.amazon.com/eks/latest/userguide/cluster-autoscaler.html#ca-create-policy 32 | resources: [ 33 | '*', 34 | ], 35 | actions: [ 36 | 'autoscaling:DescribeAutoScalingGroups', 37 | 'autoscaling:DescribeAutoScalingInstances', 38 | 'autoscaling:DescribeLaunchConfigurations', 39 | 'autoscaling:DescribeTags', 40 | 'ec2:DescribeLaunchTemplateVersions', 41 | ], 42 | 43 | }); 44 | // Cluster Autoscaler tags resources using the tags below, so scope resources to those tags 45 | // Create CfnJson as variables are not allowed to be in keys for key:value pairs. 46 | const clusterAutoscalerPolicyStatementWriteJson = new CfnJson(stack, 'clusterAutoscalerPolicyStatementWriteJson', { 47 | value: { 48 | 'autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled': 'true', 49 | [`autoscaling:ResourceTag/kubernetes.io/cluster/${clusterName}`]: 'owned', 50 | }, 51 | }); 52 | const clusterAutoscalerPolicyStatementWrite = new PolicyStatement({ 53 | resources: [ 54 | '*', 55 | ], 56 | actions: [ 57 | 'autoscaling:SetDesiredCapacity', 58 | 'autoscaling:TerminateInstanceInAutoScalingGroup', 59 | 'autoscaling:UpdateAutoScalingGroup', 60 | ], 61 | conditions: { 62 | StringEquals: clusterAutoscalerPolicyStatementWriteJson, 63 | }, 64 | }, 65 | ); 66 | return new Policy(stack, 'clusterAutoscalerPolicy', { 67 | statements: [ 68 | clusterAutoscalerPolicyStatementWrite, 69 | clusterAutoscalerSAPolicyStatementDescribe, 70 | ], 71 | roles: [ 72 | roleSA, 73 | ], 74 | }); 75 | } 76 | 77 | export function createEFSPolicy(stack: Stack, clusterName: string, roleSA: IRole): Policy { 78 | // Scope EFS Permissions DescribeFileSystems,DescribeMountTargets on all, but limit create/delete AccessPoint to tag eks:clustername: 79 | const readEFSResources = new PolicyStatement({ 80 | resources: [ 81 | '*', 82 | ], 83 | actions: [ 84 | 'elasticfilesystem:DescribeFileSystems', 85 | 'elasticfilesystem:DescribeMountTargets', 86 | ], 87 | }); 88 | const writeEFSResourcesRequest = new PolicyStatement({ 89 | resources: [ 90 | '*', 91 | ], 92 | actions: [ 93 | 'elasticfilesystem:CreateAccessPoint', 94 | ], 95 | // Requires tags in Request call 96 | conditions: { 97 | StringEquals: { 98 | 'aws:RequestTag/eks/cluster-name': clusterName, 99 | }, 100 | }, 101 | }); 102 | const writeEFSResourcesResource = new PolicyStatement({ 103 | resources: [ 104 | '*', 105 | ], 106 | actions: [ 107 | 'elasticfilesystem:DeleteAccessPoint', 108 | ], 109 | conditions: { 110 | // Requires already created resource to contain tags 111 | StringEquals: { 112 | 'aws:ResourceTag/eks/cluster-name': clusterName, 113 | }, 114 | }, 115 | }); 116 | return new Policy(stack, 'efsPolicy', { 117 | roles: [ 118 | roleSA, 119 | ], 120 | statements: [ 121 | readEFSResources, 122 | writeEFSResourcesRequest, 123 | writeEFSResourcesResource, 124 | ], 125 | }); 126 | } 127 | 128 | export function createEBSPolicy(stack: Stack, clusterName: string, roleSA: IRole): Policy { 129 | // Scope permissions to describe on all resources, some APIs like ec2:DescribeAvailabilityZones do not Resource types 130 | // https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonec2.html#amazonec2-actions-as-permissions 131 | const readEBSPolicy = new PolicyStatement({ 132 | resources: [ 133 | '*', 134 | ], 135 | actions: [ 136 | 'ec2:DescribeAvailabilityZones', 137 | 'ec2:DescribeInstances', 138 | 'ec2:DescribeSnapshots', 139 | 'ec2:DescribeTags', 140 | 'ec2:DescribeVolumes', 141 | 'ec2:DescribeVolumesModifications', 142 | ], 143 | }); 144 | // Scope to createTags only creation of volumes and snapshots 145 | const createTags = new PolicyStatement({ 146 | resources: [ 147 | `arn:aws:ec2:${stack.region}:${stack.account}:volume/*`, 148 | `arn:aws:ec2:${stack.region}:${stack.account}:snapshot/*`, 149 | ], 150 | conditions: { 151 | StringEquals: { 152 | 'ec2:CreateAction': [ 153 | 'CreateVolume', 154 | 'CreateSnapshot', 155 | ], 156 | }, 157 | }, 158 | actions: [ 159 | 'ec2:CreateTags', 160 | ], 161 | }); 162 | // Scope deletion of tags on volumes and snapshots that already contain eks:cluster-name: MY_CLUSTER_NAME 163 | const deleteTags = new PolicyStatement({ 164 | resources: [ 165 | `arn:aws:ec2:${stack.region}:${stack.account}:volume/*`, 166 | `arn:aws:ec2:${stack.region}:${stack.account}:snapshot/*`, 167 | ], 168 | conditions: { 169 | StringEquals: { 170 | 'aws:ResourceTag/eks:cluster-name': clusterName, 171 | }, 172 | }, 173 | actions: [ 174 | 'ec2:DeleteTags', 175 | ], 176 | }); 177 | // Scope Attach/Detach/Modify of EBS policies to tags eks:cluster-name': MY_CLUSTER_NAME 178 | const modifyVolume = new PolicyStatement({ 179 | resources: [ 180 | `arn:aws:ec2:${stack.region}:${stack.account}:instance/*`, 181 | `arn:aws:ec2:${stack.region}:${stack.account}:volume/*`, 182 | ], 183 | actions: [ 184 | 'ec2:AttachVolume', 185 | 'ec2:DetachVolume', 186 | 'ec2:ModifyVolume', 187 | ], 188 | conditions: { 189 | StringEquals: { 190 | 'aws:ResourceTag/eks:cluster-name': clusterName, 191 | }, 192 | }, 193 | }); 194 | // Scope CreateVolume only when Request contains tags eks:cluster-name: MY_CLUSTER_NAME 195 | const createVolume = new PolicyStatement({ 196 | resources: [ 197 | '*', 198 | ], 199 | conditions: { 200 | StringEquals: { 201 | 'aws:RequestTag/eks:cluster-name': clusterName, 202 | }, 203 | }, 204 | actions: [ 205 | 'ec2:CreateVolume', 206 | ], 207 | }); 208 | // Scope DeleteVolume only when Resource contains tag eks:cluster-name: MY_CLUSTER_NAME 209 | const deleteVolume = new PolicyStatement({ 210 | resources: [ 211 | '*', 212 | ], 213 | conditions: { 214 | StringEquals: { 215 | 'aws:ResourceTag/eks:cluster-name': clusterName, 216 | }, 217 | }, 218 | actions: [ 219 | 'ec2:DeleteVolume', 220 | 'ec2:DetachVolume', 221 | 'ec2:AttachVolume', 222 | 'ec2:ModifyVolume', 223 | ], 224 | }); 225 | // Scope Permission to createsnapshot only when Request contains tag eks:cluster-name: MY_CLUSTER_NAME 226 | const createSnapshot = new PolicyStatement({ 227 | resources: [ 228 | '*', 229 | ], 230 | conditions: { 231 | StringEquals: { 232 | 'aws:RequestTag/eks:cluster-name': clusterName, 233 | }, 234 | }, 235 | actions: [ 236 | 'ec2:CreateSnapshot', 237 | ], 238 | }); 239 | // Scope Permission to DeleteSnapshot only when Resource contains tag eks:cluster-name: MY_CLUSTER_NAME 240 | const deleteSnapshot = new PolicyStatement({ 241 | resources: [ 242 | '*', 243 | ], 244 | conditions: { 245 | StringEquals: { 246 | 'aws:ResourceTag/eks:cluster-name': clusterName, 247 | }, 248 | }, 249 | actions: [ 250 | 'ec2:DeleteSnapshot', 251 | ], 252 | }); 253 | return new Policy(stack, 'ebsDriverPolicy', { 254 | roles: [ 255 | roleSA, 256 | ], 257 | statements: [ 258 | readEBSPolicy, 259 | createTags, 260 | deleteTags, 261 | createVolume, 262 | deleteVolume, 263 | modifyVolume, 264 | createSnapshot, 265 | deleteSnapshot, 266 | ], 267 | }); 268 | } 269 | 270 | export function createAlbIngressPolicy(stack: Stack, clusterName: string, roleSA: IRole): Policy { 271 | /* Permissions are board to include all functionality of ALB, Permissions can be removed as fit, refer to annotations to see which actions are needed 272 | https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.2/guide/ingress/annotations/ 273 | Custom Permission set can be generated by using IAM generated policy 274 | https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_generate-policy.html 275 | */ 276 | 277 | // Permission to create ELB ServiceLinkRole if not already created, scoped to only elasticloadbalancing service role 278 | const serviceLinkedRole = new PolicyStatement({ 279 | actions: [ 280 | 'iam:CreateServiceLinkedRole', 281 | ], 282 | resources: ['*'], 283 | conditions: { 284 | StringEquals: { 285 | 'iam:AWSServiceName': 'elasticloadbalancing.amazonaws.com', 286 | }, 287 | }, 288 | }); 289 | // Permission needs to self discovery networking attributes 290 | const readPolicy = new PolicyStatement({ 291 | actions: [ 292 | 'ec2:DescribeAccountAttributes', 293 | 'ec2:DescribeAddresses', 294 | 'ec2:DescribeAvailabilityZones', 295 | 'ec2:DescribeInternetGateways', 296 | 'ec2:DescribeVpcs', 297 | 'ec2:DescribeSubnets', 298 | 'ec2:DescribeSecurityGroups', 299 | 'ec2:DescribeInstances', 300 | 'ec2:DescribeNetworkInterfaces', 301 | 'ec2:DescribeTags', 302 | 'ec2:GetCoipPoolUsage', 303 | 'ec2:DescribeCoipPools', 304 | 'elasticloadbalancing:DescribeLoadBalancers', 305 | 'elasticloadbalancing:DescribeLoadBalancerAttributes', 306 | 'elasticloadbalancing:DescribeListeners', 307 | 'elasticloadbalancing:DescribeListenerCertificates', 308 | 'elasticloadbalancing:DescribeSSLPolicies', 309 | 'elasticloadbalancing:DescribeRules', 310 | 'elasticloadbalancing:DescribeTargetGroups', 311 | 'elasticloadbalancing:DescribeTargetGroupAttributes', 312 | 'elasticloadbalancing:DescribeTargetHealth', 313 | 'elasticloadbalancing:DescribeTags', 314 | ], 315 | resources: ['*'], 316 | }); 317 | // Additional Permissions for shield, waf acm and cognito feature set enablement 318 | const readPolicyAdd = new PolicyStatement({ 319 | actions: [ 320 | 'cognito-idp:DescribeUserPoolClient', 321 | 'acm:ListCertificates', 322 | 'acm:DescribeCertificate', 323 | 'iam:ListServerCertificates', 324 | 'iam:GetServerCertificate', 325 | 'waf-regional:GetWebACL', 326 | 'waf-regional:GetWebACLForResource', 327 | 'waf-regional:AssociateWebACL', 328 | 'waf-regional:DisassociateWebACL', 329 | 'wafv2:GetWebACL', 330 | 'wafv2:GetWebACLForResource', 331 | 'wafv2:AssociateWebACL', 332 | 'wafv2:DisassociateWebACL', 333 | 'shield:GetSubscriptionState', 334 | 'shield:DescribeProtection', 335 | 'shield:CreateProtection', 336 | 'shield:DeleteProtection', 337 | ], 338 | resources: ['*'], 339 | }); 340 | // Enable usage of ingress rule for security groups created outside of controller 341 | const writeSG = new PolicyStatement({ 342 | actions: [ 343 | 'ec2:AuthorizeSecurityGroupIngress', 344 | 'ec2:RevokeSecurityGroupIngress', 345 | ], 346 | resources: ['*'], 347 | }); 348 | // Enable controller to automatically create security groups, tags may be added later 349 | const createSG = new PolicyStatement({ 350 | 351 | actions: [ 352 | 'ec2:CreateSecurityGroup', 353 | ], 354 | resources: ['*'], 355 | }); 356 | // Give tagging permission to actions that create the resource, CreateSecurityGroup 357 | const createTags = new PolicyStatement({ 358 | 359 | actions: [ 360 | 'ec2:CreateTags', 361 | ], 362 | resources: ['arn:aws:ec2:*:*:security-group/*'], 363 | conditions: { 364 | StringEquals: { 365 | 'ec2:CreateAction': 'CreateSecurityGroup', 366 | }, 367 | Null: { 368 | 'aws:RequestTag/elbv2.k8s.aws/cluster': 'false', 369 | }, 370 | }, 371 | }); 372 | // Create and Delete Tags for security groups when at time of authorization aws:RequestTag/elbv2.k8s.aws/cluster is null 373 | const createdeleteTags = new PolicyStatement({ 374 | 375 | actions: [ 376 | 'ec2:CreateTags', 377 | 'ec2:DeleteTags', 378 | ], 379 | resources: ['arn:aws:ec2:*:*:security-group/*'], 380 | conditions: { 381 | Null: { 382 | 'aws:RequestTag/elbv2.k8s.aws/cluster': 'true', 383 | 'aws:ResourceTag/elbv2.k8s.aws/cluster': 'false', 384 | }, 385 | }, 386 | }); 387 | // Management of SecurityGroup when at time of authorization aws:ResourceTag/elbv2.k8s.aws/cluster is not null 388 | const writeSGIngress = new PolicyStatement({ 389 | 390 | actions: [ 391 | 'ec2:AuthorizeSecurityGroupIngress', 392 | 'ec2:RevokeSecurityGroupIngress', 393 | 'ec2:DeleteSecurityGroup', 394 | ], 395 | resources: ['*'], 396 | conditions: { 397 | Null: { 398 | 'aws:ResourceTag/elbv2.k8s.aws/cluster': 'false', 399 | }, 400 | }, 401 | }); 402 | // Allow creation of LoadBalancer/TargetGroup only if Request contains Tags eks:cluster-name: MY_CLUSTER_NAME 403 | const createLoadBalancer = new PolicyStatement({ 404 | 405 | actions: [ 406 | 'elasticloadbalancing:CreateLoadBalancer', 407 | 'elasticloadbalancing:CreateTargetGroup', 408 | ], 409 | resources: ['*'], 410 | conditions: { 411 | StringEquals: { 412 | 'aws:RequestTag/eks:cluster-name': clusterName, 413 | }, 414 | }, 415 | }); 416 | // Management of LoadBalancer Listeners and Rules 417 | // TODO Scope to use tags with release of v2.2.0 418 | // https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/1966 419 | const createLoadBalancerAdd = new PolicyStatement({ 420 | 421 | actions: [ 422 | 'elasticloadbalancing:CreateListener', 423 | 'elasticloadbalancing:DeleteListener', 424 | 'elasticloadbalancing:CreateRule', 425 | 'elasticloadbalancing:DeleteRule', 426 | ], 427 | resources: ['*'], 428 | }); 429 | // Management of ELB Tags when at authorization time aws:RequestTag/elbv2.k8s.aws/cluster is null 430 | const loadBalancerTags = new PolicyStatement( 431 | { 432 | 433 | actions: [ 434 | 'elasticloadbalancing:AddTags', 435 | 'elasticloadbalancing:RemoveTags', 436 | ], 437 | resources: [ 438 | 'arn:aws:elasticloadbalancing:*:*:targetgroup/*/*', 439 | 'arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*', 440 | 'arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*', 441 | ], 442 | conditions: { 443 | Null: { 444 | 'aws:RequestTag/elbv2.k8s.aws/cluster': 'true', 445 | 'aws:ResourceTag/elbv2.k8s.aws/cluster': 'false', 446 | }, 447 | }, 448 | }); 449 | // Management of ListenerTags 450 | // TODO Scope using Tags RequestTags for AddTags 451 | // https://docs.aws.amazon.com/service-authorization/latest/reference/list_elasticloadbalancingv2.html#elasticloadbalancingv2-actions-as-permissions 452 | const loadBalancerListenersTags = new PolicyStatement({ 453 | 454 | actions: [ 455 | 'elasticloadbalancing:AddTags', 456 | 'elasticloadbalancing:RemoveTags', 457 | ], 458 | resources: [ 459 | 'arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*', 460 | 'arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*', 461 | 'arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*', 462 | 'arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*', 463 | ], 464 | }); 465 | // Management of LoadBalancer Targetgroup and Attributes, scoped to when at authorization time aws:ResourceTag/elbv2.k8s.aws/cluster is not null 466 | const modifyLoadBalancer = new PolicyStatement({ 467 | 468 | actions: [ 469 | 'elasticloadbalancing:ModifyLoadBalancerAttributes', 470 | 'elasticloadbalancing:SetIpAddressType', 471 | 'elasticloadbalancing:SetSecurityGroups', 472 | 'elasticloadbalancing:SetSubnets', 473 | 'elasticloadbalancing:ModifyTargetGroup', 474 | 'elasticloadbalancing:ModifyTargetGroupAttributes', 475 | ], 476 | resources: ['*'], 477 | conditions: { 478 | Null: { 479 | 'aws:ResourceTag/elbv2.k8s.aws/cluster': 'false', 480 | }, 481 | }, 482 | }); 483 | // Delete Load Balancer and TargetGroups to tag eks:cluster-name : MY_CLUSTER_NAME 484 | const deleteLoadBalancer = new PolicyStatement({ 485 | resources: ['*'], 486 | actions: [ 487 | 'elasticloadbalancing:DeleteTargetGroup', 488 | 'elasticloadbalancing:DeleteLoadBalancer', 489 | ], 490 | conditions: { 491 | StringEquals: { 492 | 'aws:ResourceTag/eks:cluster-name': clusterName, 493 | }, 494 | }, 495 | }); 496 | // Management of Target scoped to target-groups 497 | const registerTarget = new PolicyStatement({ 498 | 499 | actions: [ 500 | 'elasticloadbalancing:RegisterTargets', 501 | 'elasticloadbalancing:DeregisterTargets', 502 | ], 503 | resources: ['arn:aws:elasticloadbalancing:*:*:targetgroup/*/*'], 504 | }); 505 | // Management of LoadBalancer Certs, WebACL and Rules 506 | const modifyLoadBalancerCerts = new PolicyStatement({ 507 | 508 | actions: [ 509 | 'elasticloadbalancing:SetWebAcl', 510 | 'elasticloadbalancing:ModifyListener', 511 | 'elasticloadbalancing:AddListenerCertificates', 512 | 'elasticloadbalancing:RemoveListenerCertificates', 513 | 'elasticloadbalancing:ModifyRule', 514 | ], 515 | resources: ['*'], 516 | }); 517 | 518 | return new Policy(stack, 'albIngressPolicy', { 519 | roles: [ 520 | roleSA, 521 | ], 522 | statements: [ 523 | modifyLoadBalancer, 524 | readPolicy, 525 | writeSG, 526 | createSG, 527 | readPolicyAdd, 528 | createTags, 529 | createdeleteTags, 530 | writeSGIngress, 531 | createLoadBalancer, 532 | loadBalancerTags, 533 | createLoadBalancerAdd, 534 | loadBalancerListenersTags, 535 | registerTarget, 536 | modifyLoadBalancer, 537 | modifyLoadBalancerCerts, 538 | deleteLoadBalancer, 539 | serviceLinkedRole, 540 | ], 541 | }); 542 | } 543 | -------------------------------------------------------------------------------- /lib/vpc-stack.ts: -------------------------------------------------------------------------------- 1 | import { Stack } from 'aws-cdk-lib'; 2 | import { 3 | GatewayVpcEndpointAwsService, 4 | Vpc, 5 | FlowLogTrafficType, 6 | FlowLogDestination, 7 | InterfaceVpcEndpoint, 8 | } from 'aws-cdk-lib/aws-ec2'; 9 | 10 | // Private Endpoints 11 | // https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#vpc-endpoints-private-clusters 12 | // EKS VPC Endpoint RoadMap https://github.com/aws/containers-roadmap/issues/298 13 | 14 | export function addEndpoint (stack: Stack, vpc: Vpc): void { 15 | // Additional VPC Endpoint for EKS https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#vpc-endpoints-private-clusters 16 | (() => new InterfaceVpcEndpoint(stack, 'ecrapiVpcEndpoint', { 17 | open: true, 18 | vpc: vpc, 19 | service: { 20 | name: `com.amazonaws.${stack.region}.ecr.api`, 21 | port: 443, 22 | }, 23 | privateDnsEnabled: true, 24 | }))(); 25 | 26 | (() => new InterfaceVpcEndpoint(stack, 'ecradkrVpcEndpoint', { 27 | open: true, 28 | vpc: vpc, 29 | service: { 30 | name: `com.amazonaws.${stack.region}.ecr.dkr`, 31 | port: 443, 32 | }, 33 | privateDnsEnabled: true, 34 | }))(); 35 | } 36 | 37 | export const eksVpc = { 38 | cidr: '10.0.0.0/16', 39 | maxAzs: 3, 40 | // S3/DynamoDB https://docs.aws.amazon.com/vpc/latest/privatelink/vpce-gateway.html 41 | gatewayEndpoints: { 42 | // S3 Gateway https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html#types-of-vpc-endpoints-for-s3 43 | // S3 Gateway vs Private Endpoint https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html#types-of-vpc-endpoints-for-s3 44 | S3: { 45 | service: GatewayVpcEndpointAwsService.S3, 46 | }, 47 | 48 | }, 49 | flowLogs: { 50 | VpcFlowlogs: { 51 | destination: FlowLogDestination.toCloudWatchLogs(), 52 | trafficType: FlowLogTrafficType.ALL, 53 | }, 54 | }, 55 | // TWO Nat Gateways for higher availability 56 | natGateways: 2, 57 | }; 58 | -------------------------------------------------------------------------------- /manifests/awsSecretsManifest.yaml: -------------------------------------------------------------------------------- 1 | # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # From https://github.com/aws/secrets-store-csi-driver-provider-aws/blob/main/deployment/aws-provider-installer.yaml 3 | # SPDX-License-Identifier: Apache-2.0 4 | 5 | # https://kubernetes.io/docs/reference/access-authn-authz/rbac 6 | apiVersion: v1 7 | kind: ServiceAccount 8 | metadata: 9 | name: csi-secrets-store-provider-aws 10 | namespace: kube-system 11 | --- 12 | apiVersion: rbac.authorization.k8s.io/v1 13 | kind: ClusterRole 14 | metadata: 15 | name: csi-secrets-store-provider-aws-cluster-role 16 | rules: 17 | - apiGroups: [""] 18 | resources: ["serviceaccounts/token"] 19 | verbs: ["create"] 20 | - apiGroups: [""] 21 | resources: ["serviceaccounts"] 22 | verbs: ["get"] 23 | - apiGroups: [""] 24 | resources: ["pods"] 25 | verbs: ["get"] 26 | - apiGroups: [""] 27 | resources: ["nodes"] 28 | verbs: ["get"] 29 | --- 30 | apiVersion: rbac.authorization.k8s.io/v1 31 | kind: ClusterRoleBinding 32 | metadata: 33 | name: csi-secrets-store-provider-aws-cluster-rolebinding 34 | roleRef: 35 | apiGroup: rbac.authorization.k8s.io 36 | kind: ClusterRole 37 | name: csi-secrets-store-provider-aws-cluster-role 38 | subjects: 39 | - kind: ServiceAccount 40 | name: csi-secrets-store-provider-aws 41 | namespace: kube-system 42 | --- 43 | apiVersion: apps/v1 44 | kind: DaemonSet 45 | metadata: 46 | namespace: kube-system 47 | name: csi-secrets-store-provider-aws 48 | labels: 49 | app: csi-secrets-store-provider-aws 50 | spec: 51 | updateStrategy: 52 | type: RollingUpdate 53 | selector: 54 | matchLabels: 55 | app: csi-secrets-store-provider-aws 56 | template: 57 | metadata: 58 | labels: 59 | app: csi-secrets-store-provider-aws 60 | spec: 61 | serviceAccountName: csi-secrets-store-provider-aws 62 | hostNetwork: true 63 | containers: 64 | - name: provider-aws-installer 65 | image: public.ecr.aws/aws-secrets-manager/secrets-store-csi-driver-provider-aws:1.0.r1-10-g1942553-2021.06.04.00.07-linux-amd64 66 | imagePullPolicy: Always 67 | args: 68 | - --provider-volume=/etc/kubernetes/secrets-store-csi-providers 69 | resources: 70 | requests: 71 | cpu: 50m 72 | memory: 100Mi 73 | limits: 74 | cpu: 50m 75 | memory: 100Mi 76 | volumeMounts: 77 | - mountPath: "/etc/kubernetes/secrets-store-csi-providers" 78 | name: providervol 79 | - name: mountpoint-dir 80 | mountPath: /var/lib/kubelet/pods 81 | mountPropagation: HostToContainer 82 | volumes: 83 | - name: providervol 84 | hostPath: 85 | path: "/etc/kubernetes/secrets-store-csi-providers" 86 | - name: mountpoint-dir 87 | hostPath: 88 | path: /var/lib/kubelet/pods 89 | type: DirectoryOrCreate 90 | nodeSelector: 91 | kubernetes.io/os: linux 92 | -------------------------------------------------------------------------------- /manifests/consoleViewOnlyGroup.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | # From https://github.com/awsdocs/amazon-eks-user-guide/blob/master/doc_source/add-user-role.md 3 | # SPDX-License-Identifier: MIT-0 4 | 5 | apiVersion: rbac.authorization.k8s.io/v1 6 | kind: ClusterRole 7 | metadata: 8 | name: eks-console-dashboard-full-access-clusterrole 9 | rules: 10 | - apiGroups: 11 | - "" 12 | resources: 13 | - nodes 14 | - namespaces 15 | - pods 16 | verbs: 17 | - get 18 | - list 19 | - apiGroups: 20 | - apps 21 | resources: 22 | - deployments 23 | - daemonsets 24 | - statefulsets 25 | - replicasets 26 | verbs: 27 | - get 28 | - list 29 | - apiGroups: 30 | - batch 31 | resources: 32 | - jobs 33 | verbs: 34 | - get 35 | - list 36 | --- 37 | apiVersion: rbac.authorization.k8s.io/v1 38 | kind: ClusterRoleBinding 39 | metadata: 40 | name: eks-console-dashboard-full-access-binding 41 | subjects: 42 | - kind: Group 43 | name: eks-console-dashboard-full-access-group 44 | apiGroup: rbac.authorization.k8s.io 45 | roleRef: 46 | kind: ClusterRole 47 | name: eks-console-dashboard-full-access-clusterrole 48 | apiGroup: rbac.authorization.k8s.io 49 | -------------------------------------------------------------------------------- /manifests/fluentBitSetup.yaml: -------------------------------------------------------------------------------- 1 | # From https://github.com/aws-samples/amazon-cloudwatch-container-insights/blob/master/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml 2 | # SPDX-License-Identifier: MIT-0 3 | 4 | apiVersion: v1 5 | kind: ServiceAccount 6 | metadata: 7 | name: fluent-bit 8 | namespace: amazon-cloudwatch 9 | --- 10 | apiVersion: rbac.authorization.k8s.io/v1 11 | kind: ClusterRole 12 | metadata: 13 | name: fluent-bit-role 14 | rules: 15 | - nonResourceURLs: 16 | - /metrics 17 | verbs: 18 | - get 19 | - apiGroups: [""] 20 | resources: 21 | - namespaces 22 | - pods 23 | - pods/logs 24 | verbs: ["get", "list", "watch"] 25 | --- 26 | apiVersion: rbac.authorization.k8s.io/v1 27 | kind: ClusterRoleBinding 28 | metadata: 29 | name: fluent-bit-role-binding 30 | roleRef: 31 | apiGroup: rbac.authorization.k8s.io 32 | kind: ClusterRole 33 | name: fluent-bit-role 34 | subjects: 35 | - kind: ServiceAccount 36 | name: fluent-bit 37 | namespace: amazon-cloudwatch 38 | --- 39 | apiVersion: v1 40 | kind: ConfigMap 41 | metadata: 42 | name: fluent-bit-config 43 | namespace: amazon-cloudwatch 44 | labels: 45 | k8s-app: fluent-bit 46 | data: 47 | fluent-bit.conf: | 48 | [SERVICE] 49 | Flush 5 50 | Log_Level info 51 | Daemon off 52 | Parsers_File parsers.conf 53 | HTTP_Server ${HTTP_SERVER} 54 | HTTP_Listen 0.0.0.0 55 | HTTP_Port ${HTTP_PORT} 56 | storage.path /var/fluent-bit/state/flb-storage/ 57 | storage.sync normal 58 | storage.checksum off 59 | storage.backlog.mem_limit 5M 60 | 61 | @INCLUDE application-log.conf 62 | @INCLUDE dataplane-log.conf 63 | @INCLUDE host-log.conf 64 | 65 | application-log.conf: | 66 | [INPUT] 67 | Name tail 68 | Tag application.* 69 | Exclude_Path /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy* 70 | Path /var/log/containers/*.log 71 | Docker_Mode On 72 | Docker_Mode_Flush 5 73 | Docker_Mode_Parser container_firstline 74 | Parser docker 75 | DB /var/fluent-bit/state/flb_container.db 76 | Mem_Buf_Limit 50MB 77 | Skip_Long_Lines On 78 | Refresh_Interval 10 79 | Rotate_Wait 30 80 | storage.type filesystem 81 | Read_from_Head ${READ_FROM_HEAD} 82 | 83 | [INPUT] 84 | Name tail 85 | Tag application.* 86 | Path /var/log/containers/fluent-bit* 87 | Parser docker 88 | DB /var/fluent-bit/state/flb_log.db 89 | Mem_Buf_Limit 5MB 90 | Skip_Long_Lines On 91 | Refresh_Interval 10 92 | Read_from_Head ${READ_FROM_HEAD} 93 | 94 | [INPUT] 95 | Name tail 96 | Tag application.* 97 | Path /var/log/containers/cloudwatch-agent* 98 | Docker_Mode On 99 | Docker_Mode_Flush 5 100 | Docker_Mode_Parser cwagent_firstline 101 | Parser docker 102 | DB /var/fluent-bit/state/flb_cwagent.db 103 | Mem_Buf_Limit 5MB 104 | Skip_Long_Lines On 105 | Refresh_Interval 10 106 | Read_from_Head ${READ_FROM_HEAD} 107 | 108 | [FILTER] 109 | Name kubernetes 110 | Match application.* 111 | Kube_URL https://kubernetes.default.svc:443 112 | Kube_Tag_Prefix application.var.log.containers. 113 | Merge_Log On 114 | Merge_Log_Key log_processed 115 | K8S-Logging.Parser On 116 | K8S-Logging.Exclude Off 117 | Labels Off 118 | Annotations Off 119 | 120 | [OUTPUT] 121 | Name cloudwatch_logs 122 | Match application.* 123 | region ${AWS_REGION} 124 | log_group_name /aws/containerinsights/${CLUSTER_NAME}/application 125 | log_stream_prefix ${HOST_NAME}- 126 | auto_create_group true 127 | extra_user_agent container-insights 128 | 129 | dataplane-log.conf: | 130 | [INPUT] 131 | Name systemd 132 | Tag dataplane.systemd.* 133 | Systemd_Filter _SYSTEMD_UNIT=docker.service 134 | Systemd_Filter _SYSTEMD_UNIT=kubelet.service 135 | DB /var/fluent-bit/state/systemd.db 136 | Path /var/log/journal 137 | Read_From_Tail ${READ_FROM_TAIL} 138 | 139 | [INPUT] 140 | Name tail 141 | Tag dataplane.tail.* 142 | Path /var/log/containers/aws-node*, /var/log/containers/kube-proxy* 143 | Docker_Mode On 144 | Docker_Mode_Flush 5 145 | Docker_Mode_Parser container_firstline 146 | Parser docker 147 | DB /var/fluent-bit/state/flb_dataplane_tail.db 148 | Mem_Buf_Limit 50MB 149 | Skip_Long_Lines On 150 | Refresh_Interval 10 151 | Rotate_Wait 30 152 | storage.type filesystem 153 | Read_from_Head ${READ_FROM_HEAD} 154 | 155 | [FILTER] 156 | Name modify 157 | Match dataplane.systemd.* 158 | Rename _HOSTNAME hostname 159 | Rename _SYSTEMD_UNIT systemd_unit 160 | Rename MESSAGE message 161 | Remove_regex ^((?!hostname|systemd_unit|message).)*$ 162 | 163 | [FILTER] 164 | Name aws 165 | Match dataplane.* 166 | imds_version v1 167 | 168 | [OUTPUT] 169 | Name cloudwatch_logs 170 | Match dataplane.* 171 | region ${AWS_REGION} 172 | log_group_name /aws/containerinsights/${CLUSTER_NAME}/dataplane 173 | log_stream_prefix ${HOST_NAME}- 174 | auto_create_group true 175 | extra_user_agent container-insights 176 | 177 | host-log.conf: | 178 | [INPUT] 179 | Name tail 180 | Tag host.dmesg 181 | Path /var/log/dmesg 182 | Parser syslog 183 | DB /var/fluent-bit/state/flb_dmesg.db 184 | Mem_Buf_Limit 5MB 185 | Skip_Long_Lines On 186 | Refresh_Interval 10 187 | Read_from_Head ${READ_FROM_HEAD} 188 | 189 | [INPUT] 190 | Name tail 191 | Tag host.messages 192 | Path /var/log/messages 193 | Parser syslog 194 | DB /var/fluent-bit/state/flb_messages.db 195 | Mem_Buf_Limit 5MB 196 | Skip_Long_Lines On 197 | Refresh_Interval 10 198 | Read_from_Head ${READ_FROM_HEAD} 199 | 200 | [INPUT] 201 | Name tail 202 | Tag host.secure 203 | Path /var/log/secure 204 | Parser syslog 205 | DB /var/fluent-bit/state/flb_secure.db 206 | Mem_Buf_Limit 5MB 207 | Skip_Long_Lines On 208 | Refresh_Interval 10 209 | Read_from_Head ${READ_FROM_HEAD} 210 | 211 | [FILTER] 212 | Name aws 213 | Match host.* 214 | imds_version v1 215 | 216 | [OUTPUT] 217 | Name cloudwatch_logs 218 | Match host.* 219 | region ${AWS_REGION} 220 | log_group_name /aws/containerinsights/${CLUSTER_NAME}/host 221 | log_stream_prefix ${HOST_NAME}. 222 | auto_create_group true 223 | extra_user_agent container-insights 224 | 225 | parsers.conf: | 226 | [PARSER] 227 | Name docker 228 | Format json 229 | Time_Key time 230 | Time_Format %Y-%m-%dT%H:%M:%S.%LZ 231 | 232 | [PARSER] 233 | Name syslog 234 | Format regex 235 | Regex ^(?