├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── publish.yml │ └── validate.yml ├── .gitignore ├── .golangci.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── charts └── eks-ng-ami-updater │ ├── Chart.yaml │ ├── templates │ ├── _helpers.tpl │ ├── cronjob.yaml │ ├── namespace.yaml │ └── serviceaccount.yaml │ └── values.yaml ├── go.mod ├── go.sum ├── main.go └── pkg ├── aws ├── ami.go ├── ami_test.go ├── aws.go ├── clusters.go ├── clusters_test.go ├── ec2.go ├── eks.go ├── nodegroups.go ├── nodegroups_test.go ├── regions.go ├── regions_test.go ├── ssm.go └── tests.go ├── flags └── flags.go ├── logs └── log.go ├── updater └── updater.go └── utils └── utils.go /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @loomhq/infrastructure 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: 'bug' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Please ensure you are using the latest version of EKS NG Ami Updater. 11 | 12 | **Describe the bug** 13 | A clear and concise description of what the bug is. 14 | 15 | **To Reproduce** 16 | Steps to reproduce the behavior. 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Additional context** 22 | Add any other context about the problem here. 23 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: 'enhancement' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Dependabot will create PRs to update our third party github actions versions 4 | - package-ecosystem: github-actions 5 | open-pull-requests-limit: 100 6 | directory: / 7 | schedule: 8 | time: "16:00" 9 | interval: daily 10 | allow: 11 | - dependency-type: all 12 | 13 | - package-ecosystem: gomod 14 | open-pull-requests-limit: 100 15 | directory: / 16 | schedule: 17 | time: "16:00" 18 | interval: daily 19 | allow: 20 | - dependency-type: all 21 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | env: 9 | AWS_REGION: us-west-2 10 | APP_NAME: eks-ng-ami-updater 11 | AWS_ECR_REPO: public.ecr.aws/loom 12 | 13 | jobs: 14 | publish: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout Repo 18 | uses: actions/checkout@v4 19 | - name: Install Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version: stable # https://golang.org/dl/ 23 | - name: Set up QEMU 24 | uses: docker/setup-qemu-action@v3 25 | - name: Set up Docker Buildx 26 | uses: docker/setup-buildx-action@v3 27 | - name: AWS Credentials 28 | uses: aws-actions/configure-aws-credentials@v4 29 | with: 30 | aws-region: us-east-1 31 | aws-access-key-id: ${{ secrets.CI_PUBLIC_REPOS_AWS_ACCESS_KEY_ID }} 32 | aws-secret-access-key: ${{ secrets.CI_PUBLIC_REPOS_AWS_SECRET_ACCESS_KEY }} 33 | - name: ECR Login 34 | uses: aws-actions/amazon-ecr-login@v2 35 | with: 36 | registry-type: public 37 | mask-password: true 38 | - name: Build & Push 39 | run: make docker-push IMG=${AWS_ECR_REPO}/${APP_NAME} TAG=${GITHUB_REF_NAME} 40 | -------------------------------------------------------------------------------- /.github/workflows/validate.yml: -------------------------------------------------------------------------------- 1 | name: Validate 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | 7 | jobs: 8 | validate: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Install Go 12 | uses: actions/setup-go@v5 13 | with: 14 | go-version: stable # https://golang.org/dl/ 15 | - name: Checkout 16 | uses: actions/checkout@v4 17 | - name: Hadolint 18 | uses: hadolint/hadolint-action@v3.1.0 19 | with: 20 | ignore: DL3029 # build amd64 image 21 | - name: Go Mod Tidy 22 | run: test -z $(go mod tidy) 23 | - name: Lint 24 | uses: golangci/golangci-lint-action@v6.3.3 25 | - name: Test 26 | run: make test 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Test binary, build with `go test -c` 2 | *.test 3 | 4 | # Output of the go coverage tool, specifically when used with LiteIDE 5 | *.out 6 | 7 | # Kubernetes Generated files - skip generated files, except for vendored files 8 | !vendor/**/zz_generated.* 9 | 10 | # editor and IDE paraphernalia 11 | .idea 12 | *.swp 13 | *.swo 14 | .vscode 15 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | 4 | linters: 5 | enable: 6 | - asciicheck 7 | - bidichk 8 | - bodyclose 9 | - containedctx 10 | - contextcheck 11 | - copyloopvar 12 | - durationcheck 13 | - errcheck 14 | - errname 15 | - errorlint 16 | - exhaustive 17 | - forcetypeassert 18 | - funlen 19 | - gochecknoglobals 20 | - gocognit 21 | - goconst 22 | - gocritic 23 | - gocyclo 24 | - godot 25 | - gofmt 26 | - goimports 27 | - goprintffuncname 28 | - gosec 29 | - gosimple 30 | - govet 31 | - ineffassign 32 | - ireturn 33 | - maintidx 34 | - makezero 35 | - misspell 36 | - mnd 37 | - nakedret 38 | - nestif 39 | - nilerr 40 | - nilnil 41 | - nlreturn 42 | - noctx 43 | - nolintlint 44 | - paralleltest 45 | - predeclared 46 | - staticcheck 47 | - stylecheck 48 | - tagliatelle 49 | - tenv 50 | - thelper 51 | - tparallel 52 | - typecheck 53 | - unconvert 54 | - unused 55 | - whitespace 56 | - wrapcheck 57 | 58 | linters-settings: 59 | errcheck: 60 | check-blank: true 61 | check-type-assertions: true 62 | funlen: 63 | lines: 80 64 | goconst: 65 | min-occurrences: 3 66 | gocognit: 67 | min-complexity: 25 68 | gocyclo: 69 | min-complexity: 25 70 | makezero: 71 | always: true 72 | nakedret: 73 | max-func-lines: 0 74 | nestif: 75 | min-complexity: 6 76 | nolintlint: 77 | require-specific: true 78 | require-explanation: true 79 | allow-leading-space: false 80 | whitespace: 81 | multi-if: true 82 | multi-func: true 83 | 84 | issues: 85 | max-same-issues: 0 # unlimited 86 | max-issues-per-linter: 0 # unlimited 87 | 88 | exclude-rules: 89 | - path: pkg/aws/ami.go 90 | linters: 91 | - wrapcheck # errors are wrapped in other functions 92 | - path: pkg/updater/updater.go 93 | linters: 94 | - wrapcheck # errors are wrapped in other functions 95 | - gocognit # the main function can have bigger complexity 96 | - nestif # the main function can have bigger complexity 97 | - funlen # the main function can have bigger complexity 98 | - gocyclo # the main function can have bigger complexity 99 | - path: pkg/aws/nodegroups.go 100 | linters: 101 | - wrapcheck # errors are wrapped in other functions 102 | - path: pkg/aws/clusters.go 103 | linters: 104 | - wrapcheck # errors are wrapped in other functions 105 | - path: pkg/aws/regions.go 106 | linters: 107 | - wrapcheck # errors are wrapped in other functions 108 | - path: pkg/aws/updater.go 109 | linters: 110 | - nestif 111 | - path: pkg/aws/ami_test.go 112 | linters: 113 | - funlen # test function can be long 114 | - path: pkg/aws/nodegroups_test.go 115 | linters: 116 | - funlen # test function can be long 117 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.21 as builder 3 | ARG TARGETOS 4 | ARG TARGETARCH 5 | 6 | WORKDIR /workspace 7 | # Copy the Go Modules manifests 8 | COPY go.mod go.mod 9 | COPY go.sum go.sum 10 | # cache deps before building and copying source so that we don't need to re-download as much 11 | # and so that source changes don't invalidate our downloaded layer 12 | RUN go mod download 13 | 14 | # Copy the go source 15 | COPY main.go main.go 16 | COPY pkg/ pkg/ 17 | 18 | # Build 19 | # the GOARCH has not a default value to allow the binary be built according to the host where the command 20 | # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO 21 | # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, 22 | # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. 23 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o eks-ng-ami-updater main.go 24 | 25 | # Use distroless as minimal base image to package the manager binary 26 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 27 | FROM gcr.io/distroless/static:nonroot 28 | WORKDIR / 29 | COPY --from=builder /workspace/eks-ng-ami-updater . 30 | USER 65532:65532 31 | 32 | ENTRYPOINT ["/eks-ng-ami-updater"] 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Loom Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Image URL to use all building/pushing image targets 2 | IMG ?= aws-ng-ami-updater 3 | TAG ?= latest 4 | 5 | ##@ General 6 | 7 | .PHONY: help 8 | help: ## Display this help. 9 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 10 | 11 | .PHONY: fmt 12 | fmt: ## Run go fmt against code. 13 | go fmt ./... 14 | 15 | .PHONY: vet 16 | vet: ## Run go vet against code. 17 | go vet ./... 18 | 19 | .PHONY: test 20 | test: fmt vet ## Run tests. 21 | go test ./... 22 | 23 | ##@ Build 24 | 25 | .PHONY: docker-build 26 | docker-build: test ## Build docker image. 27 | docker buildx build \ 28 | -t ${IMG}:${TAG} \ 29 | -t ${IMG}:latest \ 30 | --load \ 31 | . 32 | 33 | .PHONY: docker-push 34 | docker-push: ## Push docker image. 35 | docker buildx build \ 36 | --platform=linux/amd64,linux/arm64 \ 37 | -t ${IMG}:${TAG} \ 38 | -t ${IMG}:latest \ 39 | --push \ 40 | . 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EKS Node Group AMI Updater 2 | 3 | [![Validation](https://github.com/loomhq/eks-ng-ami-updater/actions/workflows/validate.yml/badge.svg)](https://github.com/loomhq/eks-ng-ami-updater/actions/workflows/validate.yml) 4 | [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://opensource.org/license/mit/) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/loomhq/eks-ng-ami-updater)](https://goreportcard.com/badge/github.com/loomhq/eks-ng-ami-updater) 6 | 7 | EKS NG AMI Updater is an open source project that can be used to update kubernetes node group images. It is deployed as cronjob that runs weekly. By default it will find all node groups in all your EKS clusters and update them to the newest node group AMI if there is one available. 8 | 9 | ## Installation 10 | 11 | To install the EKS NG AMI Updater in your cluster, execute the following commands: 12 | 13 | #### Create IAM role 14 | 15 | Create AWS role named `eks-nk-ami-updater` within below policy: 16 | 17 | ``` 18 | "Version": "2012-10-17", 19 | "Statement": [ 20 | { 21 | "Effect": "Allow", 22 | "Action": [ 23 | "ec2:DescribeRegions", 24 | "ec2:DescribeImages", 25 | "ec2:DescribeLaunchTemplateVersions", 26 | "ec2:RunInstances", 27 | "ec2:CreateTags" 28 | ], 29 | "Resource": "*" 30 | }, 31 | { 32 | "Effect": "Allow", 33 | "Action": [ 34 | "eks:DescribeNodegroup", 35 | "eks:ListNodegroups", 36 | "eks:ListClusters", 37 | "eks:UpdateNodegroupVersion" 38 | ], 39 | "Resource": "*" 40 | }, 41 | { 42 | "Effect": "Allow", 43 | "Action": "ssm:GetParameter", 44 | "Resource": "*" 45 | } 46 | ] 47 | ``` 48 | 49 | #### Deploy application 50 | 51 | Use below command to deploy EKS NG AMI Updater into your cluster (replace `` with your AWS account ID): 52 | 53 | `helm install eks-nk-ami-updater oci://public.ecr.aws/loom/eks-ng-ami-updater --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam:::role/eks-ng-ami-updater"` 54 | 55 | ## Parameters 56 | 57 | | Command line flags | Value keys | Type | Default | Description | 58 | | ---------------------- | ------------------------------- | ------ | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | 59 | | --debug | cmdOptions.debug | bool | false | set log level to debug (eg. `--debug=true`) | 60 | | --dryrun | cmdOptions.dryrun | bool | false | set dryrun mode (eg. `--dryrun=true`) | 61 | | --nodegroups | cmdOptions.nodegroups | string | "" | limit update amis to specified nodegroups (eg. `--nodegroups=eu-west-1:cluster-1:ngMain,eu-west-2:clusterStage:nodegroupStage1`) | 62 | | --regions | cmdOptions.regions | string | "" | limit update amis to nodegroups from specified regions only (eg. `--regions=eu-west-1,us-west-1`) | 63 | | --skip-newer-than-days | cmdOptions.skip-newer-than-days | int | 0 | skip ami update if the latest available in AWS ami image was published in less than provided number of days (eg. `--skip-newer-than-days=7`) | 64 | | --tag | cmdOptions.tag | string | "" | update amis only for nodegroups within this tag (eg. `--tag=env:production`) | 65 | | n/a | schedule | string | "30 7 * * 0" | schedule run within [cron syntax](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#schedule-syntax) | 66 | 67 | All flags are connected with the AND operator. E.g. if we use two such flags `"--regions=eu-west-1 --nodegroups=us-west-2:cluster-1:nodegroup1"` then no images will be updated due to mismatched regions. 68 | 69 | ## Examples 70 | 71 | `eks-ng-ami-updater --regions=us-west-1,us-west-2 --tag=env:production` - all nodes from any node groups from any clusters which are run in us-west-1 or us-west-1 region AND which have env tag set to production, will be updated. 72 | 73 | `eks-ng-ami-updater --nodegroups=eu-west-1:cluster-1:ngMain --skip-newer-than-days=7` - all nodes from 'ngMain' node group from 'cluster-1' cluster will be updated but only if the newest availiable AMI is older than 7 days. 74 | 75 | ## FAQ 76 | 77 | **Q:** I want to run updates in a testing environment first then in production a few days later. How do I do that? \ 78 | **A**: You can deploy EKS NG AMI Updater twice. The first one can be configured to do updates only for testing clusters and the second one can be configured (different 'schedule' definition) only for production. 79 | 80 | **Q:** I don't want to be the first person in the world to use a new just-published AWS AMI image. What can I do? \ 81 | **A:** You can use the `skip-newer-than-days` parameter to define a delay (we recommend max 7 days delay). 82 | 83 | **Q:** I set `skip-newer-than-days` parameter to 60 days and my AMI images haven't been updated for last 80 days. Is this normal? \ 84 | **A:** EKS NG AMI Updater is checking the release date for the last (newest) published by AWS AMI image. So if e.g. AWS releases new AMI images every 20 days than the newest availiable AWS AMI image will be always newer than the 60 day delay that you set. This is why we recommend setting the 'skip-newer-than-days' parameter to a max of 7 days. 85 | 86 | ## Maintainers 87 | 88 | This project was created by [Andrzej Wisniewski](https://github.com/AndrzejWisniewski) at [Loom](https://github.com/loomhq/). 89 | -------------------------------------------------------------------------------- /charts/eks-ng-ami-updater/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "v1.0.0" 3 | description: A Helm chart for EKS NG AMI Updater 4 | name: eks-ng-ami-updater 5 | version: 1.0.0 6 | keywords: 7 | - "ami updater" 8 | - "ami nodegroup" -------------------------------------------------------------------------------- /charts/eks-ng-ami-updater/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{- define "eks-ng-ami-updater.name" -}} 2 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 3 | {{- end -}} 4 | 5 | {{- define "eks-ng-ami-updater.chart" -}} 6 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{- define "eks-ng-ami-updater.serviceAccountName" -}} 10 | {{- if .Values.serviceAccount.create -}} 11 | {{ default (include "eks-ng-ami-updater.name" .) .Values.serviceAccount.name }} 12 | {{- else -}} 13 | {{ default "default" .Values.serviceAccount.name }} 14 | {{- end -}} 15 | {{- end -}} -------------------------------------------------------------------------------- /charts/eks-ng-ami-updater/templates/cronjob.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: batch/v1 2 | kind: CronJob 3 | metadata: 4 | namespace: {{ template "eks-ng-ami-updater.name" . }} 5 | name: {{ template "eks-ng-ami-updater.name" . }} 6 | labels: 7 | app: {{ template "eks-ng-ami-updater.name" . }} 8 | chart: {{ template "eks-ng-ami-updater.chart" . }} 9 | release: {{ .Release.Name }} 10 | spec: 11 | schedule: "{{ .Values.schedule }}" 12 | jobTemplate: 13 | metadata: 14 | namespace: {{ template "eks-ng-ami-updater.name" . }} 15 | labels: 16 | app: {{ template "eks-ng-ami-updater.name" . }} 17 | release: {{ .Release.Name }} 18 | {{- with .Values.annotations }} 19 | annotations: 20 | {{- toYaml . | nindent 8 }} 21 | {{- end }} 22 | {{- with .Values.nodeSelector }} 23 | nodeSelector: 24 | {{- toYaml . | nindent 8 }} 25 | {{- end }} 26 | {{- with .Values.affinity }} 27 | affinity: 28 | {{- toYaml . | nindent 8 }} 29 | {{- end }} 30 | {{- with .Values.tolerations }} 31 | tolerations: 32 | {{- toYaml . | nindent 8 }} 33 | {{- end }} 34 | spec: 35 | backoffLimit: 0 36 | template: 37 | spec: 38 | serviceAccountName: {{ template "eks-ng-ami-updater.serviceAccountName" . }} 39 | {{- if .Values.schedulerName }} 40 | schedulerName: {{ .Values.schedulerName | quote }} 41 | {{- end }} 42 | containers: 43 | - name: {{ .Chart.Name }} 44 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 45 | imagePullPolicy: {{ .Values.image.pullPolicy }} 46 | securityContext: 47 | privileged: false 48 | runAsUser: 1000 49 | runAsGroup: 1000 50 | runAsNonRoot: true 51 | readOnlyRootFilesystem: true 52 | command: 53 | - ./eks-ng-ami-updater 54 | {{- range $key, $value := .Values.cmdOptions }} 55 | - --{{ $key }}{{ if $value }}={{ $value }}{{ end }} 56 | {{- end }} 57 | restartPolicy: "Never" 58 | {{- with .Values.resources }} 59 | resources: 60 | {{- toYaml .Values.resources | nindent 12 }} 61 | {{- end }} -------------------------------------------------------------------------------- /charts/eks-ng-ami-updater/templates/namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ template "eks-ng-ami-updater.name" . }} 5 | 6 | 7 | -------------------------------------------------------------------------------- /charts/eks-ng-ami-updater/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.serviceAccount.create }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | {{- with .Values.serviceAccount.annotations }} 6 | annotations: 7 | {{- toYaml . | nindent 4 }} 8 | {{- end }} 9 | namespace: {{ template "eks-ng-ami-updater.name" . }} 10 | name: {{ template "eks-ng-ami-updater.serviceAccountName" . }} 11 | labels: 12 | app: {{ template "eks-ng-ami-updater.name" . }} 13 | chart: {{ template "eks-ng-ami-updater.chart" . }} 14 | release: {{ .Release.Name }} 15 | {{- end }} -------------------------------------------------------------------------------- /charts/eks-ng-ami-updater/values.yaml: -------------------------------------------------------------------------------- 1 | image: 2 | repository: public.ecr.aws/loom/eks-ng-ami-updater 3 | tag: latest 4 | pullPolicy: IfNotPresent 5 | 6 | nameOverride: "" 7 | 8 | schedule: "30 7 * * 0" 9 | 10 | cmdOptions: 11 | dryrun: true 12 | debug: true 13 | 14 | annotations: {} 15 | 16 | resources: {} 17 | 18 | nodeSelector: {} 19 | 20 | tolerations: [] 21 | 22 | affinity: {} 23 | 24 | serviceAccount: 25 | create: true 26 | name: "" 27 | annotations: {} 28 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/loomhq/eks-ng-ami-updater 2 | 3 | go 1.23 4 | toolchain go1.24.1 5 | 6 | require github.com/rs/zerolog v1.34.0 7 | 8 | require ( 9 | github.com/davecgh/go-spew v1.1.1 // indirect 10 | github.com/jmespath/go-jmespath v0.4.0 // indirect 11 | github.com/kr/pretty v0.3.1 // indirect 12 | github.com/pmezard/go-difflib v1.0.0 // indirect 13 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 14 | gopkg.in/yaml.v2 v2.4.0 // indirect 15 | gopkg.in/yaml.v3 v3.0.1 // indirect 16 | ) 17 | 18 | require ( 19 | github.com/aws/aws-sdk-go v1.55.7 20 | github.com/mattn/go-colorable v0.1.14 // indirect 21 | github.com/mattn/go-isatty v0.0.20 // indirect 22 | github.com/pkg/errors v0.9.1 23 | github.com/stretchr/testify v1.10.0 24 | golang.org/x/sync v0.14.0 25 | golang.org/x/sys v0.33.0 // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= 2 | github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= 3 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 4 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 9 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 10 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 11 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 12 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 13 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 14 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 15 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 16 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 17 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 18 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 19 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 20 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 21 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 22 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 23 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 24 | github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 25 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 26 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 27 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 28 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 29 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 30 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 31 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 32 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 33 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 34 | github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 35 | github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= 36 | github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 39 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 40 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 41 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 42 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 43 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 44 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 45 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 46 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 47 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 48 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 49 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 50 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 51 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 52 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 53 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 54 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 55 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/rs/zerolog/log" 5 | 6 | "github.com/loomhq/eks-ng-ami-updater/pkg/flags" 7 | "github.com/loomhq/eks-ng-ami-updater/pkg/logs" 8 | "github.com/loomhq/eks-ng-ami-updater/pkg/updater" 9 | ) 10 | 11 | func main() { 12 | debugVar, dryrunVar, skipNewerThanDays, regionsVar, nodegroupsVar, tagVar := flags.Setup() 13 | ctx := logs.Setup(debugVar) 14 | 15 | err := updater.UpdateAmi(dryrunVar, skipNewerThanDays, regionsVar, nodegroupsVar, tagVar, ctx) 16 | if err != nil { 17 | log.Fatal().Err(err).Msg("Unable to update ami") 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /pkg/aws/ami.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | awsLib "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/ec2" 11 | "github.com/aws/aws-sdk-go/service/eks" 12 | "github.com/aws/aws-sdk-go/service/ssm" 13 | "github.com/rs/zerolog/log" 14 | ) 15 | 16 | func GetLatestAmiWithinSsm(amiType, amiVersion, region string, awsSsm SSM, awsEc2 Ec2, ctx context.Context) (time.Time, *string, error) { 17 | var ssmPath string 18 | 19 | ssmBootlerocketPathPrefix := "/aws/service/bottlerocket/aws-k8s-" 20 | ssmOptimizedPathPrefix := "/aws/service/eks/optimized-ami/" 21 | ssmPathSuffix := "/image_id" 22 | 23 | switch amiType { 24 | // https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami-bottlerocket.html 25 | case "BOTTLEROCKET_x86_64": 26 | ssmPath = ssmBootlerocketPathPrefix + amiVersion + "/x86_64/latest" + ssmPathSuffix 27 | case "BOTTLEROCKET_x86_64_NVIDIA": 28 | ssmPath = ssmBootlerocketPathPrefix + amiVersion + "-nvidia/x86_64/latest" + ssmPathSuffix 29 | case "BOTTLEROCKET_ARM_64": 30 | ssmPath = ssmBootlerocketPathPrefix + amiVersion + "/arm64/latest" + ssmPathSuffix 31 | case "BOTTLEROCKET_ARM_64_NVIDIA": 32 | ssmPath = ssmBootlerocketPathPrefix + amiVersion + "-nvidia/arm64/latest" + ssmPathSuffix 33 | // https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html 34 | case "AL2_x86_64": 35 | ssmPath = ssmOptimizedPathPrefix + amiVersion + "/amazon-linux-2/recommended" + ssmPathSuffix 36 | case "AL2_x86_64_GPU": 37 | ssmPath = ssmOptimizedPathPrefix + amiVersion + "/amazon-linux-2-gpu/recommended" + ssmPathSuffix 38 | case "AL2_ARM_64": 39 | ssmPath = ssmOptimizedPathPrefix + amiVersion + "/amazon-linux-2-arm64/recommended" + ssmPathSuffix 40 | // https://docs.aws.amazon.com/eks/latest/userguide/retrieve-windows-ami-id.html 41 | case "WINDOWS_CORE_2019_x86_64": 42 | ssmPath = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-" + amiVersion + ssmPathSuffix 43 | case "WINDOWS_FULL_2019_x86_64": 44 | ssmPath = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-" + amiVersion + ssmPathSuffix 45 | case "WINDOWS_CORE_2022_x86_64": 46 | ssmPath = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-" + amiVersion + ssmPathSuffix 47 | case "WINDOWS_FULL_2022_x86_64": 48 | ssmPath = "/aws/service/ami-windows-latest/Windows_Server-2022-English-Full-EKS_Optimized-" + amiVersion + ssmPathSuffix 49 | default: 50 | return time.Time{}, nil, fmt.Errorf("nodegroup's ami type (%s) is not recognize", amiType) 51 | } 52 | 53 | output, err := awsSsm.GetParameter(&ssm.GetParameterInput{ 54 | Name: &ssmPath, 55 | WithDecryption: new(bool), 56 | }) 57 | if err != nil { 58 | return time.Time{}, nil, err 59 | } 60 | 61 | awsLatestAmiImageLocation, err := GetLatestAmiWithinEc2(*output.Parameter.Value, awsEc2, ctx) 62 | if err != nil { 63 | return time.Time{}, nil, err 64 | } 65 | 66 | return *output.Parameter.LastModifiedDate, awsLatestAmiImageLocation, nil 67 | } 68 | 69 | func GetLatestAmiWithinEc2(amiVersion string, awsEc2 Ec2, ctx context.Context) (*string, error) { 70 | input := &ec2.DescribeImagesInput{ 71 | ImageIds: []*string{ 72 | awsLib.String(amiVersion), 73 | }, 74 | Owners: []*string{ 75 | awsLib.String("amazon"), 76 | }, 77 | } 78 | 79 | result, err := awsEc2.DescribeImages(input) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | return result.Images[0].ImageLocation, nil 85 | } 86 | 87 | func IsTheSameAmiVersion(nodegroup NodeGroup, ngAmiType, ngAmiVersion, ngAmiReleaseVersion string, awsSsm SSM, awsEc2 Ec2, ctx context.Context) (bool, error) { 88 | var awsLatestAmiReleaseVersion string 89 | 90 | logWithContext := log.Ctx(ctx).With().Str("function", "IsTheSameAmiVersion").Logger() 91 | 92 | _, awsLatestAmiImageLocation, err := GetLatestAmiWithinSsm(ngAmiType, ngAmiVersion, nodegroup.Region, awsSsm, awsEc2, ctx) 93 | if err != nil { 94 | return false, fmt.Errorf("region: %s, cluster: %s, nodegroup: %s : %w", nodegroup.Region, nodegroup.ClusterName, nodegroup.ClusterName, err) 95 | } 96 | 97 | awsLatestAmiImageLocationSplites := strings.Split(*awsLatestAmiImageLocation, "-") 98 | last := len(awsLatestAmiImageLocationSplites) 99 | switch strings.Split(ngAmiType, "_")[0] { 100 | case "BOTTLEROCKET": 101 | awsLatestAmiReleaseVersion = awsLatestAmiImageLocationSplites[last-2] + "-" + awsLatestAmiImageLocationSplites[last-1] 102 | ngAmiReleaseVersion = "v" + ngAmiReleaseVersion 103 | case "AL2", "WINDOWS": 104 | awsLatestAmiReleaseVersion = awsLatestAmiImageLocationSplites[last-1] 105 | ngAmiReleaseVersion = "v" + strings.Split(ngAmiReleaseVersion, "-")[1] 106 | default: 107 | return true, fmt.Errorf("nodegroup's ami type (%s) is not recognize", ngAmiType) 108 | } 109 | 110 | isTheSameAmiVersion := true 111 | if awsLatestAmiReleaseVersion != ngAmiReleaseVersion { 112 | isTheSameAmiVersion = false 113 | } 114 | 115 | logWithContext.Debug().Str("ngAmiReleaseVersion", ngAmiReleaseVersion).Str("awsLatestAmiReleaseVersion", awsLatestAmiReleaseVersion). 116 | Str("region", nodegroup.Region).Str("nodegroup", nodegroup.NodegroupName).Str("cluster", nodegroup.ClusterName).Msg("nodegroup and aws latest ami versions are compared") 117 | 118 | return isTheSameAmiVersion, nil 119 | } 120 | 121 | func IsLastAmiOldEnough(skipNewerThan uint, nodegroup NodeGroup, today time.Time, ngAmiType, ngAmiVersion string, awsSsm SSM, awsEc2 Ec2, ctx context.Context) (bool, error) { 122 | logWithContext := log.Ctx(ctx).With().Str("function", "IsLastAmiOldEnough").Logger() 123 | 124 | amiLastModifiedDate, _, err := GetLatestAmiWithinSsm(ngAmiType, ngAmiVersion, nodegroup.Region, awsSsm, awsEc2, ctx) 125 | if err != nil { 126 | return false, fmt.Errorf("region: %s, cluster: %s, nodegroup: %s : %w", nodegroup.Region, nodegroup.ClusterName, nodegroup.ClusterName, err) 127 | } 128 | 129 | hoursToSkip := -24 * time.Duration(skipNewerThan) * time.Hour //nolint:gosec // no overflow risk 130 | criticalDay := today.Add(hoursToSkip).UTC() 131 | 132 | logWithContext.Debug().Time("criticalDay", criticalDay).Uint("skipNewerThan", skipNewerThan). 133 | Str("region", nodegroup.Region).Str("nodegroup", nodegroup.NodegroupName).Str("cluster", nodegroup.ClusterName).Msg("ami criticalDay is calculated") 134 | 135 | if amiLastModifiedDate.Before(criticalDay) { 136 | return true, nil 137 | } 138 | 139 | return false, nil 140 | } 141 | 142 | func AmiUpdate(region, cluster, nodegroup string, dryrun bool, ctx context.Context) error { 143 | logWithContext := log.Ctx(ctx).With().Str("function", "AmiUpdate").Logger() 144 | 145 | if dryrun { 146 | log.Info().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Msg("drying is true. exiting") 147 | 148 | return nil 149 | } 150 | 151 | svcEks, err := EksClientSetup(region) 152 | if err != nil { 153 | logWithContext.Debug().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Err(err).Msg("Error") 154 | 155 | return err 156 | } 157 | 158 | log.Info().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Msg("starting ami update") 159 | 160 | _, err = svcEks.UpdateNodegroupVersion(&eks.UpdateNodegroupVersionInput{ 161 | ClusterName: awsLib.String(cluster), 162 | NodegroupName: awsLib.String(nodegroup), 163 | }) 164 | if err != nil { 165 | logWithContext.Debug().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Err(err).Msg("Error") 166 | 167 | return err 168 | } 169 | logWithContext.Debug().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Msg("started properly") 170 | 171 | logWithContext.Debug().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Msg("waiting for finish") 172 | err = svcEks.WaitUntilNodegroupActive(&eks.DescribeNodegroupInput{ 173 | ClusterName: awsLib.String(cluster), 174 | NodegroupName: awsLib.String(nodegroup), 175 | }) 176 | if err != nil { 177 | logWithContext.Debug().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Err(err).Msg("Error") 178 | 179 | return err 180 | } 181 | 182 | log.Info().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Msg("finished ami update properly") 183 | 184 | return nil 185 | } 186 | -------------------------------------------------------------------------------- /pkg/aws/ami_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | awsLib "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/ec2" 11 | "github.com/aws/aws-sdk-go/service/ssm" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func toTimePtr(t time.Time) *time.Time { 16 | return &t 17 | } 18 | 19 | func TestIsLastAmiOldEnough(t *testing.T) { 20 | t.Parallel() 21 | 22 | tests := []struct { 23 | name string 24 | mockedOutputGetParameterSsm ssm.GetParameterOutput 25 | mockedOutputGetParameterEc2 ec2.DescribeImagesOutput 26 | skipNewerThanDays uint 27 | nodegroup NodeGroup 28 | ngAmiType string 29 | ngAmiVersion string 30 | today time.Time 31 | expectedValue bool 32 | expectedError error 33 | }{ 34 | { 35 | name: "latest ami in aws is not newer than skipNewerThanDays (ami will be updated)", 36 | ngAmiType: "BOTTLEROCKET_x86_64", 37 | ngAmiVersion: "1.24", 38 | mockedOutputGetParameterSsm: ssm.GetParameterOutput{ 39 | Parameter: &ssm.Parameter{ 40 | LastModifiedDate: toTimePtr(time.Date(2023, time.January, 30, 10, 0, 0, 0, time.UTC)), 41 | Value: awsLib.String("ami-08a3df9f52daf9b5f"), 42 | }, 43 | }, 44 | mockedOutputGetParameterEc2: ec2.DescribeImagesOutput{ 45 | Images: []*ec2.Image{ 46 | { 47 | ImageLocation: awsLib.String("amazon/bottlerocket-aws-k8s-1.24-x86_64-v1.14.0-9cd59298"), 48 | }, 49 | }, 50 | }, 51 | skipNewerThanDays: 10, 52 | nodegroup: NodeGroup{}, 53 | today: time.Date(2023, time.April, 31, 10, 0, 0, 0, time.UTC), 54 | expectedValue: true, 55 | expectedError: nil, 56 | }, 57 | { 58 | name: "latest ami in aws is newer than skipNewerThanDays (ami will not be updated)", 59 | ngAmiType: "BOTTLEROCKET_x86_64", 60 | ngAmiVersion: "1.24", 61 | mockedOutputGetParameterSsm: ssm.GetParameterOutput{ 62 | Parameter: &ssm.Parameter{ 63 | LastModifiedDate: toTimePtr(time.Date(2023, time.January, 30, 10, 0, 0, 0, time.UTC)), 64 | Value: awsLib.String("ami-08a3df9f52daf9b5f"), 65 | }, 66 | }, 67 | mockedOutputGetParameterEc2: ec2.DescribeImagesOutput{ 68 | Images: []*ec2.Image{ 69 | { 70 | ImageLocation: awsLib.String("amazon/bottlerocket-aws-k8s-1.24-x86_64-v1.14.0-9cd59298"), 71 | }, 72 | }, 73 | }, 74 | skipNewerThanDays: 10, 75 | nodegroup: NodeGroup{}, 76 | today: time.Date(2023, time.February, 1, 10, 0, 0, 0, time.UTC), 77 | expectedValue: false, 78 | expectedError: nil, 79 | }, 80 | { 81 | name: "one day and 1 minute diff", 82 | ngAmiType: "BOTTLEROCKET_x86_64", 83 | ngAmiVersion: "1.24", 84 | mockedOutputGetParameterSsm: ssm.GetParameterOutput{ 85 | Parameter: &ssm.Parameter{ 86 | LastModifiedDate: toTimePtr(time.Date(2023, time.January, 15, 9, 59, 0, 0, time.UTC)), 87 | Value: awsLib.String("ami-08a3df9f52daf9b5f"), 88 | }, 89 | }, 90 | mockedOutputGetParameterEc2: ec2.DescribeImagesOutput{ 91 | Images: []*ec2.Image{ 92 | { 93 | ImageLocation: awsLib.String("amazon/bottlerocket-aws-k8s-1.24-x86_64-v1.14.0-9cd59298"), 94 | }, 95 | }, 96 | }, 97 | skipNewerThanDays: 1, 98 | nodegroup: NodeGroup{}, 99 | today: time.Date(2023, time.January, 16, 10, 0, 0, 0, time.UTC), 100 | expectedValue: true, 101 | expectedError: nil, 102 | }, 103 | { 104 | name: "unrecognize ami type", 105 | ngAmiType: "SOMETHINGNEW_x86_64", 106 | ngAmiVersion: "1.24", 107 | mockedOutputGetParameterSsm: ssm.GetParameterOutput{ 108 | Parameter: &ssm.Parameter{ 109 | LastModifiedDate: toTimePtr(time.Date(2023, time.January, 30, 10, 0, 0, 0, time.UTC)), 110 | Value: awsLib.String("ami-08a3df9f52daf9b5f"), 111 | }, 112 | }, 113 | mockedOutputGetParameterEc2: ec2.DescribeImagesOutput{ 114 | Images: []*ec2.Image{ 115 | { 116 | ImageLocation: awsLib.String("amazon/bottlerocket-aws-k8s-1.24-x86_64-v1.14.0-9cd59298"), 117 | }, 118 | }, 119 | }, 120 | skipNewerThanDays: 10, 121 | nodegroup: NodeGroup{}, 122 | today: time.Date(2023, time.February, 1, 10, 0, 0, 0, time.UTC), 123 | expectedValue: false, 124 | expectedError: fmt.Errorf("region: , cluster: , nodegroup: : %w", fmt.Errorf("nodegroup's ami type (SOMETHINGNEW_x86_64) is not recognize")), 125 | }, 126 | { 127 | name: "AL2 ami type", 128 | ngAmiType: "AL2_x86_64", 129 | ngAmiVersion: "1.28.", 130 | mockedOutputGetParameterSsm: ssm.GetParameterOutput{ 131 | Parameter: &ssm.Parameter{ 132 | LastModifiedDate: toTimePtr(time.Date(2023, time.January, 30, 10, 0, 0, 0, time.UTC)), 133 | Value: awsLib.String("ami-08a3df9f52daf9b5f"), 134 | }, 135 | }, 136 | mockedOutputGetParameterEc2: ec2.DescribeImagesOutput{ 137 | Images: []*ec2.Image{ 138 | { 139 | ImageLocation: awsLib.String("amazon/amazon-eks-node-1.28-v20240202"), 140 | }, 141 | }, 142 | }, 143 | skipNewerThanDays: 10, 144 | nodegroup: NodeGroup{}, 145 | today: time.Date(2023, time.February, 1, 10, 0, 0, 0, time.UTC), 146 | expectedValue: false, 147 | expectedError: nil, 148 | }, 149 | } 150 | 151 | for _, test := range tests { 152 | fmt.Printf("test: %s\n", test.name) 153 | awsSsm := testSsm{ 154 | OutputGetParameter: &test.mockedOutputGetParameterSsm, 155 | } 156 | awsEc2 := testEc2{ 157 | OutputImages: &test.mockedOutputGetParameterEc2, 158 | } 159 | 160 | output, err := IsLastAmiOldEnough(test.skipNewerThanDays, test.nodegroup, test.today, test.ngAmiType, test.ngAmiVersion, awsSsm, awsEc2, context.Background()) 161 | 162 | assert.Equal(t, test.expectedValue, output) 163 | assert.Equal(t, test.expectedError, err) 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /pkg/aws/aws.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | 6 | awsLib "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/session" 8 | "github.com/aws/aws-sdk-go/service/ec2" 9 | "github.com/aws/aws-sdk-go/service/eks" 10 | "github.com/aws/aws-sdk-go/service/ssm" 11 | ) 12 | 13 | func EksClientSetup(awsRegion string) (*eks.EKS, error) { 14 | session, err := session.NewSession(&awsLib.Config{ 15 | Region: awsLib.String(awsRegion)}, 16 | ) 17 | if err != nil { 18 | return nil, fmt.Errorf("error creating new Eks session in %s region: %w", awsRegion, err) 19 | } 20 | 21 | return eks.New(session), nil 22 | } 23 | 24 | func SsmClientSetup(awsRegion string) (*ssm.SSM, error) { 25 | session, err := session.NewSession(&awsLib.Config{ 26 | Region: awsLib.String(awsRegion)}, 27 | ) 28 | if err != nil { 29 | return nil, fmt.Errorf("error creating new Ssm session in %s region: %w", awsRegion, err) 30 | } 31 | 32 | return ssm.New(session), nil 33 | } 34 | 35 | func Ec2ClientSetup() (*ec2.EC2, error) { 36 | session, err := session.NewSession(&awsLib.Config{}) 37 | if err != nil { 38 | return nil, fmt.Errorf("error creating new Ec2 session: %w", err) 39 | } 40 | 41 | return ec2.New(session), nil 42 | } 43 | -------------------------------------------------------------------------------- /pkg/aws/clusters.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go/service/eks" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | func GetClusters(region string, awsEks EKS, ctx context.Context) ([]string, error) { 11 | var clusters []string 12 | logWithContext := log.Ctx(ctx).With().Str("function", "GetClusters").Logger() 13 | 14 | logWithContext.Debug().Str("region", region).Msg("looking for clusters") 15 | 16 | input := &eks.ListClustersInput{} 17 | 18 | for { 19 | output, err := awsEks.ListClusters(input) 20 | if err != nil { 21 | return nil, err 22 | } 23 | for _, v := range output.Clusters { 24 | clusters = append(clusters, *v) 25 | } 26 | if output.NextToken != nil { 27 | logWithContext.Debug().Str("region", region).Strs("clusters", clusters).Str("token", *output.NextToken).Msg("ListClusters request exceed maxResults") 28 | input = &eks.ListClustersInput{ 29 | NextToken: output.NextToken, 30 | } 31 | } else { 32 | break 33 | } 34 | } 35 | 36 | logWithContext.Debug().Str("region", region).Strs("clusters", clusters).Msg("clusters have been selected") 37 | 38 | return clusters, nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/aws/clusters_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | awsLib "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/eks" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestGetClusters(t *testing.T) { 14 | t.Parallel() 15 | 16 | tests := []struct { 17 | name string 18 | mockedOutput eks.ListClustersOutput 19 | region string 20 | expectedValue []string 21 | expectedError error 22 | }{ 23 | {name: "full listing at once (without nextToken)", 24 | mockedOutput: eks.ListClustersOutput{ 25 | Clusters: []*string{ 26 | awsLib.String("cluster-1"), 27 | awsLib.String("cluster-2"), 28 | }, 29 | NextToken: nil, 30 | }, 31 | region: "us-west-1", 32 | expectedValue: []string{"cluster-1", "cluster-2"}, 33 | expectedError: nil, 34 | }, 35 | {name: "exceed maxResults (nextToken is set)", 36 | mockedOutput: eks.ListClustersOutput{ 37 | Clusters: []*string{ 38 | awsLib.String("cluster-1"), 39 | awsLib.String("cluster-2"), 40 | }, 41 | NextToken: awsLib.String("111"), 42 | }, 43 | region: "us-west-1", 44 | expectedValue: []string{"cluster-1", "cluster-2", "cluster-1"}, 45 | expectedError: nil, 46 | }, 47 | } 48 | 49 | for _, test := range tests { 50 | fmt.Printf("test: %s\n", test.name) 51 | awsEks := testEks{OutputListClusters: &test.mockedOutput} 52 | 53 | output, err := GetClusters(test.region, awsEks, context.Background()) 54 | 55 | assert.Equal(t, test.expectedValue, output) 56 | assert.Equal(t, test.expectedError, err) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pkg/aws/ec2.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/service/ec2" 7 | ) 8 | 9 | type Ec2 interface { 10 | DescribeRegions(input *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) 11 | DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) 12 | } 13 | 14 | type RealEc2 struct { 15 | Svc *ec2.EC2 16 | } 17 | 18 | func (t RealEc2) DescribeRegions(input *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) { 19 | result, err := t.Svc.DescribeRegions(input) 20 | if err != nil { 21 | return nil, fmt.Errorf("error describing regions: %w", err) 22 | } 23 | 24 | return result, nil 25 | } 26 | 27 | func (t RealEc2) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { 28 | result, err := t.Svc.DescribeImages(input) 29 | if err != nil { 30 | return nil, fmt.Errorf("error describing images: %w", err) 31 | } 32 | 33 | return result, nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/aws/eks.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/service/eks" 7 | ) 8 | 9 | type EKS interface { 10 | ListClusters(input *eks.ListClustersInput) (*eks.ListClustersOutput, error) 11 | ListNodegroups(input *eks.ListNodegroupsInput) (*eks.ListNodegroupsOutput, error) 12 | DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) 13 | } 14 | 15 | type RealEks struct { 16 | Svc *eks.EKS 17 | } 18 | 19 | func (t RealEks) ListClusters(input *eks.ListClustersInput) (*eks.ListClustersOutput, error) { 20 | result, err := t.Svc.ListClusters(input) 21 | if err != nil { 22 | return nil, fmt.Errorf("error listing clusters: %w", err) 23 | } 24 | 25 | return result, nil 26 | } 27 | 28 | func (t RealEks) ListNodegroups(input *eks.ListNodegroupsInput) (*eks.ListNodegroupsOutput, error) { 29 | result, err := t.Svc.ListNodegroups(input) 30 | if err != nil { 31 | return nil, fmt.Errorf("error listing nodegroups: %w", err) 32 | } 33 | 34 | return result, nil 35 | } 36 | 37 | func (t RealEks) DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) { 38 | result, err := t.Svc.DescribeNodegroup(input) 39 | if err != nil { 40 | return nil, fmt.Errorf("error describing nodegroup: %w", err) 41 | } 42 | 43 | return result, nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/aws/nodegroups.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go/service/eks" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | type NodeGroup struct { 11 | Region string 12 | ClusterName string 13 | NodegroupName string 14 | } 15 | 16 | func GetNodegroupsFromCluster(clusterName, region string, awsEks EKS, ctx context.Context) ([]string, error) { 17 | var nodegroups []string 18 | logWithContext := log.Ctx(ctx).With().Str("function", "GetNodegroupsFromCluster").Logger() 19 | 20 | logWithContext.Debug().Str("region", region).Str("cluster", clusterName).Msg("looking for nodegroups") 21 | input := &eks.ListNodegroupsInput{ 22 | ClusterName: &clusterName, 23 | } 24 | 25 | for { 26 | output, err := awsEks.ListNodegroups(input) 27 | if err != nil { 28 | return nil, err 29 | } 30 | for _, v := range output.Nodegroups { 31 | nodegroups = append(nodegroups, *v) 32 | } 33 | 34 | if output.NextToken != nil { 35 | logWithContext.Debug().Str("region", region).Str("cluster", clusterName).Str("token", *output.NextToken).Msg("ListNodegroups request exceed maxResults") 36 | input = &eks.ListNodegroupsInput{ 37 | NextToken: output.NextToken, 38 | } 39 | } else { 40 | break 41 | } 42 | } 43 | 44 | logWithContext.Debug().Str("region", region).Str("cluster", clusterName).Strs("nodegroup", nodegroups).Msg("nodegroups have been selected") 45 | 46 | return nodegroups, nil 47 | } 48 | 49 | func GetNodegroupsFromRegion(region string, ctx context.Context) ([]NodeGroup, error) { 50 | var nodeGroupFromCluster []NodeGroup 51 | logWithContext := log.Ctx(ctx).With().Str("function", "GetNodegroupsFromRegion").Logger() 52 | 53 | svcEks, err := EksClientSetup(region) 54 | if err != nil { 55 | return nil, err 56 | } 57 | awsEks := RealEks{Svc: svcEks} 58 | 59 | clusters, err := GetClusters(region, awsEks, ctx) 60 | if err != nil { 61 | return nil, err 62 | } 63 | 64 | for _, cluster := range clusters { 65 | nodegroups, err := GetNodegroupsFromCluster(cluster, region, awsEks, ctx) 66 | if err != nil { 67 | return nil, err 68 | } 69 | for _, nodegroup := range nodegroups { 70 | nodeGroupFromCluster = append(nodeGroupFromCluster, NodeGroup{Region: region, ClusterName: cluster, NodegroupName: nodegroup}) 71 | logWithContext.Debug().Str("region", region).Str("cluster", cluster).Str("nodegroup", nodegroup).Msg("add nodegroup to the ami upgrade nodegroups checking list") 72 | } 73 | } 74 | 75 | return nodeGroupFromCluster, nil 76 | } 77 | 78 | func GetNodegroupDescription(nodegroup NodeGroup, awsEks EKS, ctx context.Context) (eks.DescribeNodegroupOutput, error) { 79 | logWithContext := log.Ctx(ctx).With().Str("function", "GetNodegroupDescription").Logger() 80 | 81 | output, err := awsEks.DescribeNodegroup(&eks.DescribeNodegroupInput{ 82 | ClusterName: &nodegroup.ClusterName, 83 | NodegroupName: &nodegroup.NodegroupName, 84 | }) 85 | if err != nil { 86 | return eks.DescribeNodegroupOutput{}, err 87 | } 88 | 89 | logWithContext.Debug().Str("AmiType", *output.Nodegroup.AmiType).Str("AmiVersion", *output.Nodegroup.Version). 90 | Str("region", nodegroup.Region).Str("cluster", nodegroup.ClusterName).Str("nodegroup", nodegroup.NodegroupName).Msg("actual nodegroup AMI Id") 91 | 92 | for k, v := range output.Nodegroup.Tags { 93 | logWithContext.Debug().Str("tagKey", k).Str("tagValue", *v).Msgf("Tags from %s nodegroup", *output.Nodegroup.NodegroupName) 94 | } 95 | 96 | return *output, nil 97 | } 98 | 99 | func HasNodegroupTag(tag string, nodegroupTags map[string]*string, ctx context.Context) (bool, error) { 100 | logWithContext := log.Ctx(ctx).With().Str("function", "HasNodegroupTag").Logger() 101 | 102 | for k, v := range nodegroupTags { 103 | tagTemp := k + ":" + *v 104 | if tag == tagTemp { 105 | logWithContext.Debug().Str("tagVar", tag).Str("nodegroupTag", tagTemp).Msg("Flag tag is found for nodegroup") 106 | 107 | return true, nil 108 | } 109 | logWithContext.Debug().Str("tagVar", tag).Str("nodegroupTag", tagTemp).Msg("Flag tag is not the same as checking nodegroup tag") 110 | } 111 | 112 | return false, nil 113 | } 114 | -------------------------------------------------------------------------------- /pkg/aws/nodegroups_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | awsLib "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/eks" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestGetNodegroupsFromCluster(t *testing.T) { 14 | t.Parallel() 15 | 16 | tests := []struct { 17 | name string 18 | mockedOutput eks.ListNodegroupsOutput 19 | clusterName string 20 | region string 21 | expectedValue []string 22 | expectedError error 23 | }{ 24 | {name: "full listing at once (without nextToken)", 25 | mockedOutput: eks.ListNodegroupsOutput{ 26 | Nodegroups: []*string{ 27 | awsLib.String("nodegroup-1"), 28 | awsLib.String("nodegroup-2"), 29 | }, 30 | }, 31 | clusterName: "cluster-1", 32 | region: "us-west-1", 33 | expectedValue: []string{"nodegroup-1", "nodegroup-2"}, 34 | expectedError: nil, 35 | }, 36 | {name: "exceed maxResults (nextToken is set)", 37 | mockedOutput: eks.ListNodegroupsOutput{ 38 | NextToken: awsLib.String("111"), 39 | Nodegroups: []*string{ 40 | awsLib.String("nodegroup-1"), 41 | awsLib.String("nodegroup-2"), 42 | }, 43 | }, 44 | clusterName: "cluster-1", 45 | region: "us-west-1", 46 | expectedValue: []string{"nodegroup-1", "nodegroup-2", "nodegroup-1"}, 47 | expectedError: nil, 48 | }, 49 | } 50 | 51 | for _, test := range tests { 52 | fmt.Printf("test: %s\n", test.name) 53 | awsEks := testEks{OutputListNodegroups: &test.mockedOutput} 54 | 55 | output, err := GetNodegroupsFromCluster(test.clusterName, test.region, awsEks, context.Background()) 56 | 57 | assert.Equal(t, test.expectedValue, output) 58 | assert.Equal(t, test.expectedError, err) 59 | } 60 | } 61 | 62 | func TestGetNodegroupTags(t *testing.T) { 63 | t.Parallel() 64 | 65 | tests := []struct { 66 | name string 67 | tag string 68 | nodegroupTags map[string]*string 69 | expectedValue bool 70 | expectedError error 71 | }{ 72 | {name: "tag is found in nodegroup Tags", 73 | tag: "env:staging", 74 | nodegroupTags: map[string]*string{ 75 | "tag1": awsLib.String("value1"), 76 | "env": awsLib.String("staging"), 77 | "tag3": awsLib.String("value3"), 78 | }, 79 | expectedValue: true, 80 | expectedError: nil, 81 | }, 82 | {name: "tag is not found in nodegroup Tags", 83 | tag: "env:production", 84 | nodegroupTags: map[string]*string{ 85 | "tag1": awsLib.String("value1"), 86 | "env": awsLib.String("staging"), 87 | "tag3": awsLib.String("value3"), 88 | }, 89 | expectedValue: false, 90 | expectedError: nil, 91 | }, 92 | {name: "nodegroup tags list is empty", 93 | tag: "env:staging", 94 | nodegroupTags: map[string]*string{}, 95 | expectedValue: false, 96 | expectedError: nil, 97 | }, 98 | {name: "only key is provided for tag #1", 99 | tag: "env", 100 | nodegroupTags: map[string]*string{ 101 | "tag1": awsLib.String("value1"), 102 | "env": awsLib.String("staging"), 103 | "tag3": awsLib.String("value3"), 104 | }, 105 | expectedValue: false, 106 | expectedError: nil, 107 | }, 108 | {name: "only key is provided for tag #2", 109 | tag: "env:", 110 | nodegroupTags: map[string]*string{ 111 | "tag1": awsLib.String("value1"), 112 | "env": awsLib.String(""), 113 | "tag3": awsLib.String("value3"), 114 | }, 115 | expectedValue: true, 116 | expectedError: nil, 117 | }, 118 | {name: "only value is provided for tag #1", 119 | tag: ":staging", 120 | nodegroupTags: map[string]*string{ 121 | "tag1": awsLib.String("value1"), 122 | "env": awsLib.String("staging"), 123 | "tag3": awsLib.String("value3"), 124 | }, 125 | expectedValue: false, 126 | expectedError: nil, 127 | }, 128 | {name: "only value is provided for tag #2", 129 | tag: ":staging", 130 | nodegroupTags: map[string]*string{ 131 | "tag1": awsLib.String("value1"), 132 | "": awsLib.String("staging"), 133 | "tag3": awsLib.String("value3"), 134 | }, 135 | expectedValue: true, 136 | expectedError: nil, 137 | }, 138 | } 139 | 140 | for _, test := range tests { 141 | fmt.Printf("test: %s\n", test.name) 142 | 143 | output, err := HasNodegroupTag(test.tag, test.nodegroupTags, context.Background()) 144 | 145 | assert.Equal(t, test.expectedValue, output) 146 | assert.Equal(t, test.expectedError, err) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /pkg/aws/regions.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/ec2" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func GetRegionsToCheck(regionsVar []string, awsEc2 Ec2, ctx context.Context) ([]string, error) { 12 | var regions []string 13 | logWithContext := log.Ctx(ctx).With().Str("function", "GetRegionsToCheck").Logger() 14 | 15 | if len(regionsVar) == 0 { 16 | logWithContext.Debug().Msg("looking for regions") 17 | 18 | input := &ec2.DescribeRegionsInput{AllRegions: aws.Bool(true)} 19 | result, err := awsEc2.DescribeRegions(input) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | for _, v := range result.Regions { 25 | regions = append(regions, *v.RegionName) 26 | } 27 | 28 | logWithContext.Debug().Strs("regions", regions).Msg("regions have been selected") 29 | } else { 30 | regions = regionsVar 31 | } 32 | 33 | return regions, nil 34 | } 35 | -------------------------------------------------------------------------------- /pkg/aws/regions_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | awsLib "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/service/ec2" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestGetRegionsToCheck(t *testing.T) { 14 | t.Parallel() 15 | 16 | tests := []struct { 17 | name string 18 | mockedOutput ec2.DescribeRegionsOutput 19 | regionsVar []string 20 | expectedValue []string 21 | expectedError error 22 | }{ 23 | {name: "regionsVar cli parameter is set #1", 24 | mockedOutput: ec2.DescribeRegionsOutput{ 25 | Regions: []*ec2.Region{ 26 | {RegionName: awsLib.String("us-west-1")}, 27 | {RegionName: awsLib.String("us-west-2")}, 28 | {RegionName: awsLib.String("us-west-3")}, 29 | }, 30 | }, 31 | regionsVar: []string{"eu-west-1", "eu-west-2", "eu-west-3"}, 32 | expectedValue: []string{"eu-west-1", "eu-west-2", "eu-west-3"}, 33 | expectedError: nil, 34 | }, 35 | {name: "regionsVar cli parameter is set #2", 36 | mockedOutput: ec2.DescribeRegionsOutput{}, 37 | regionsVar: []string{"eu-west-1"}, 38 | expectedValue: []string{"eu-west-1"}, 39 | expectedError: nil, 40 | }, 41 | {name: "regionsVar cli parameter is not set #1", 42 | mockedOutput: ec2.DescribeRegionsOutput{ 43 | Regions: []*ec2.Region{ 44 | {RegionName: awsLib.String("us-west-1")}, 45 | }, 46 | }, 47 | regionsVar: []string{}, 48 | expectedValue: []string{"us-west-1"}, 49 | expectedError: nil, 50 | }, 51 | {name: "regionsVar cli parameter is not set #2", 52 | mockedOutput: ec2.DescribeRegionsOutput{ 53 | Regions: []*ec2.Region{ 54 | {RegionName: awsLib.String("us-west-1")}, 55 | {RegionName: awsLib.String("us-west-2")}, 56 | {RegionName: awsLib.String("us-west-3")}, 57 | }, 58 | }, 59 | regionsVar: []string{}, 60 | expectedValue: []string{"us-west-1", "us-west-2", "us-west-3"}, 61 | expectedError: nil, 62 | }, 63 | } 64 | 65 | for _, test := range tests { 66 | fmt.Printf("test: %s\n", test.name) 67 | awsEc2 := testEc2{OutputRegions: &test.mockedOutput} 68 | regionsVar := test.regionsVar 69 | 70 | output, err := GetRegionsToCheck(regionsVar, awsEc2, context.Background()) 71 | 72 | assert.Equal(t, test.expectedValue, output) 73 | assert.Equal(t, test.expectedError, err) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/aws/ssm.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/aws/aws-sdk-go/service/ssm" 7 | ) 8 | 9 | type SSM interface { 10 | GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) 11 | } 12 | 13 | type RealSsm struct { 14 | Svc *ssm.SSM 15 | } 16 | 17 | func (t RealSsm) GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { 18 | result, err := t.Svc.GetParameter(input) 19 | if err != nil { 20 | return nil, fmt.Errorf("error getting parameters: %w", err) 21 | } 22 | 23 | return result, nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/aws/tests.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | awsLib "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/service/ec2" 6 | "github.com/aws/aws-sdk-go/service/eks" 7 | "github.com/aws/aws-sdk-go/service/ssm" 8 | ) 9 | 10 | type testEks struct { 11 | OutputListClusters *eks.ListClustersOutput 12 | OutputListNodegroups *eks.ListNodegroupsOutput 13 | OutputDescribeNodegroup *eks.DescribeNodegroupOutput 14 | } 15 | 16 | type testEc2 struct { 17 | OutputRegions *ec2.DescribeRegionsOutput 18 | OutputImages *ec2.DescribeImagesOutput 19 | } 20 | 21 | type testSsm struct { 22 | OutputGetParameter *ssm.GetParameterOutput 23 | } 24 | 25 | func (t testEks) ListClusters(input *eks.ListClustersInput) (*eks.ListClustersOutput, error) { 26 | output := t.OutputListClusters 27 | 28 | if t.OutputListClusters.NextToken != nil { 29 | if *t.OutputListClusters.NextToken == "111" { 30 | t.OutputListClusters.NextToken = awsLib.String("222") 31 | } else if *t.OutputListClusters.NextToken == "222" { 32 | t.OutputListClusters.NextToken = nil 33 | output.Clusters = []*string{t.OutputListClusters.Clusters[0]} 34 | } 35 | } 36 | 37 | return output, nil 38 | } 39 | 40 | func (t testEks) ListNodegroups(input *eks.ListNodegroupsInput) (*eks.ListNodegroupsOutput, error) { 41 | output := t.OutputListNodegroups 42 | 43 | if t.OutputListNodegroups.NextToken != nil { 44 | if *t.OutputListNodegroups.NextToken == "111" { 45 | t.OutputListNodegroups.NextToken = awsLib.String("222") 46 | } else if *t.OutputListNodegroups.NextToken == "222" { 47 | t.OutputListNodegroups.NextToken = nil 48 | output.Nodegroups = []*string{t.OutputListNodegroups.Nodegroups[0]} 49 | } 50 | } 51 | 52 | return output, nil 53 | } 54 | 55 | func (t testEks) DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) { 56 | output := t.OutputDescribeNodegroup 57 | 58 | return output, nil 59 | } 60 | 61 | func (t testEc2) DescribeRegions(input *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) { 62 | var output = t.OutputRegions 63 | 64 | return output, nil 65 | } 66 | func (t testEc2) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { 67 | var output = t.OutputImages 68 | 69 | return output, nil 70 | } 71 | 72 | func (t testSsm) GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { 73 | var output = t.OutputGetParameter 74 | 75 | return output, nil 76 | } 77 | -------------------------------------------------------------------------------- /pkg/flags/flags.go: -------------------------------------------------------------------------------- 1 | package flags 2 | 3 | import ( 4 | "flag" 5 | "strings" 6 | ) 7 | 8 | func Setup() (bool, bool, uint, []string, []string, string) { 9 | var debug bool 10 | var dryrun bool 11 | var skipNewerThanDays uint 12 | var tag string 13 | var regions []string 14 | var nodegroups []string 15 | 16 | flag.BoolVar(&debug, "debug", false, "set log level to debug (eg. '--debug=true')") 17 | flag.BoolVar(&dryrun, "dryrun", false, "set dryrun mode (eg. '--dryrun=true')") 18 | flag.UintVar(&skipNewerThanDays, "skip-newer-than-days", 0, "skip ami update if the latest available ami was published in less than provided number of days (eg. '--skip-newer-than-days=7')") 19 | flag.StringVar(&tag, "tag", "", "update amis only for nodegroups within this tag (eg. '--tag=env:production')") 20 | flag.Func("nodegroups", "update amis for (only specified here) nodegroups (eg. '--nodegroups=eu-west-1:cluster-1:ngMain,eu-west-2:clusterStage:nodegroupStage1')", func(s string) error { 21 | nodegroups = strings.Split(s, ",") 22 | 23 | return nil 24 | }) 25 | flag.Func("regions", "update amis for all nodegroups from those regions only (eg. '--regions=eu-west-1,us-west-1')", func(s string) error { 26 | regions = strings.Split(s, ",") 27 | 28 | return nil 29 | }) 30 | flag.Parse() 31 | 32 | return debug, dryrun, skipNewerThanDays, regions, nodegroups, tag 33 | } 34 | -------------------------------------------------------------------------------- /pkg/logs/log.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/rs/zerolog" 8 | ) 9 | 10 | func Setup(debug bool) context.Context { 11 | if debug { 12 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 13 | } else { 14 | zerolog.SetGlobalLevel(zerolog.InfoLevel) 15 | } 16 | 17 | return zerolog.New(os.Stdout).WithContext(context.Background()) 18 | } 19 | -------------------------------------------------------------------------------- /pkg/updater/updater.go: -------------------------------------------------------------------------------- 1 | package updater 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | 8 | "github.com/loomhq/eks-ng-ami-updater/pkg/aws" 9 | "github.com/loomhq/eks-ng-ami-updater/pkg/utils" 10 | "github.com/pkg/errors" 11 | "github.com/rs/zerolog/log" 12 | "golang.org/x/sync/errgroup" 13 | ) 14 | 15 | func GetNodeGroupsToUpdateAmi(skipNewerThanDays uint, regionsVar, nodegroupsVar []string, tagVar string, ctx context.Context) ([]aws.NodeGroup, error) { 16 | var nodegroupsToUpdateAmi []aws.NodeGroup 17 | var nodegroupsFromRegion []aws.NodeGroup 18 | var nodegroupsReadyForAmiUpdate []aws.NodeGroup 19 | var regions []string 20 | var isOldEnough bool 21 | var nodegroupHasTag bool 22 | var regionIsAllowed bool 23 | 24 | logWithContext := log.Ctx(ctx).With().Str("function", "GetNodeGroupsToUpdateAmi").Logger() 25 | 26 | if len(nodegroupsVar) > 0 { 27 | for _, s := range nodegroupsVar { 28 | nodegroupSplit := strings.Split(s, ":") 29 | regionIsAllowed = true 30 | if len(regionsVar) > 0 { 31 | regionIsAllowed = utils.Contains(regionsVar, nodegroupSplit[0]) 32 | } 33 | if regionIsAllowed { 34 | nodegroupsToUpdateAmi = append(nodegroupsToUpdateAmi, aws.NodeGroup{Region: nodegroupSplit[0], ClusterName: nodegroupSplit[1], NodegroupName: nodegroupSplit[2]}) 35 | logWithContext.Debug().Str("region", nodegroupSplit[0]).Str("cluster", nodegroupSplit[2]).Str("nodegroup", nodegroupSplit[1]).Strs("nodegrpoupsVar", nodegroupsVar).Strs("regionsVar", regionsVar).Msg("add nodegroup to the ami upgrade nodegroups checking list") 36 | } else { 37 | logWithContext.Debug().Str("region", nodegroupSplit[0]).Str("cluster", nodegroupSplit[2]).Str("nodegroup", nodegroupSplit[1]).Strs("nodegrpoupsVar", nodegroupsVar).Strs("regionsVar", regionsVar).Msg("nodegroup is not in the regions defined by 'regions' flag") 38 | } 39 | } 40 | } else { 41 | svcEc2, err := aws.Ec2ClientSetup() 42 | if err != nil { 43 | return nil, err 44 | } 45 | awsEc2 := aws.RealEc2{Svc: svcEc2} 46 | 47 | regions, err = aws.GetRegionsToCheck(regionsVar, awsEc2, ctx) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | for _, region := range regions { 53 | nodegroupsFromRegion, err = aws.GetNodegroupsFromRegion(region, ctx) 54 | if err != nil { 55 | return nil, err 56 | } 57 | nodegroupsToUpdateAmi = append(nodegroupsToUpdateAmi, nodegroupsFromRegion...) 58 | for _, nodegroup := range nodegroupsFromRegion { 59 | logWithContext.Debug().Str("region", nodegroup.Region).Str("cluster", nodegroup.ClusterName).Str("nodegroup", nodegroup.NodegroupName).Strs("regionsVar", regionsVar).Msg("add nodegroup to the ami upgrade nodegroups checking list") 60 | } 61 | } 62 | } 63 | 64 | for _, nodegroup := range nodegroupsToUpdateAmi { 65 | svcEks, err := aws.EksClientSetup(nodegroup.Region) 66 | if err != nil { 67 | return nil, err 68 | } 69 | awsEks := aws.RealEks{Svc: svcEks} 70 | 71 | svcSsm, err := aws.SsmClientSetup(nodegroup.Region) 72 | if err != nil { 73 | return nil, err 74 | } 75 | awsSsm := aws.RealSsm{Svc: (svcSsm)} 76 | 77 | svcEc2, err := aws.Ec2ClientSetup() 78 | if err != nil { 79 | return nil, err 80 | } 81 | awsEc2 := aws.RealEc2{Svc: svcEc2} 82 | 83 | nodegroupDescription, err := aws.GetNodegroupDescription(nodegroup, awsEks, ctx) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | nodegroupHasTag = true 89 | if tagVar != "" { 90 | nodegroupHasTag, err = aws.HasNodegroupTag(tagVar, nodegroupDescription.Nodegroup.Tags, ctx) 91 | if err != nil { 92 | return nil, err 93 | } 94 | if !nodegroupHasTag { 95 | logWithContext.Debug().Str("region", nodegroup.Region).Str("cluster", nodegroup.ClusterName).Str("nodegroup", nodegroup.NodegroupName).Msgf("skip ami update for this nodegroup ('%s' tag is not exist)", tagVar) 96 | } 97 | } 98 | 99 | isTheSameAmiVersion, err := aws.IsTheSameAmiVersion(nodegroup, *nodegroupDescription.Nodegroup.AmiType, *nodegroupDescription.Nodegroup.Version, *nodegroupDescription.Nodegroup.ReleaseVersion, awsSsm, awsEc2, ctx) 100 | if err != nil { 101 | return nil, err 102 | } 103 | if isTheSameAmiVersion { 104 | logWithContext.Debug().Str("region", nodegroup.Region).Str("cluster", nodegroup.ClusterName).Str("nodegroup", nodegroup.NodegroupName).Msg("skip ami update for this nodegroup (the newest ami is already in use)") 105 | } 106 | 107 | isOldEnough = true 108 | if skipNewerThanDays > 0 && nodegroupHasTag { 109 | today := time.Now() 110 | isOldEnough, err = aws.IsLastAmiOldEnough(skipNewerThanDays, nodegroup, today, *nodegroupDescription.Nodegroup.AmiType, *nodegroupDescription.Nodegroup.Version, awsSsm, awsEc2, ctx) 111 | if err != nil { 112 | return nil, err 113 | } 114 | if !isOldEnough { 115 | logWithContext.Debug().Str("region", nodegroup.Region).Str("cluster", nodegroup.ClusterName).Str("nodegroup", nodegroup.NodegroupName).Msg("skip ami update for this nodegroup (latest available ami for this nodegroup is too new)") 116 | } 117 | } 118 | 119 | if isOldEnough && nodegroupHasTag && !isTheSameAmiVersion { 120 | nodegroupsReadyForAmiUpdate = append(nodegroupsReadyForAmiUpdate, nodegroup) 121 | logWithContext.Info().Str("region", nodegroup.Region).Str("cluster", nodegroup.ClusterName).Str("nodegroup", nodegroup.NodegroupName).Msg("nodegroup is ready for update") 122 | } 123 | } 124 | 125 | if len(nodegroupsReadyForAmiUpdate) == 0 { 126 | logWithContext.Info().Msg("no nodegroups are ready for ami update") 127 | } 128 | 129 | return nodegroupsReadyForAmiUpdate, nil 130 | } 131 | 132 | func UpdateAmi(dryrun bool, skipNewerThanDays uint, regionsVar, nodegroupsVar []string, tagVar string, ctx context.Context) error { 133 | var errorGroup errgroup.Group 134 | 135 | nodegroups, err := GetNodeGroupsToUpdateAmi(skipNewerThanDays, regionsVar, nodegroupsVar, tagVar, ctx) 136 | if err != nil { 137 | return err 138 | } 139 | 140 | for _, nodegroup := range nodegroups { 141 | errorGroup.Go(func() error { 142 | return aws.AmiUpdate(nodegroup.Region, nodegroup.ClusterName, nodegroup.NodegroupName, dryrun, ctx) 143 | }) 144 | } 145 | 146 | return errors.Wrap(errorGroup.Wait(), "at least one nodegroup can not be updated") 147 | } 148 | -------------------------------------------------------------------------------- /pkg/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | func Contains(s []string, str string) bool { 4 | for _, v := range s { 5 | if v == str { 6 | return true 7 | } 8 | } 9 | 10 | return false 11 | } 12 | --------------------------------------------------------------------------------