├── .github ├── CHANGELOG.md ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DEVELOPER.md ├── ISSUE_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── push.yaml │ └── unit-test.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cmd ├── common.go ├── enroll.go ├── root.go ├── serve.go └── version.go ├── codecov.yml ├── examples └── lifecycle-manager.yaml ├── go.mod ├── go.sum ├── main.go └── pkg ├── enroll └── enroll.go ├── log ├── log.go └── retry_logger.go ├── service ├── autoscaling.go ├── autoscaling_test.go ├── deregistrator.go ├── elb.go ├── elb_test.go ├── elbv2.go ├── elbv2_test.go ├── events.go ├── events_test.go ├── lifecycle.go ├── manager.go ├── metrics.go ├── metrics_test.go ├── nodes.go ├── nodes_test.go ├── server.go ├── server_test.go ├── sqs.go ├── sqs_test.go └── target.go └── version └── version.go /.github/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.6.3 4 | 5 | * Bump github.com/spf13/cobra from 1.7.0 to 1.8.0 by @dependabot in #172 6 | * Bump github.com/aws/aws-sdk-go from 1.47.11 to 1.48.11 by @dependabot in #174 7 | * Bump github.com/keikoproj/inverse-exp-backoff from 0.0.3 to 0.0.4 by @dependabot in #175 8 | * Bump k8s.io/api from 0.25.15 to 0.25.16 by @dependabot in #173 9 | * Increase qps and burst to 100 by @ZihanJiang96 in #184 10 | 11 | ## v0.6.2 12 | 13 | **NOTE:** Beginning with this release, the docker image tag for `lifecycle-manager` will be prefixed with `v`. For example, `v0.6.2` instead of `0.6.2`. This is to align with the [Semantic Versioning](https://semver.org/) specification. 14 | 15 | ### Fixed 16 | * Improve dependabot configuration by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/154 17 | * use the copied node obj to drain the node by @ZihanJiang96 in https://github.com/keikoproj/lifecycle-manager/pull/165 18 | 19 | ### What's Changed 20 | * Bump github.com/avast/retry-go from 2.4.1+incompatible to 2.7.0+incompatible by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/110 21 | * Bump golang.org/x/sync from 0.0.0-20190911185100-cd5d95a43a6e to 0.3.0 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/120 22 | * Bump github.com/emicklei/go-restful from 2.9.5+incompatible to 2.16.0+incompatible by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/123 23 | * Bump github.com/spf13/cobra from 1.1.1 to 1.7.0 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/125 24 | * Bump github.com/aws/aws-sdk-go from 1.25.0 to 1.45.9 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/134 25 | * Bump actions/checkout from 3 to 4 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/138 26 | * Bump codecov/codecov-action from 3 to 4 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/140 27 | * Update to codecov@v3 unit-test.yaml by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/142 28 | * Bump docker/login-action from 2 to 3 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/141 29 | * Bump docker/metadata-action from 4 to 5 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/137 30 | * Bump docker/build-push-action from 4 to 5 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/136 31 | * Update to Golang 1.19 by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/148 32 | * Bump github.com/sirupsen/logrus from 1.6.0 to 1.9.3 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/143 33 | * Bump k8s.io/kubectl from 0.20.4 to 0.20.15 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/146 34 | * Bump docker/setup-qemu-action from 2 to 3 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/135 35 | * Bump docker/setup-buildx-action from 2 to 3 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/139 36 | * Bump github.com/aws/aws-sdk-go from 1.45.9 to 1.45.16 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/152 37 | * Bump github.com/prometheus/client_golang from 1.7.1 to 1.16.0 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/145 38 | * Update aws-sdk-go-cache and inverse-exp-backoff by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/153 39 | * Update client-go and kubectl to v0.25.14 by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/157 40 | * Bump github.com/aws/aws-sdk-go from 1.45.16 to 1.45.18 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/156 41 | * Bump github.com/prometheus/client_golang from 1.16.0 to 1.17.0 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/155 42 | * Bump golang.org/x/net from 0.12.0 to 0.17.0 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/159 43 | * Bump k8s.io/client-go from 0.25.14 to 0.25.15 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/160 44 | * Bump golang.org/x/sync from 0.3.0 to 0.5.0 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/166 45 | * Bump github.com/aws/aws-sdk-go from 1.45.18 to 1.47.11 by @dependabot in https://github.com/keikoproj/lifecycle-manager/pull/167 46 | * Add keiko-admins and keiko-maintainers as codeowners by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/168 47 | * Update kubectl to v0.25.15 by @tekenstam in https://github.com/keikoproj/lifecycle-manager/pull/169 48 | 49 | ### New Contributors 50 | * @tekenstam made their first contribution in https://github.com/keikoproj/lifecycle-manager/pull/142 51 | * @ZihanJiang96 made their first contribution in https://github.com/keikoproj/lifecycle-manager/pull/165 52 | 53 | **Full Changelog**: https://github.com/keikoproj/lifecycle-manager/compare/0.6.1...v0.6.2 54 | 55 | 56 | ## 0.6.1 57 | + Fix broken build badge (#108) 58 | + Include draintimeout and change node drain method. (#121) 59 | + Node deletion (#127) 60 | 61 | ## 0.6.0 62 | 63 | + Bug fix: Allow drain retries even with timeout failures (#104) 64 | + Package version update: Update docker actions to v4 (#106) 65 | + Package version update: Build updates, dependabot + GH Actions (#103) 66 | + Deperecate: Cleanup vendors (#94) 67 | + Package version update: Bump docker/login-action from 1 to 2 (#97) 68 | + Package version update: Update builders (#96) 69 | 70 | ## 0.5.1 71 | 72 | + Support cross-compliation for arm (#76) 73 | 74 | ## 0.5.0 75 | 76 | + Documentation fixes (#67) 77 | + Update kubectl binary to 1.18.14 (#68) 78 | + Allow lifecycle events longer than 1hr (#69) 79 | + Annotate node with queue URL (#71) 80 | + Move to Github Actions (#73) 81 | + Support Kubernetes 1.19 exclusion labels (#72) 82 | 83 | ## 0.4.3 84 | 85 | + Fix deadlock when resuming in-progress terminations (#63) 86 | 87 | ## 0.4.2 88 | 89 | + Update kubectl to 1.16.5 (#60) 90 | 91 | ## 0.4.1 92 | 93 | + Add option for refreshing expired AWS token (#57) 94 | + add Kind, firsttimestamp and count to v1.Event (#56) 95 | + Refactor message validation (#55) 96 | + Enrollment CLI (#53) 97 | 98 | ## 0.4.0 99 | 100 | + Separate drain timeout for nodes in Unknown state (#51) 101 | + Bucket drain events using semaphore (#49) 102 | 103 | ## 0.3.4 104 | 105 | + More efficient deregistration (#42) 106 | + Logging improvements (#42) 107 | + Waiters - use inverse exponential backoff (#42) 108 | + Error handling improvements (#42) 109 | + No cache flushing on DeregisterInstances (#42) 110 | 111 | ## 0.3.3 112 | 113 | + Bugfix: Proceed with drain failure (#37) 114 | + Bugfix: Drop goroutines when instance abandoned (#34) 115 | + Idempotency - resume operations after pod restart (#35) 116 | + API Caching - cache AWS calls to improve performance (#35) 117 | 118 | ## 0.3.2 119 | 120 | + Better naming for event reasons (#32) 121 | + Expose prometheus metrics (#29) 122 | 123 | ## 0.3.1 124 | 125 | + Documentation fixes (#19, #20) 126 | + Logging improvements and fixes (#21, #16) 127 | + Event publishing improvements (#26) 128 | + Better mechanism for AWS API calls to avoid being throttled (#25) 129 | + Bug fix: complete events when they fail (#27) 130 | 131 | ## 0.3.0 132 | 133 | + Improved error handling 134 | + Support classic-elb deregistration 135 | + Kubernetes event publishing 136 | 137 | ## 0.2.0 138 | 139 | + Support `--with-deregister` flag to deregister ALB instances 140 | + Support `--log-level` flag to set the logging verbosity 141 | + Add pagination and retries in AWS calls 142 | 143 | ## 0.1.0 144 | 145 | + Initial alpha release 146 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # PLEASE READ: 2 | 3 | # This is a comment. 4 | # Each line is a file pattern followed by one or more owners. 5 | 6 | # These owners will be the default owners for everything in 7 | # the repo. Unless a later match takes precedence, 8 | # review when someone opens a pull request. 9 | * @keikoproj/authorized-approvers 10 | 11 | # Admins own root and CI. 12 | .github/** @keikoproj/keiko-admins @keikoproj/keiko-maintainers 13 | /* @keikoproj/keiko-admins @keikoproj/keiko-maintainers 14 | -------------------------------------------------------------------------------- /.github/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | We welcome participation from individuals and groups of all backgrounds who want to benefit the broader open source community 4 | through participation in this project. We are dedicated to ensuring a productive, safe and educational experience for all. 5 | 6 | ## Guidelines 7 | 8 | Be welcoming 9 | 10 | * Make it easy for new members to learn and contribute. Help them along the path. Don't make them jump through hoops. 11 | 12 | Be considerate 13 | 14 | * There is a live person at the other end of the Internet. Consider how your comments will affect them. It is often better to give a quick but useful reply than to delay to compose a more thorough reply. 15 | 16 | Be respectful 17 | 18 | * Not everyone is Linus Torvalds, and this is probably a good thing :) but everyone is deserving of respect and consideration for wanting to benefit the broader community. Criticize ideas but respect the person. Saying something positive before you criticize lets the other person know that your criticism is not personal. 19 | 20 | Be patient 21 | 22 | * We have diverse backgrounds. It will take time and effort to understand each others' points of view. Some of us have day jobs and other responsibilities and may take time to respond to requests. 23 | 24 | ## Relevant References 25 | 26 | * 27 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to contribute 2 | 3 | 1. Fork it () 4 | 2. Open an issue and discuss the feature / bug 5 | 3. Create your feature branch (`git checkout -b feature/fooBar`) 6 | 4. Commit your changes (`git commit -am 'Add some fooBar'`) 7 | 5. Push to the branch (`git push origin feature/fooBar`) 8 | 6. Make sure unit tests and BDD is passing 9 | 7. Create a new Pull Request 10 | 11 | ## How to report a bug 12 | 13 | * What did you do? (how to reproduce) 14 | * What did you see? (include logs and screenshots as appropriate) 15 | * What did you expect? 16 | 17 | ## How to contribute a bug fix 18 | 19 | * Open an issue and discuss it. 20 | * Create a pull request for your fix. 21 | 22 | ## How to suggest a new feature 23 | 24 | * Open an issue and discuss it. 25 | -------------------------------------------------------------------------------- /.github/DEVELOPER.md: -------------------------------------------------------------------------------- 1 | # Development reference 2 | 3 | This document will walk you through setting up a basic testing environment, running unit tests and testing changes locally. 4 | This document will also assume a running Kubernetes cluster on AWS. 5 | 6 | ## Running locally 7 | 8 | You can run the `main.go` file with the appropriate command line arguments to invoke lifecycle-manager locally. 9 | The `--local-mode` flag tells lifecycle-manager to use a local kubeconfig from the provided path instead of `InClusterAuth`. 10 | You might want to export/set AWS credentials or profile if you are running in local mode. 11 | 12 | Make sure you already have an autoscaling group configured to post lifecycle hooks to an SQS queue. 13 | 14 | ### Example 15 | 16 | ```bash 17 | $ make build 18 | $ ./bin/lifecycle-manager serve --kubectl-path /usr/local/bin/kubectl --local-mode /path/to/.kube/config --queue-name my-queue --region us-west-2 19 | 20 | time="2019-09-28T05:15:58Z" level=info msg="starting lifecycle-manager service v0.2.0" 21 | time="2019-09-28T05:15:58Z" level=info msg="region = us-west-2" 22 | time="2019-09-28T05:15:58Z" level=info msg="queue = https://sqs.us-west-2.amazonaws.com/123456789012/my-queue" 23 | time="2019-09-28T05:15:58Z" level=info msg="polling interval seconds = 10" 24 | time="2019-09-28T05:15:58Z" level=info msg="drain timeout seconds = 300" 25 | time="2019-09-28T05:15:58Z" level=info msg="spawning sqs poller" 26 | time="2019-09-28T05:15:58Z" level=debug msg="polling for messages from queue" 27 | ``` 28 | 29 | Any terminating lifecycle hook sent to `my-queue` SQS queue, will now be processed by lifecycle-manager and nodes will be pre-drained. 30 | 31 | ## Running unit tests 32 | 33 | Using the `Makefile` you can run basic unit tests. 34 | 35 | ### Example 36 | 37 | ```bash 38 | $ make test 39 | go test ./... -coverprofile ./coverage.txt 40 | ? github.com/keikoproj/lifecycle-manager [no test files] 41 | ? github.com/keikoproj/lifecycle-manager/cmd [no test files] 42 | ? github.com/keikoproj/lifecycle-manager/pkg/log [no test files] 43 | ok github.com/keikoproj/lifecycle-manager/pkg/service 6.347s coverage: 63.8% of statements 44 | ? github.com/keikoproj/lifecycle-manager/pkg/version [no test files] 45 | go tool cover -html=./coverage.txt -o cover.html 46 | ``` 47 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **Is this a BUG REPORT or FEATURE REQUEST?**: 2 | 3 | **What happened**: 4 | 5 | **What you expected to happen**: 6 | 7 | **How to reproduce it (as minimally and precisely as possible)**: 8 | 9 | **Anything else we need to know?**: 10 | 11 | **Environment**: 12 | 13 | - Kubernetes version: 14 | 15 | ```bash 16 | kubectl version -o yaml 17 | ``` 18 | 19 | **Other debugging information (if applicable)**: 20 | 21 | - relevant logs: 22 | 23 | ```bash 24 | kubectl logs 25 | ``` 26 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | schedule: 11 | interval: "monthly" 12 | ignore: 13 | - dependency-name: "k8s.io*" ## K8s module version updates should be done explicitly 14 | update-types: ["version-update:semver-major", "version-update:semver-minor"] 15 | - dependency-name: "sigs.k8s.io*" ## K8s module version updates should be done explicitly 16 | update-types: ["version-update:semver-major", "version-update:semver-minor"] 17 | - dependency-name: "*" ## Major version updates should be done explicitly 18 | update-types: ["version-update:semver-major"] 19 | 20 | - package-ecosystem: "github-actions" 21 | directory: "/" 22 | schedule: 23 | interval: "monthly" 24 | 25 | - package-ecosystem: "docker" 26 | directory: "/" 27 | schedule: 28 | interval: "monthly" 29 | ignore: 30 | - dependency-name: "golang" ## Golang version should be done explicitly -------------------------------------------------------------------------------- /.github/workflows/push.yaml: -------------------------------------------------------------------------------- 1 | name: push 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: 7 | - '*' 8 | 9 | jobs: 10 | push: 11 | name: push 12 | if: github.repository_owner == 'keikoproj' 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v4 18 | 19 | - name: Set up Docker Buildx 20 | id: buildx 21 | uses: docker/setup-buildx-action@v3 22 | with: 23 | install: true 24 | version: latest 25 | 26 | - name: Available platforms 27 | run: echo ${{ steps.buildx.outputs.platforms }} 28 | 29 | - name: Set up QEMU 30 | id: qemu 31 | uses: docker/setup-qemu-action@v3 32 | with: 33 | platforms: all 34 | 35 | - name: Login to DockerHub 36 | uses: docker/login-action@v3 37 | with: 38 | username: ${{ secrets.DOCKERHUB_USERNAME }} 39 | password: ${{ secrets.DOCKERHUB_TOKEN }} 40 | 41 | - name: Docker meta 42 | id: docker_meta 43 | uses: docker/metadata-action@v5 44 | with: 45 | images: ${{ github.repository_owner }}/lifecycle-manager 46 | 47 | - name: Build and push 48 | uses: docker/build-push-action@v6 49 | with: 50 | context: . 51 | file: ./Dockerfile 52 | platforms: linux/amd64,linux/arm/v7,linux/arm64 53 | push: true 54 | tags: ${{ steps.docker_meta.outputs.tags }} -------------------------------------------------------------------------------- /.github/workflows/unit-test.yaml: -------------------------------------------------------------------------------- 1 | name: unit-test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | unit-test: 11 | if: github.repository_owner == 'keikoproj' 12 | name: unit-test 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Set up Go 1.x 16 | uses: actions/setup-go@v5 17 | with: 18 | go-version: ^1.24 19 | cache: true 20 | 21 | - name: Check out code into the Go module directory 22 | uses: actions/checkout@v4 23 | 24 | - name: Test 25 | run: | 26 | make test 27 | 28 | - name: Upload to Codecov 29 | uses: codecov/codecov-action@v5 30 | with: 31 | file: ./coverage.txt 32 | token: ${{ secrets.CODECOV_TOKEN }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage.txt 2 | cover.html 3 | bin 4 | bin/* 5 | .DS_Store 6 | .idea 7 | .tool-versions 8 | .windsurfrules 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build Stage 2 | FROM --platform=$BUILDPLATFORM golang:1.24-alpine as builder 3 | ARG TARGETOS TARGETARCH 4 | LABEL REPO="https://github.com/keikoproj/lifecycle-manager" 5 | 6 | WORKDIR /go/src/github.com/keikoproj/lifecycle-manager 7 | COPY . . 8 | 9 | RUN apk update && apk add --no-cache build-base make git ca-certificates && update-ca-certificates 10 | RUN addgroup -g 10001 -S lifecycle-manager && \ 11 | adduser --disabled-password \ 12 | --gecos "" \ 13 | --home "/nonexistent" \ 14 | --shell "/sbin/nologin" \ 15 | --no-create-home \ 16 | -G lifecycle-manager \ 17 | --uid 10001 \ 18 | lifecycle-manager 19 | ADD https://storage.googleapis.com/kubernetes-release/release/v1.25.12/bin/linux/amd64/kubectl /usr/local/bin/kubectl 20 | RUN chmod 777 /usr/local/bin/kubectl 21 | RUN make build 22 | 23 | # Final Stage 24 | FROM scratch 25 | 26 | ARG GIT_COMMIT 27 | ARG VERSION 28 | LABEL REPO="https://github.com/keikoproj/lifecycle-manager" 29 | LABEL GIT_COMMIT=$GIT_COMMIT 30 | LABEL VERSION=$VERSION 31 | 32 | COPY --from=builder /etc/passwd /etc/passwd 33 | COPY --from=builder /etc/group /etc/group 34 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 35 | COPY --from=builder /usr/local/bin/kubectl /usr/local/bin/kubectl 36 | COPY --from=builder /go/src/github.com/keikoproj/lifecycle-manager/bin/lifecycle-manager /bin/lifecycle-manager 37 | 38 | USER lifecycle-manager 39 | 40 | CMD ["/bin/lifecycle-manager", "--help"] 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2017-2018 The Keiko Authors 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build build-alpine clean test help default 2 | 3 | BIN_NAME=lifecycle-manager 4 | 5 | VERSION := $(shell grep "const Version " pkg/version/version.go | sed -E 's/.*"(.+)"$$/\1/') 6 | GIT_COMMIT=$(shell git rev-parse HEAD) 7 | GIT_DIRTY=$(shell test -n "`git status --porcelain`" && echo "+CHANGES" || true) 8 | BUILD_DATE=$(shell date '+%Y-%m-%d-%H:%M:%S') 9 | IMAGE_NAME ?= keikoproj/lifecycle-manager:latest 10 | TARGETOS ?= linux 11 | TARGETARCH ?= amd64 12 | LDFLAGS=-ldflags "-X github.com/keikoproj/lifecycle-manager/version.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -X github.com/keikoproj/lifecycle-manager/version.BuildDate=${BUILD_DATE}" 13 | TEST_FLAGS ?= 14 | 15 | default: test 16 | 17 | help: 18 | @echo 'Management commands for lifecycle-manager:' 19 | @echo 20 | @echo 'Usage:' 21 | @echo ' make build Compile the project.' 22 | @echo ' make get-deps runs dep ensure, mostly used for ci.' 23 | @echo ' make docker Build final docker image with just the go binary inside' 24 | @echo ' make tag Tag image created by package with latest, git commit and version' 25 | @echo ' make test Run tests on a compiled project.' 26 | @echo ' make vtest Run tests on a compiled project.' 27 | @echo ' make clean Clean the directory tree.' 28 | @echo 29 | 30 | build: 31 | @echo "building ${BIN_NAME} ${VERSION}" 32 | @echo "GOPATH=${GOPATH}" 33 | CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build ${LDFLAGS} -o bin/${BIN_NAME} github.com/keikoproj/lifecycle-manager 34 | 35 | get-deps: 36 | dep ensure 37 | 38 | docker-build: 39 | @echo "building image ${BIN_NAME} ${VERSION} $(GIT_COMMIT)" 40 | docker build --build-arg VERSION=${VERSION} --build-arg GIT_COMMIT=$(GIT_COMMIT) -t $(IMAGE_NAME) . 41 | 42 | docker-push: 43 | docker push ${IMAGE_NAME} 44 | 45 | clean: 46 | @test ! -e bin/${BIN_NAME} || rm bin/${BIN_NAME} 47 | 48 | vtest: TEST_FLAGS += -v 49 | 50 | test vtest: 51 | go test ./... $(TEST_FLAGS) -timeout 30s -coverprofile ./coverage.txt 52 | go tool cover -html=./coverage.txt -o cover.html 53 | 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lifecycle-manager 2 | 3 | [![Build Status](https://github.com/keikoproj/lifecycle-manager/actions/workflows/unit-test.yaml/badge.svg)](https://github.com/keikoproj/lifecycle-manager/actions/workflows/unit-test.yaml) 4 | [![codecov](https://codecov.io/gh/keikoproj/lifecycle-manager/branch/master/graph/badge.svg)](https://codecov.io/gh/keikoproj/lifecycle-manager) 5 | [![Go Report Card](https://goreportcard.com/badge/github.com/keikoproj/lifecycle-manager)](https://goreportcard.com/report/github.com/keikoproj/lifecycle-manager) 6 | ![version](https://img.shields.io/badge/version-0.6.3-green.svg?cacheSeconds=2592000) 7 | > Graceful AWS scaling event on Kubernetes using lifecycle hooks 8 | 9 | lifecycle-manager is a service that can be deployed to a Kubernetes cluster in order to make AWS autoscaling events more graceful using draining 10 | 11 | Certain termination activities such as AZRebalance or TerminateInstanceInAutoScalingGroup API calls can cause autoscaling groups to terminate instances without having them properly drain first. 12 | 13 | This can cause apps to experience errors when they are abruptly terminated. 14 | 15 | lifecycle-manager uses lifecycle hooks from the autoscaling group (via SQS) to pre-drain the instances for you. 16 | 17 | In addition to node draining, lifecycle-manager also tries to deregister the instance from any discovered ALB target group, this helps with pre-draining for the ALB instances prior to shutdown in order to avoid in-flight 5xx errors on your ALB - this feature is currently supported for `aws-alb-ingress-controller`. 18 | 19 | ## Usage 20 | 21 | 1. Configure your scaling groups to notify lifecycle-manager of terminations. you can use the provided enrollment CLI by running 22 | 23 | ```bash 24 | $ make build 25 | ... 26 | 27 | $ ./bin/lifecycle-manager enroll --region us-west-2 --queue-name lifecycle-manager-queue --notification-role-name my-notification-role --target-scaling-groups scaling-group-1,scaling-group-2 --overwrite 28 | 29 | INFO[0000] starting enrollment for scaling groups [scaling-group-1 scaling-group-2] 30 | INFO[0000] creating notification role 'my-notification-role' 31 | INFO[0000] notification role 'my-notification-role' already exist, updating... 32 | INFO[0000] attaching notification policy 'arn:aws:iam::aws:policy/service-role/AutoScalingNotificationAccessRole' 33 | INFO[0001] created notification role 'arn:aws:iam::000000000000:role/my-notification-role' 34 | INFO[0001] creating SQS queue 'lifecycle-manager-queue' 35 | INFO[0001] created queue 'arn:aws:sqs:us-west-2:000000000000:lifecycle-manager-queue' 36 | INFO[0001] creating lifecycle hook for 'scaling-group-1' 37 | INFO[0002] creating lifecycle hook for 'scaling-group-2' 38 | INFO[0002] successfully enrolled 2 scaling groups 39 | INFO[0002] Queue Name: lifecycle-manager-queue 40 | INFO[0002] Queue URL: https://sqs.us-west-2.amazonaws.com/000000000000/lifecycle-manager-queue 41 | ``` 42 | 43 | Alternatively, you can simply follow the [AWS docs](https://docs.aws.amazon.com/autoscaling/ec2/userguide/lifecycle-hooks.html#sqs-notifications) to create an SQS queue named `lifecycle-manager-queue`, a notification role, and a lifecycle-hook on your autoscaling group pointing to the created queue. 44 | 45 | Configured scaling groups will now publish termination hooks to the SQS queue you created. 46 | 47 | 2. Deploy lifecycle-manager to your cluster: 48 | 49 | ```bash 50 | kubectl create namespace lifecycle-manager 51 | 52 | kubectl apply -f https://raw.githubusercontent.com/keikoproj/lifecycle-manager/master/examples/lifecycle-manager.yaml 53 | ``` 54 | 55 | Modifications may be needed if you used a different queue name than mentioned above 56 | 57 | 3. Kill an instance in your scaling group and watch it getting drained: 58 | 59 | ```bash 60 | $ aws autoscaling terminate-instance-in-auto-scaling-group --instance-id i-0868736e381bf942a --region us-west-2 --no-should-decrement-desired-capacity 61 | { 62 | "Activity": { 63 | "ActivityId": "5285b629-6a18-0a43-7c3c-f76bac8205f0", 64 | "AutoScalingGroupName": "scaling-group-1", 65 | "Description": "Terminating EC2 instance: i-0868736e381bf942a", 66 | "Cause": "At 2019-10-02T02:44:11Z instance i-0868736e381bf942a was taken out of service in response to a user request.", 67 | "StartTime": "2019-10-02T02:44:11.394Z", 68 | "StatusCode": "InProgress", 69 | "Progress": 0, 70 | "Details": "{\"Subnet ID\":\"subnet-0bf9bc85fEXAMPLE\",\"Availability Zone\":\"us-west-2c\"}" 71 | } 72 | } 73 | 74 | $ kubectl logs lifecycle-manager 75 | time="2020-03-10T23:44:20Z" level=info msg="starting lifecycle-manager service v0.3.4" 76 | time="2020-03-10T23:44:20Z" level=info msg="region = us-west-2" 77 | time="2020-03-10T23:44:20Z" level=info msg="queue = lifecycle-manager-queue" 78 | time="2020-03-10T23:44:20Z" level=info msg="polling interval seconds = 10" 79 | time="2020-03-10T23:44:20Z" level=info msg="node drain timeout seconds = 300" 80 | time="2020-03-10T23:44:20Z" level=info msg="node drain retry interval seconds = 30" 81 | time="2020-03-10T23:44:20Z" level=info msg="with alb deregister = true" 82 | time="2020-03-10T23:44:20Z" level=info msg="starting metrics server on /metrics:8080" 83 | time="2020-03-11T07:24:37Z" level=info msg="i-0868736e381bf942a> received termination event" 84 | time="2020-03-11T07:24:37Z" level=info msg="i-0868736e381bf942a> sending heartbeat (1/24)" 85 | time="2020-03-11T07:24:37Z" level=info msg="i-0868736e381bf942a> draining node/ip-10-105-232-73.us-west-2.compute.internal" 86 | time="2020-03-11T07:24:37Z" level=info msg="i-0868736e381bf942a> completed drain for node/ip-10-105-232-73.us-west-2.compute.internal" 87 | time="2020-03-11T07:24:45Z" level=info msg="i-0868736e381bf942a> starting load balancer drain worker" 88 | ... 89 | time="2020-03-11T07:24:49Z" level=info msg="event ce25c321-ec67-3f0b-c156-a7c1f75caf1a completed processing" 90 | time="2020-03-11T07:24:49Z" level=info msg="i-0868736e381bf942a> setting lifecycle event as completed with result: CONTINUE" 91 | time="2020-03-11T07:24:49Z" level=info msg="event ce25c321-ec67-3f0b-c156-a7c1f75caf1a for instance i-0868736e381bf942a completed after 12.054675203s" 92 | ``` 93 | 94 | ### Required AWS Auth 95 | 96 | ```json 97 | { 98 | "Effect": "Allow", 99 | "Action": [ 100 | "autoscaling:DescribeLifecycleHooks", 101 | "autoscaling:CompleteLifecycleAction", 102 | "autoscaling:RecordLifecycleActionHeartbeat", 103 | "sqs:ReceiveMessage", 104 | "sqs:DeleteMessage", 105 | "sqs:GetQueueUrl", 106 | "ec2:DescribeSecurityGroups", 107 | "ec2:DescribeClassicLinkInstances", 108 | "ec2:DescribeInstances", 109 | "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", 110 | "elasticloadbalancing:DescribeInstanceHealth", 111 | "elasticloadbalancing:DescribeLoadBalancers", 112 | "elasticloadbalancing:DeregisterTargets", 113 | "elasticloadbalancing:DescribeTargetHealth", 114 | "elasticloadbalancing:DescribeTargetGroups" 115 | ], 116 | "Resource": "*" 117 | } 118 | ``` 119 | 120 | # Flags 121 | | Name | Default | Type | Description | 122 | |:------:|:---------:|:------:|:-------------:| 123 | | local-mode | "" | String | absolute path to kubeconfig | 124 | | region | "" | String | AWS region to operate in | 125 | | queue-name | "" | String | the name of the SQS queue to consume lifecycle hooks from | 126 | | kubectl-path | "/usr/local/bin/kubectl" | String | the path to kubectl binary | 127 | | log-level | "info" | String | the logging level (info, warning, debug) | 128 | | max-drain-concurrency | 32 | Int | maximum number of node drains to process in parallel | 129 | | max-time-to-process | 3600 | Int | max time to spend processing an event | 130 | | drain-timeout | 300 | Int | hard time limit for draining healthy nodes | 131 | | drain-timeout-unknown | 30 | Int | hard time limit for draining nodes that are in unknown state | 132 | | drain-interval | 30 | Int | interval in seconds for which to retry draining | 133 | | drain-retries | 3 | Int | number of times to retry the node drain operation | 134 | | polling-interval | 10 | Int | interval in seconds for which to poll SQS | 135 | | with-deregister | true | Bool | try to deregister deleting instance from target groups | 136 | | refresh-expired-credentials | false | Bool | refreshes expired credentials (requires shared credentials file) | 137 | | deregister-target-types | "classic-elb,target-group" | String | comma separated list of target types to deregister instance from (classic-elb, target-group) | 138 | 139 | 140 | ## Release History 141 | 142 | Please see [CHANGELOG.md](.github/CHANGELOG.md). 143 | 144 | ## ❤ Contributing ❤ 145 | 146 | Please see [CONTRIBUTING.md](.github/CONTRIBUTING.md). 147 | 148 | ## Developer Guide 149 | 150 | Please see [DEVELOPER.md](.github/DEVELOPER.md). 151 | -------------------------------------------------------------------------------- /cmd/common.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/credentials" 8 | "github.com/aws/aws-sdk-go/aws/defaults" 9 | "github.com/aws/aws-sdk-go/aws/request" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/aws/aws-sdk-go/service/autoscaling" 12 | "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" 13 | "github.com/aws/aws-sdk-go/service/elb" 14 | "github.com/aws/aws-sdk-go/service/elb/elbiface" 15 | "github.com/aws/aws-sdk-go/service/elbv2" 16 | "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" 17 | "github.com/aws/aws-sdk-go/service/iam" 18 | "github.com/aws/aws-sdk-go/service/iam/iamiface" 19 | "github.com/aws/aws-sdk-go/service/sqs" 20 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 21 | "github.com/keikoproj/aws-sdk-go-cache/cache" 22 | "github.com/keikoproj/lifecycle-manager/pkg/log" 23 | "k8s.io/client-go/kubernetes" 24 | "k8s.io/client-go/rest" 25 | "k8s.io/client-go/tools/clientcmd" 26 | ) 27 | 28 | func newKubernetesClient(localMode string) *kubernetes.Clientset { 29 | var config *rest.Config 30 | var err error 31 | 32 | if localMode != "" { 33 | // use kubeconfig 34 | config, err = clientcmd.BuildConfigFromFlags("", localMode) 35 | if err != nil { 36 | log.Fatalf("cannot load kubernetes config from '%v', Err=%s", localMode, err) 37 | } 38 | } else { 39 | // use InCluster auth 40 | config, err = rest.InClusterConfig() 41 | if err != nil { 42 | log.Fatalln("cannot load kubernetes config from InCluster") 43 | } 44 | config.QPS = 100 45 | config.Burst = 100 46 | } 47 | return kubernetes.NewForConfigOrDie(config) 48 | } 49 | 50 | func newIAMClient(region string) iamiface.IAMAPI { 51 | config := aws.NewConfig().WithRegion(region) 52 | config = config.WithCredentialsChainVerboseErrors(true) 53 | sess, err := session.NewSession(config) 54 | if err != nil { 55 | log.Fatalf("failed to create iam client, %v", err) 56 | } 57 | return iam.New(sess) 58 | } 59 | 60 | func newAWSSession(region string) (*session.Session, error) { 61 | config := aws.NewConfig().WithRegion(region) 62 | config = config.WithCredentialsChainVerboseErrors(true) 63 | 64 | if refreshExpiredCredentials { 65 | filename := os.Getenv("AWS_SHARED_CREDENTIALS_FILE") 66 | if filename == "" { 67 | filename = defaults.SharedCredentialsFilename() 68 | } 69 | 70 | profile := os.Getenv("AWS_PROFILE") 71 | if profile == "" { 72 | profile = "default" 73 | } 74 | 75 | // When corehandlers.AfterRetryHandler calls Config.Credentials.Expire, 76 | // the SharedCredentialsProvider forces refreshing credentials from file. 77 | // With the SDK's default credential chain, the file is never read. 78 | config.WithCredentials(credentials.NewCredentials(&credentials.SharedCredentialsProvider{ 79 | Filename: filename, 80 | Profile: profile, 81 | })) 82 | } 83 | 84 | config = request.WithRetryer(config, log.NewRetryLogger(DefaultRetryer)) 85 | sess, err := session.NewSession(config) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return sess, nil 91 | } 92 | 93 | func newELBv2Client(region string, cacheCfg *cache.Config) elbv2iface.ELBV2API { 94 | sess, err := newAWSSession(region) 95 | if err != nil { 96 | log.Fatalf("failed to create AWS session, %s", err) 97 | } 98 | 99 | cache.AddCaching(sess, cacheCfg) 100 | cacheCfg.SetCacheTTL("elasticloadbalancing", "DescribeTargetHealth", DescribeTargetHealthTTL) 101 | cacheCfg.SetCacheTTL("elasticloadbalancing", "DescribeTargetGroups", DescribeTargetGroupsTTL) 102 | cacheCfg.SetCacheMutating("elasticloadbalancing", "DeregisterTargets", false) 103 | sess.Handlers.Complete.PushFront(func(r *request.Request) { 104 | ctx := r.HTTPRequest.Context() 105 | log.Debugf("cache hit => %v, service => %s.%s", 106 | cache.IsCacheHit(ctx), 107 | r.ClientInfo.ServiceName, 108 | r.Operation.Name, 109 | ) 110 | }) 111 | 112 | return elbv2.New(sess) 113 | } 114 | 115 | func newELBClient(region string, cacheCfg *cache.Config) elbiface.ELBAPI { 116 | sess, err := newAWSSession(region) 117 | if err != nil { 118 | log.Fatalf("failed to create AWS session, %s", err) 119 | } 120 | 121 | cache.AddCaching(sess, cacheCfg) 122 | cacheCfg.SetCacheTTL("elasticloadbalancing", "DescribeInstanceHealth", DescribeInstanceHealthTTL) 123 | cacheCfg.SetCacheTTL("elasticloadbalancing", "DescribeLoadBalancers", DescribeLoadBalancersTTL) 124 | cacheCfg.SetCacheMutating("elasticloadbalancing", "DeregisterInstancesFromLoadBalancer", false) 125 | sess.Handlers.Complete.PushFront(func(r *request.Request) { 126 | ctx := r.HTTPRequest.Context() 127 | log.Debugf("cache hit => %v, service => %s.%s", 128 | cache.IsCacheHit(ctx), 129 | r.ClientInfo.ServiceName, 130 | r.Operation.Name, 131 | ) 132 | }) 133 | 134 | return elb.New(sess) 135 | } 136 | 137 | func newSQSClient(region string) sqsiface.SQSAPI { 138 | sess, err := newAWSSession(region) 139 | if err != nil { 140 | log.Fatalf("failed to create AWS session, %s", err) 141 | } 142 | 143 | return sqs.New(sess) 144 | } 145 | 146 | func newASGClient(region string) autoscalingiface.AutoScalingAPI { 147 | sess, err := newAWSSession(region) 148 | if err != nil { 149 | log.Fatalf("failed to create AWS session, %s", err) 150 | } 151 | 152 | return autoscaling.New(sess) 153 | } 154 | -------------------------------------------------------------------------------- /cmd/enroll.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/keikoproj/lifecycle-manager/pkg/enroll" 5 | "github.com/keikoproj/lifecycle-manager/pkg/log" 6 | "github.com/spf13/cobra" 7 | ) 8 | 9 | var ( 10 | overwrite bool 11 | enrollRegion string 12 | enrollQueueName string 13 | notificationRoleName string 14 | heartbeatTimeout uint 15 | targetScalingGroups []string 16 | ) 17 | 18 | // enrollCmd represents the enroll command 19 | var enrollCmd = &cobra.Command{ 20 | Use: "enroll", 21 | Short: "creates an SQS queue and enrolls ASGs to send hooks to it", 22 | Long: `enroll does the initial required setup to start running lifecycle-manager, you can use 23 | this CLI as a pre-setup step to running the controller`, 24 | Run: func(cmd *cobra.Command, args []string) { 25 | // argument validation 26 | validateEnroll() 27 | log.SetLevel(logLevel) 28 | 29 | // prepare auth clients 30 | auth := enroll.EnrollmentAuthenticator{ 31 | ScalingGroupClient: newASGClient(enrollRegion), 32 | SQSClient: newSQSClient(enrollRegion), 33 | IAMClient: newIAMClient(enrollRegion), 34 | } 35 | 36 | // prepare runtime context 37 | context := &enroll.EnrollmentContext{ 38 | Region: enrollRegion, 39 | QueueName: enrollQueueName, 40 | NotificationRoleName: notificationRoleName, 41 | TargetScalingGroups: targetScalingGroups, 42 | HeartbeatTimeout: heartbeatTimeout, 43 | Overwrite: overwrite, 44 | } 45 | 46 | e := enroll.New(auth, context) 47 | e.Start() 48 | }, 49 | } 50 | 51 | func init() { 52 | rootCmd.AddCommand(enrollCmd) 53 | enrollCmd.Flags().BoolVar(&overwrite, "overwrite", false, "update resources if they already exist") 54 | enrollCmd.Flags().StringVar(&enrollRegion, "region", "", "AWS region to operate in") 55 | enrollCmd.Flags().StringVar(&enrollQueueName, "queue-name", "", "the name of the SQS queue to create") 56 | enrollCmd.Flags().StringVar(¬ificationRoleName, "notification-role-name", "", "the name of the notification IAM role to create") 57 | enrollCmd.Flags().StringSliceVar(&targetScalingGroups, "target-scaling-groups", []string{}, "comma separated list of auto scaling group names") 58 | enrollCmd.Flags().UintVar(&heartbeatTimeout, "heartbeat-timeout", 300, "lifecycle hook heartbeat timeout") 59 | } 60 | 61 | func validateEnroll() { 62 | if enrollRegion == "" { 63 | log.Fatalf("--region was not provided") 64 | } 65 | 66 | if enrollQueueName == "" { 67 | log.Fatalf("--queue-name was not provided") 68 | } 69 | 70 | if notificationRoleName == "" { 71 | log.Fatalf("--notification-role-name was not provided") 72 | } 73 | 74 | if len(targetScalingGroups) == 0 { 75 | log.Fatalf("--target-scaling-groups was not provided") 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var cfgFile string 11 | 12 | // rootCmd represents the base command when called without any subcommands 13 | var rootCmd = &cobra.Command{ 14 | Use: "lifecycle-manager", 15 | Short: "lifecycle-manager makes AWS scaling event on Kubernetes graceful using lifecycle hooks", 16 | Long: `lifecycle-manager is a service that can be deployed to a Kubernetes cluster in order to make AWS autoscaling events more graceful using draining 17 | 18 | lifecycle-manager serve --kubectl-path /usr/local/bin/kubectl --queue-name test-queue --region us-west-2`, 19 | } 20 | 21 | // Execute adds all child commands to the root command and sets flags appropriately. 22 | // This is called by main.main(). It only needs to happen once to the rootCmd. 23 | func Execute() { 24 | if err := rootCmd.Execute(); err != nil { 25 | fmt.Println(err) 26 | os.Exit(1) 27 | } 28 | } 29 | 30 | func init() { 31 | cobra.OnInitialize() 32 | } 33 | -------------------------------------------------------------------------------- /cmd/serve.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws/client" 9 | "github.com/keikoproj/aws-sdk-go-cache/cache" 10 | "github.com/keikoproj/lifecycle-manager/pkg/log" 11 | "github.com/keikoproj/lifecycle-manager/pkg/service" 12 | "github.com/spf13/cobra" 13 | "golang.org/x/sync/semaphore" 14 | ) 15 | 16 | const ( 17 | CacheDefaultTTL time.Duration = time.Second * 0 18 | DescribeTargetHealthTTL time.Duration = 120 * time.Second 19 | DescribeInstanceHealthTTL time.Duration = 120 * time.Second 20 | DescribeTargetGroupsTTL time.Duration = 300 * time.Second 21 | DescribeLoadBalancersTTL time.Duration = 300 * time.Second 22 | CacheMaxItems int64 = 5000 23 | CacheItemsToPrune uint32 = 500 24 | ) 25 | 26 | var ( 27 | localMode string 28 | region string 29 | queueName string 30 | kubectlLocalPath string 31 | nodeName string 32 | logLevel string 33 | deregisterTargetGroups bool 34 | deregisterTargetTypes []string 35 | refreshExpiredCredentials bool 36 | drainRetryIntervalSeconds int 37 | maxDrainConcurrency int64 38 | drainTimeoutSeconds int 39 | drainTimeoutUnknownSeconds int 40 | drainRetryAttempts int 41 | pollingIntervalSeconds int 42 | maxTimeToProcessSeconds int64 43 | 44 | // DefaultRetryer is the default retry configuration for some AWS API calls 45 | DefaultRetryer = client.DefaultRetryer{ 46 | NumMaxRetries: 250, 47 | MinThrottleDelay: time.Second * 5, 48 | MaxThrottleDelay: time.Second * 60, 49 | MinRetryDelay: time.Second * 1, 50 | MaxRetryDelay: time.Second * 5, 51 | } 52 | ) 53 | 54 | // serveCmd represents the serve command 55 | var serveCmd = &cobra.Command{ 56 | Use: "serve", 57 | Short: "start the lifecycle-manager service", 58 | Long: `Start watching lifecycle events for a given queue`, 59 | Run: func(cmd *cobra.Command, args []string) { 60 | // argument validation 61 | validateServe() 62 | log.SetLevel(logLevel) 63 | cacheCfg := cache.NewConfig(CacheDefaultTTL, 1*time.Hour, CacheMaxItems, CacheItemsToPrune) 64 | 65 | // prepare auth clients 66 | auth := service.Authenticator{ 67 | ScalingGroupClient: newASGClient(region), 68 | SQSClient: newSQSClient(region), 69 | ELBv2Client: newELBv2Client(region, cacheCfg), 70 | ELBClient: newELBClient(region, cacheCfg), 71 | KubernetesClient: newKubernetesClient(localMode), 72 | } 73 | 74 | // prepare runtime context 75 | context := service.ManagerContext{ 76 | CacheConfig: cacheCfg, 77 | KubectlLocalPath: kubectlLocalPath, 78 | QueueName: queueName, 79 | DrainTimeoutSeconds: int64(drainTimeoutSeconds), 80 | DrainTimeoutUnknownSeconds: int64(drainTimeoutUnknownSeconds), 81 | PollingIntervalSeconds: int64(pollingIntervalSeconds), 82 | DrainRetryIntervalSeconds: int64(drainRetryIntervalSeconds), 83 | MaxDrainConcurrency: semaphore.NewWeighted(maxDrainConcurrency), 84 | MaxTimeToProcessSeconds: int64(maxTimeToProcessSeconds), 85 | DrainRetryAttempts: uint(drainRetryAttempts), 86 | Region: region, 87 | WithDeregister: deregisterTargetGroups, 88 | DeregisterTargetTypes: deregisterTargetTypes, 89 | } 90 | 91 | s := service.New(auth, context) 92 | s.Start() 93 | }, 94 | } 95 | 96 | func init() { 97 | rootCmd.AddCommand(serveCmd) 98 | serveCmd.Flags().StringVar(&localMode, "local-mode", "", "absolute path to kubeconfig") 99 | serveCmd.Flags().StringVar(®ion, "region", "", "AWS region to operate in") 100 | serveCmd.Flags().StringVar(&queueName, "queue-name", "", "the name of the SQS queue to consume lifecycle hooks from") 101 | serveCmd.Flags().StringVar(&kubectlLocalPath, "kubectl-path", "/usr/local/bin/kubectl", "the path to kubectl binary") 102 | serveCmd.Flags().StringVar(&logLevel, "log-level", "info", "the logging level (info, warning, debug)") 103 | serveCmd.Flags().Int64Var(&maxDrainConcurrency, "max-drain-concurrency", 32, "maximum number of node drains to process in parallel") 104 | serveCmd.Flags().Int64Var(&maxTimeToProcessSeconds, "max-time-to-process", 3600, "max time to spend processing an event") 105 | serveCmd.Flags().IntVar(&drainTimeoutSeconds, "drain-timeout", 300, "hard time limit for draining healthy nodes") 106 | serveCmd.Flags().IntVar(&drainTimeoutUnknownSeconds, "drain-timeout-unknown", 30, "hard time limit for draining nodes that are in unknown state") 107 | serveCmd.Flags().IntVar(&drainRetryIntervalSeconds, "drain-interval", 30, "interval in seconds for which to retry draining") 108 | serveCmd.Flags().IntVar(&drainRetryAttempts, "drain-retries", 3, "number of times to retry the node drain operation") 109 | serveCmd.Flags().IntVar(&pollingIntervalSeconds, "polling-interval", 10, "interval in seconds for which to poll SQS") 110 | serveCmd.Flags().BoolVar(&deregisterTargetGroups, "with-deregister", true, "try to deregister deleting instance from target groups") 111 | serveCmd.Flags().StringSliceVar(&deregisterTargetTypes, "deregister-target-types", []string{service.TargetTypeClassicELB.String(), service.TargetTypeTargetGroup.String()}, 112 | fmt.Sprintf("comma separated list of target types to deregister instance from (%s, %s)", service.TargetTypeClassicELB.String(), service.TargetTypeTargetGroup.String())) 113 | serveCmd.Flags().BoolVar(&refreshExpiredCredentials, "refresh-expired-credentials", false, "refreshes expired credentials (requires shared credentials file)") 114 | } 115 | 116 | func validateServe() { 117 | if localMode != "" { 118 | if _, err := os.Stat(localMode); os.IsNotExist(err) { 119 | log.Fatalf("provided kubeconfig path does not exist") 120 | } 121 | } 122 | 123 | if kubectlLocalPath != "" { 124 | if _, err := os.Stat(kubectlLocalPath); os.IsNotExist(err) { 125 | log.Fatalf("provided kubectl path does not exist") 126 | } 127 | } else { 128 | log.Fatalf("must provide kubectl path") 129 | } 130 | 131 | if region == "" { 132 | log.Fatalf("must provide valid AWS region name") 133 | } 134 | 135 | if queueName == "" { 136 | log.Fatalf("must provide valid SQS queue name") 137 | } 138 | 139 | if maxDrainConcurrency < 1 { 140 | log.Fatalf("--max-drain-concurrency must be set to a value higher than 0") 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/keikoproj/lifecycle-manager/pkg/version" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | // versionCmd represents the version command 11 | var versionCmd = &cobra.Command{ 12 | Use: "version", 13 | Short: "Print the version number of lifecycle-manager", 14 | Long: `All software has versions. This is lifecycle-manager`, 15 | Run: func(cmd *cobra.Command, args []string) { 16 | fmt.Println("Build Date:", version.BuildDate) 17 | fmt.Println("Git Commit:", version.GitCommit) 18 | fmt.Println("Version:", version.Version) 19 | fmt.Println("Go Version:", version.GoVersion) 20 | fmt.Println("OS / Arch:", version.OsArch) 21 | }, 22 | } 23 | 24 | func init() { 25 | rootCmd.AddCommand(versionCmd) 26 | } 27 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | range: "70...90" 3 | ignore: 4 | - "pkg/enroll" 5 | - "cmd" -------------------------------------------------------------------------------- /examples/lifecycle-manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: lifecycle-manager 5 | --- 6 | apiVersion: v1 7 | kind: ServiceAccount 8 | metadata: 9 | name: lifecycle-manager 10 | namespace: lifecycle-manager 11 | --- 12 | kind: ClusterRole 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | metadata: 15 | name: lifecycle-manager 16 | rules: 17 | - apiGroups: [""] 18 | resources: ["nodes"] 19 | verbs: ["get", "list", "patch"] 20 | - apiGroups: [""] 21 | resources: ["pods"] 22 | verbs: ["get", "list"] 23 | - apiGroups: [""] 24 | resources: ["pods/eviction"] 25 | verbs: ["create"] 26 | - apiGroups: ["extensions", "apps"] 27 | resources: ["daemonsets"] 28 | verbs: ["get"] 29 | - apiGroups: [""] 30 | resources: ["events"] 31 | verbs: ["get", "list", "create"] 32 | --- 33 | kind: ClusterRoleBinding 34 | apiVersion: rbac.authorization.k8s.io/v1 35 | metadata: 36 | name: lifecycle-manager 37 | subjects: 38 | - kind: ServiceAccount 39 | name: lifecycle-manager 40 | namespace: lifecycle-manager 41 | roleRef: 42 | kind: ClusterRole 43 | name: lifecycle-manager 44 | apiGroup: rbac.authorization.k8s.io 45 | --- 46 | apiVersion: apps/v1 47 | kind: Deployment 48 | metadata: 49 | name: lifecycle-manager 50 | namespace: lifecycle-manager 51 | labels: 52 | app: lifecycle-manager 53 | spec: 54 | replicas: 1 55 | selector: 56 | matchLabels: 57 | app: lifecycle-manager 58 | template: 59 | metadata: 60 | labels: 61 | app: lifecycle-manager 62 | spec: 63 | serviceAccountName: lifecycle-manager 64 | containers: 65 | - image: keikoproj/lifecycle-manager:latest 66 | imagePullPolicy: "Always" 67 | name: lifecycle-manager 68 | resources: 69 | limits: 70 | # tested against surges of up to 100 nodes terminating, you may want more/less memory 71 | # depending on your cluster behavior 72 | memory: 2048Mi 73 | requests: 74 | cpu: 100m 75 | memory: 256Mi 76 | command: 77 | - /bin/lifecycle-manager 78 | - serve 79 | - --queue-name=lifecycle-manager-queue 80 | - --region=us-west-2 -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/keikoproj/lifecycle-manager 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.1 6 | 7 | require ( 8 | github.com/aws/aws-sdk-go v1.55.7 9 | github.com/keikoproj/aws-sdk-go-cache v0.1.0 10 | github.com/keikoproj/inverse-exp-backoff v0.1.1 11 | github.com/pkg/errors v0.9.1 12 | github.com/prometheus/client_golang v1.22.0 13 | github.com/sirupsen/logrus v1.9.3 14 | github.com/spf13/cobra v1.9.1 15 | golang.org/x/sync v0.13.0 16 | k8s.io/api v0.32.4 17 | k8s.io/apimachinery v0.32.4 18 | k8s.io/client-go v0.32.4 19 | k8s.io/kubectl v0.32.4 20 | ) 21 | 22 | require ( 23 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 24 | github.com/MakeNowJust/heredoc v1.0.0 // indirect 25 | github.com/beorn7/perks v1.0.1 // indirect 26 | github.com/blang/semver/v4 v4.0.0 // indirect 27 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 28 | github.com/chai2010/gettext-go v1.0.2 // indirect 29 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 30 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 31 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect 32 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 33 | github.com/go-errors/errors v1.4.2 // indirect 34 | github.com/go-logr/logr v1.4.2 // indirect 35 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 36 | github.com/go-openapi/jsonreference v0.20.2 // indirect 37 | github.com/go-openapi/swag v0.23.0 // indirect 38 | github.com/gogo/protobuf v1.3.2 // indirect 39 | github.com/golang/glog v1.2.4 // indirect 40 | github.com/golang/protobuf v1.5.4 // indirect 41 | github.com/google/btree v1.0.1 // indirect 42 | github.com/google/gnostic-models v0.6.8 // indirect 43 | github.com/google/go-cmp v0.7.0 // indirect 44 | github.com/google/gofuzz v1.2.0 // indirect 45 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect 46 | github.com/google/uuid v1.6.0 // indirect 47 | github.com/gorilla/websocket v1.5.0 // indirect 48 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect 49 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 50 | github.com/jmespath/go-jmespath v0.4.0 // indirect 51 | github.com/josharian/intern v1.0.0 // indirect 52 | github.com/json-iterator/go v1.1.12 // indirect 53 | github.com/karlseguin/ccache/v2 v2.0.8 // indirect 54 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect 55 | github.com/mailru/easyjson v0.7.7 // indirect 56 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 57 | github.com/moby/spdystream v0.5.0 // indirect 58 | github.com/moby/term v0.5.0 // indirect 59 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 60 | github.com/modern-go/reflect2 v1.0.2 // indirect 61 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect 62 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 63 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect 64 | github.com/peterbourgon/diskv v2.0.1+incompatible // indirect 65 | github.com/prometheus/client_model v0.6.1 // indirect 66 | github.com/prometheus/common v0.63.0 // indirect 67 | github.com/prometheus/procfs v0.16.0 // indirect 68 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 69 | github.com/spf13/pflag v1.0.6 // indirect 70 | github.com/x448/float16 v0.8.4 // indirect 71 | github.com/xlab/treeprint v1.2.0 // indirect 72 | golang.org/x/net v0.38.0 // indirect 73 | golang.org/x/oauth2 v0.25.0 // indirect 74 | golang.org/x/sys v0.31.0 // indirect 75 | golang.org/x/term v0.30.0 // indirect 76 | golang.org/x/text v0.23.0 // indirect 77 | golang.org/x/time v0.7.0 // indirect 78 | google.golang.org/protobuf v1.36.6 // indirect 79 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 80 | gopkg.in/inf.v0 v0.9.1 // indirect 81 | gopkg.in/yaml.v3 v3.0.1 // indirect 82 | k8s.io/cli-runtime v0.32.4 // indirect 83 | k8s.io/component-base v0.32.4 // indirect 84 | k8s.io/klog/v2 v2.130.1 // indirect 85 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect 86 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect 87 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 88 | sigs.k8s.io/kustomize/api v0.18.0 // indirect 89 | sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect 90 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect 91 | sigs.k8s.io/yaml v1.4.0 // indirect 92 | ) 93 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= 2 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 3 | github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= 4 | github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= 5 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= 6 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= 7 | github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= 8 | github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= 12 | github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= 13 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 14 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 15 | github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= 16 | github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= 17 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 18 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 19 | github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= 20 | github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= 21 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 23 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 24 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 25 | github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= 26 | github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 27 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= 28 | github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= 29 | github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= 30 | github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= 31 | github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= 32 | github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= 33 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 34 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 35 | github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= 36 | github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= 37 | github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= 38 | github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= 39 | github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= 40 | github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= 41 | github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= 42 | github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= 43 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 44 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 45 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 46 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 47 | github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= 48 | github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= 49 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 50 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 51 | github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= 52 | github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= 53 | github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= 54 | github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= 55 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 56 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 57 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 58 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 59 | github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= 60 | github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 61 | github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= 62 | github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= 63 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= 64 | github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= 65 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 66 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 67 | github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= 68 | github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 69 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= 70 | github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= 71 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 72 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 73 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 74 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 75 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 76 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 77 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 78 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 79 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 80 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 81 | github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= 82 | github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= 83 | github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= 84 | github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= 85 | github.com/keikoproj/aws-sdk-go-cache v0.1.0 h1:XEDzjrXFmnSnNuzqMW128Bs0PzjeR6WCyU30Fl3JBag= 86 | github.com/keikoproj/aws-sdk-go-cache v0.1.0/go.mod h1:Ne0kppOb8GyXT7RVvKtYJm6UjqaYruPt6woaFIuuQe8= 87 | github.com/keikoproj/inverse-exp-backoff v0.1.1 h1:S/PkLppg0GRwKe227nSOycv72+8Fgub8ORMguH7X0uc= 88 | github.com/keikoproj/inverse-exp-backoff v0.1.1/go.mod h1:99uGNpxpxKz4i8Y1K7Ppou1oI7q7mE8Fc8NpT3jnS4Y= 89 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 90 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 91 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 92 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 93 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 94 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 95 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 96 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 97 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 98 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 99 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 100 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 101 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 102 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= 103 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= 104 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 105 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 106 | github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= 107 | github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= 108 | github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= 109 | github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= 110 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 111 | github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= 112 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 113 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 114 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 115 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 116 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 117 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= 118 | github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= 119 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 120 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 121 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= 122 | github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= 123 | github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= 124 | github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= 125 | github.com/onsi/gomega v1.36.3 h1:hID7cr8t3Wp26+cYnfcjR6HpJ00fdogN6dqZ1t6IylU= 126 | github.com/onsi/gomega v1.36.3/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= 127 | github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= 128 | github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= 129 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 130 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 131 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 132 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 133 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 134 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 135 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 136 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 137 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 138 | github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= 139 | github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= 140 | github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= 141 | github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= 142 | github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= 143 | github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 144 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 145 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 146 | github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= 147 | github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 148 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 149 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 150 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 151 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 152 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 153 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 154 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 155 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 156 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 157 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 158 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 159 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 160 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 161 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 162 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 163 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 164 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 165 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 166 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 167 | github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= 168 | github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= 169 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 170 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 171 | github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= 172 | github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= 173 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 174 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 175 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 176 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 177 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 178 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 179 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 180 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 181 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 182 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 183 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 184 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 185 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 186 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 187 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 188 | golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= 189 | golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 190 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 191 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 192 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 193 | golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= 194 | golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 195 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 196 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 197 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 198 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 199 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 200 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 201 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 202 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 203 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 204 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 205 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 206 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 207 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 208 | golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= 209 | golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 210 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 211 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 212 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 213 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 214 | golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= 215 | golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= 216 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 217 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 218 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 219 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 220 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 221 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 222 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 223 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 224 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 225 | gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= 226 | gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= 227 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 228 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 229 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 230 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 231 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 232 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 233 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 234 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 235 | k8s.io/api v0.32.4 h1:kw8Y/G8E7EpNy7gjB8gJZl3KJkNz8HM2YHrZPtAZsF4= 236 | k8s.io/api v0.32.4/go.mod h1:5MYFvLvweRhyKylM3Es/6uh/5hGp0dg82vP34KifX4g= 237 | k8s.io/apimachinery v0.32.4 h1:8EEksaxA7nd7xWJkkwLDN4SvWS5ot9g6Z/VZb3ju25I= 238 | k8s.io/apimachinery v0.32.4/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= 239 | k8s.io/cli-runtime v0.32.4 h1:5O9eC50+yFFODAan3QXeTJHtoe69ie/A9vWBITIn+KM= 240 | k8s.io/cli-runtime v0.32.4/go.mod h1:Zn7nvBY625sEEYGtTMMPS619nrmVxabGssHyAKFK7RA= 241 | k8s.io/client-go v0.32.4 h1:zaGJS7xoYOYumoWIFXlcVrsiYioRPrXGO7dBfVC5R6M= 242 | k8s.io/client-go v0.32.4/go.mod h1:k0jftcyYnEtwlFW92xC7MTtFv5BNcZBr+zn9jPlT9Ic= 243 | k8s.io/component-base v0.32.4 h1:HuF+2JVLbFS5GODLIfPCb1Td6b+G2HszJoArcWOSr5I= 244 | k8s.io/component-base v0.32.4/go.mod h1:10KloJEYw1keU/Xmjfy9TKJqUq7J2mYdiD1VDXoco4o= 245 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 246 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 247 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= 248 | k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= 249 | k8s.io/kubectl v0.32.4 h1:PhiS4JgnJE6/JcksIJHAQI41F4hwJ270UopbcDsutF0= 250 | k8s.io/kubectl v0.32.4/go.mod h1:XYFDrC1l3cpjWRj7UfS0LDAMlK+6eTema28NeUYFKD8= 251 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= 252 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 253 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= 254 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= 255 | sigs.k8s.io/kustomize/api v0.18.0 h1:hTzp67k+3NEVInwz5BHyzc9rGxIauoXferXyjv5lWPo= 256 | sigs.k8s.io/kustomize/api v0.18.0/go.mod h1:f8isXnX+8b+SGLHQ6yO4JG1rdkZlvhaCf/uZbLVMb0U= 257 | sigs.k8s.io/kustomize/kyaml v0.18.1 h1:WvBo56Wzw3fjS+7vBjN6TeivvpbW9GmRaWZ9CIVmt4E= 258 | sigs.k8s.io/kustomize/kyaml v0.18.1/go.mod h1:C3L2BFVU1jgcddNBE1TxuVLgS46TjObMwW5FT9FcjYo= 259 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= 260 | sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= 261 | sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= 262 | sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= 263 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/keikoproj/lifecycle-manager/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /pkg/enroll/enroll.go: -------------------------------------------------------------------------------- 1 | package enroll 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws" 5 | "github.com/aws/aws-sdk-go/aws/awserr" 6 | "github.com/aws/aws-sdk-go/service/autoscaling" 7 | "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" 8 | "github.com/aws/aws-sdk-go/service/iam" 9 | "github.com/aws/aws-sdk-go/service/iam/iamiface" 10 | "github.com/aws/aws-sdk-go/service/sqs" 11 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 12 | "github.com/keikoproj/lifecycle-manager/pkg/log" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | const ( 17 | terminationTransitionName = "autoscaling:EC2_INSTANCE_TERMINATING" 18 | defaultHookName = "lifecycle-manager" 19 | notificationRoleDescription = `Role used by lifecycle-manager for sending hooks from autoscaling to SQS` 20 | notificationPolicyARN = "arn:aws:iam::aws:policy/service-role/AutoScalingNotificationAccessRole" 21 | ) 22 | 23 | // EnrollmentAuthenticator holds clients for the enroll command 24 | type EnrollmentAuthenticator struct { 25 | ScalingGroupClient autoscalingiface.AutoScalingAPI 26 | SQSClient sqsiface.SQSAPI 27 | IAMClient iamiface.IAMAPI 28 | } 29 | 30 | type EnrollmentContext struct { 31 | QueueName string 32 | Region string 33 | NotificationRoleName string 34 | TargetScalingGroups []string 35 | HeartbeatTimeout uint 36 | Authenticator EnrollmentAuthenticator 37 | QueueURL string 38 | QueueARN string 39 | RoleARN string 40 | Overwrite bool 41 | } 42 | 43 | type Worker struct { 44 | authenticator EnrollmentAuthenticator 45 | context *EnrollmentContext 46 | } 47 | 48 | func New(auth EnrollmentAuthenticator, ctx *EnrollmentContext) *Worker { 49 | return &Worker{ 50 | authenticator: auth, 51 | context: ctx, 52 | } 53 | } 54 | 55 | func (w *Worker) Start() { 56 | var ( 57 | ctx = w.context 58 | ) 59 | 60 | log.Infof("starting enrollment for scaling groups %+v", ctx.TargetScalingGroups) 61 | 62 | if err := w.CreateNotificationRole(); err != nil { 63 | log.Fatal(err) 64 | } 65 | 66 | if err := w.CreateSQSQueue(); err != nil { 67 | log.Fatal(err) 68 | } 69 | 70 | for _, scalingGroup := range ctx.TargetScalingGroups { 71 | if err := w.CreateLifecycleHook(scalingGroup); err != nil { 72 | log.Fatal(err) 73 | } 74 | } 75 | 76 | log.Infof("successfully enrolled %v scaling groups", len(ctx.TargetScalingGroups)) 77 | log.Infof("Queue Name: %v", ctx.QueueName) 78 | log.Infof("Queue URL: %v", ctx.QueueURL) 79 | } 80 | 81 | func (w *Worker) CreateLifecycleHook(scalingGroup string) error { 82 | var ( 83 | ASGClient = w.authenticator.ScalingGroupClient 84 | ctx = w.context 85 | ) 86 | 87 | crInput := &autoscaling.PutLifecycleHookInput{ 88 | AutoScalingGroupName: aws.String(scalingGroup), 89 | HeartbeatTimeout: aws.Int64(int64(ctx.HeartbeatTimeout)), 90 | LifecycleHookName: aws.String(defaultHookName), 91 | LifecycleTransition: aws.String(terminationTransitionName), 92 | NotificationTargetARN: aws.String(ctx.QueueARN), 93 | RoleARN: aws.String(ctx.RoleARN), 94 | } 95 | 96 | log.Infof("creating lifecycle hook for '%v'", scalingGroup) 97 | if _, err := ASGClient.PutLifecycleHook(crInput); err != nil { 98 | return errors.Errorf("failed to put lifecycle hook: %v", err) 99 | } 100 | return nil 101 | } 102 | 103 | func (w *Worker) CreateSQSQueue() error { 104 | var ( 105 | SQSClient = w.authenticator.SQSClient 106 | ctx = w.context 107 | ) 108 | 109 | crInput := &sqs.CreateQueueInput{ 110 | QueueName: aws.String(ctx.QueueName), 111 | } 112 | 113 | log.Infof("creating SQS queue '%v'", ctx.QueueName) 114 | out, err := SQSClient.CreateQueue(crInput) 115 | if err != nil { 116 | return errors.Errorf("failed to create SQS queue: %v", err) 117 | } 118 | 119 | ctx.QueueURL = aws.StringValue(out.QueueUrl) 120 | 121 | desInput := &sqs.GetQueueAttributesInput{ 122 | QueueUrl: out.QueueUrl, 123 | AttributeNames: aws.StringSlice([]string{"QueueArn"}), 124 | } 125 | 126 | attr, err := SQSClient.GetQueueAttributes(desInput) 127 | if err != nil { 128 | return errors.Errorf("failed get queue attribute: %v", err) 129 | } 130 | 131 | ctx.QueueARN = aws.StringValue(attr.Attributes["QueueArn"]) 132 | log.Infof("created queue '%v'", ctx.QueueARN) 133 | return nil 134 | } 135 | 136 | func (w *Worker) CreateNotificationRole() error { 137 | var ( 138 | IAMClient = w.authenticator.IAMClient 139 | ctx = w.context 140 | alreadyExist bool 141 | ) 142 | 143 | log.Infof("creating notification role '%v'", ctx.NotificationRoleName) 144 | crInput := &iam.CreateRoleInput{ 145 | AssumeRolePolicyDocument: getAssumeRolePolicyDocument(), 146 | Description: aws.String(notificationRoleDescription), 147 | RoleName: aws.String(ctx.NotificationRoleName), 148 | } 149 | 150 | out, err := IAMClient.CreateRole(crInput) 151 | if err != nil { 152 | if awsErr, ok := err.(awserr.Error); ok { 153 | switch awsErr.Code() { 154 | case iam.ErrCodeEntityAlreadyExistsException: 155 | alreadyExist = true 156 | default: 157 | return errors.Errorf("failed to create notification role: %v", awsErr.Message()) 158 | } 159 | } 160 | } 161 | 162 | if alreadyExist { 163 | if !ctx.Overwrite { 164 | log.Warn("set flag --overwrite for re-using existing role") 165 | return errors.Errorf("failed to create notification role: %v", err) 166 | } 167 | log.Infof("notification role '%v' already exist, updating...", ctx.NotificationRoleName) 168 | input := &iam.GetRoleInput{ 169 | RoleName: aws.String(ctx.NotificationRoleName), 170 | } 171 | out, err := IAMClient.GetRole(input) 172 | if err != nil { 173 | return errors.Errorf("failed to get exisinting notification role: %v", err) 174 | } 175 | ctx.RoleARN = aws.StringValue(out.Role.Arn) 176 | } else { 177 | ctx.RoleARN = aws.StringValue(out.Role.Arn) 178 | } 179 | 180 | atInput := &iam.AttachRolePolicyInput{ 181 | PolicyArn: aws.String(notificationPolicyARN), 182 | RoleName: aws.String(ctx.NotificationRoleName), 183 | } 184 | 185 | log.Infof("attaching notification policy '%v'", notificationPolicyARN) 186 | if _, err := IAMClient.AttachRolePolicy(atInput); err != nil { 187 | return errors.Errorf("failed to attach notification role: %v", err) 188 | } 189 | 190 | log.Infof("created notification role '%v'", ctx.RoleARN) 191 | 192 | return nil 193 | } 194 | 195 | func getAssumeRolePolicyDocument() *string { 196 | doc := `{ 197 | "Version": "2012-10-17", 198 | "Statement": [ 199 | { 200 | "Effect": "Allow", 201 | "Principal": { 202 | "Service": "autoscaling.amazonaws.com" 203 | }, 204 | "Action": "sts:AssumeRole" 205 | } 206 | ] 207 | }` 208 | return aws.String(doc) 209 | } 210 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | ) 6 | 7 | // Logger defines a set of methods for writing application logs. 8 | type Logger interface { 9 | Debug(args ...interface{}) 10 | Debugf(format string, args ...interface{}) 11 | Debugln(args ...interface{}) 12 | Error(args ...interface{}) 13 | Errorf(format string, args ...interface{}) 14 | Errorln(args ...interface{}) 15 | Fatal(args ...interface{}) 16 | Fatalf(format string, args ...interface{}) 17 | Fatalln(args ...interface{}) 18 | Info(args ...interface{}) 19 | Infof(format string, args ...interface{}) 20 | Infoln(args ...interface{}) 21 | Panic(args ...interface{}) 22 | Panicf(format string, args ...interface{}) 23 | Panicln(args ...interface{}) 24 | Print(args ...interface{}) 25 | Printf(format string, args ...interface{}) 26 | Println(args ...interface{}) 27 | Warn(args ...interface{}) 28 | Warnf(format string, args ...interface{}) 29 | Warning(args ...interface{}) 30 | Warningf(format string, args ...interface{}) 31 | Warningln(args ...interface{}) 32 | Warnln(args ...interface{}) 33 | } 34 | 35 | var defaultLogger *logrus.Logger 36 | 37 | func init() { 38 | defaultLogger = newLogrusLogger() 39 | } 40 | 41 | func NewLogger() *logrus.Logger { 42 | return newLogrusLogger() 43 | } 44 | 45 | func newLogrusLogger() *logrus.Logger { 46 | l := logrus.New() 47 | l.Level = logrus.InfoLevel 48 | return l 49 | } 50 | 51 | func SetLevel(logLevel string) { 52 | switch logLevel { 53 | case "debug": 54 | defaultLogger.Level = logrus.DebugLevel 55 | case "warning": 56 | defaultLogger.Level = logrus.WarnLevel 57 | case "info": 58 | defaultLogger.Level = logrus.InfoLevel 59 | default: 60 | defaultLogger.Level = logrus.InfoLevel 61 | } 62 | } 63 | 64 | type Fields map[string]interface{} 65 | 66 | func (f Fields) With(k string, v interface{}) Fields { 67 | f[k] = v 68 | return f 69 | } 70 | 71 | func (f Fields) WithFields(f2 Fields) Fields { 72 | for k, v := range f2 { 73 | f[k] = v 74 | } 75 | return f 76 | } 77 | 78 | func WithFields(fields Fields) Logger { 79 | return defaultLogger.WithFields(logrus.Fields(fields)) 80 | } 81 | 82 | // Debug package-level convenience method. 83 | func Debug(args ...interface{}) { 84 | defaultLogger.Debug(args...) 85 | } 86 | 87 | // Debugf package-level convenience method. 88 | func Debugf(format string, args ...interface{}) { 89 | defaultLogger.Debugf(format, args...) 90 | } 91 | 92 | // Debugln package-level convenience method. 93 | func Debugln(args ...interface{}) { 94 | defaultLogger.Debugln(args...) 95 | } 96 | 97 | // Error package-level convenience method. 98 | func Error(args ...interface{}) { 99 | defaultLogger.Error(args...) 100 | } 101 | 102 | // Errorf package-level convenience method. 103 | func Errorf(format string, args ...interface{}) { 104 | defaultLogger.Errorf(format, args...) 105 | } 106 | 107 | // Errorln package-level convenience method. 108 | func Errorln(args ...interface{}) { 109 | defaultLogger.Errorln(args...) 110 | } 111 | 112 | // Fatal package-level convenience method. 113 | func Fatal(args ...interface{}) { 114 | defaultLogger.Fatal(args...) 115 | } 116 | 117 | // Fatalf package-level convenience method. 118 | func Fatalf(format string, args ...interface{}) { 119 | defaultLogger.Fatalf(format, args...) 120 | } 121 | 122 | // Fatalln package-level convenience method. 123 | func Fatalln(args ...interface{}) { 124 | defaultLogger.Fatalln(args...) 125 | } 126 | 127 | // Info package-level convenience method. 128 | func Info(args ...interface{}) { 129 | defaultLogger.Info(args...) 130 | } 131 | 132 | // Infof package-level convenience method. 133 | func Infof(format string, args ...interface{}) { 134 | defaultLogger.Infof(format, args...) 135 | } 136 | 137 | // Infoln package-level convenience method. 138 | func Infoln(args ...interface{}) { 139 | defaultLogger.Infoln(args...) 140 | } 141 | 142 | // Panic package-level convenience method. 143 | func Panic(args ...interface{}) { 144 | defaultLogger.Panic(args...) 145 | } 146 | 147 | // Panicf package-level convenience method. 148 | func Panicf(format string, args ...interface{}) { 149 | defaultLogger.Panicf(format, args...) 150 | } 151 | 152 | // Panicln package-level convenience method. 153 | func Panicln(args ...interface{}) { 154 | defaultLogger.Panicln(args...) 155 | } 156 | 157 | // Print package-level convenience method. 158 | func Print(args ...interface{}) { 159 | defaultLogger.Print(args...) 160 | } 161 | 162 | // Printf package-level convenience method. 163 | func Printf(format string, args ...interface{}) { 164 | defaultLogger.Printf(format, args...) 165 | } 166 | 167 | // Println package-level convenience method. 168 | func Println(args ...interface{}) { 169 | defaultLogger.Println(args...) 170 | } 171 | 172 | // Warn package-level convenience method. 173 | func Warn(args ...interface{}) { 174 | defaultLogger.Warn(args...) 175 | } 176 | 177 | // Warnf package-level convenience method. 178 | func Warnf(format string, args ...interface{}) { 179 | defaultLogger.Warnf(format, args...) 180 | } 181 | 182 | // Warning package-level convenience method. 183 | func Warning(args ...interface{}) { 184 | defaultLogger.Warning(args...) 185 | } 186 | 187 | // Warningf package-level convenience method. 188 | func Warningf(format string, args ...interface{}) { 189 | defaultLogger.Warningf(format, args...) 190 | } 191 | 192 | // Warningln package-level convenience method. 193 | func Warningln(args ...interface{}) { 194 | defaultLogger.Warningln(args...) 195 | } 196 | 197 | // Warnln package-level convenience method. 198 | func Warnln(args ...interface{}) { 199 | defaultLogger.Warnln(args...) 200 | } 201 | -------------------------------------------------------------------------------- /pkg/log/retry_logger.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/aws/aws-sdk-go/aws/client" 8 | "github.com/aws/aws-sdk-go/aws/request" 9 | ) 10 | 11 | type RetryLogger struct { 12 | client.DefaultRetryer 13 | } 14 | 15 | var _ request.Retryer = &RetryLogger{} 16 | 17 | func NewRetryLogger(retryer client.DefaultRetryer) *RetryLogger { 18 | return &RetryLogger{ 19 | retryer, 20 | } 21 | } 22 | 23 | func (l RetryLogger) RetryRules(r *request.Request) time.Duration { 24 | var ( 25 | duration = l.DefaultRetryer.RetryRules(r) 26 | service = r.ClientInfo.ServiceName 27 | name string 28 | err string 29 | ) 30 | 31 | if r.Operation != nil { 32 | name = r.Operation.Name 33 | } 34 | method := fmt.Sprintf("%v/%v", service, name) 35 | 36 | if r.Error != nil { 37 | err = fmt.Sprintf("%v", r.Error) 38 | } else { 39 | err = fmt.Sprintf("%d %s", r.HTTPResponse.StatusCode, r.HTTPResponse.Status) 40 | } 41 | Debugf("retryable: %v -- %v, will retry after %v", err, method, duration) 42 | 43 | return duration 44 | } 45 | -------------------------------------------------------------------------------- /pkg/service/autoscaling.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/autoscaling" 9 | "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" 10 | "github.com/keikoproj/lifecycle-manager/pkg/log" 11 | ) 12 | 13 | func sendHeartbeat(client autoscalingiface.AutoScalingAPI, event *LifecycleEvent, maxTimeToProcessSeconds int64) { 14 | var ( 15 | iterationCount = 0 16 | interval = event.heartbeatInterval 17 | instanceID = event.EC2InstanceID 18 | scalingGroupName = event.AutoScalingGroupName 19 | recommendedInterval = interval / 2 20 | ) 21 | 22 | log.Debugf("scaling-group = %v, maxInterval = %v, heartbeat = %v", scalingGroupName, interval, recommendedInterval) 23 | 24 | // max time to process an event is capped at 1hr 25 | maxIterations := int(maxTimeToProcessSeconds / recommendedInterval) 26 | 27 | for { 28 | iterationCount++ 29 | if iterationCount >= maxIterations { 30 | // hard limit in case event is not marked completed 31 | log.Warnf("%v> heartbeat extended over threshold, instance will be abandoned", instanceID) 32 | event.SetEventCompleted(true) 33 | } 34 | 35 | if event.eventCompleted { 36 | return 37 | } 38 | 39 | log.Infof("%v> sending heartbeat (%v/%v)", instanceID, iterationCount, maxIterations) 40 | err := extendLifecycleAction(client, *event) 41 | if err != nil { 42 | log.Errorf("%v> failed to send heartbeat for event: %v", instanceID, err) 43 | return 44 | } 45 | time.Sleep(time.Duration(recommendedInterval) * time.Second) 46 | } 47 | } 48 | 49 | func getHookHeartbeatInterval(client autoscalingiface.AutoScalingAPI, lifecycleHookName, scalingGroupName string) (int64, error) { 50 | input := &autoscaling.DescribeLifecycleHooksInput{ 51 | AutoScalingGroupName: aws.String(scalingGroupName), 52 | LifecycleHookNames: aws.StringSlice([]string{lifecycleHookName}), 53 | } 54 | out, err := client.DescribeLifecycleHooks(input) 55 | if err != nil { 56 | return 0, err 57 | } 58 | 59 | if len(out.LifecycleHooks) == 0 { 60 | err = fmt.Errorf("could not find lifecycle hook with name %v for scaling group %v", lifecycleHookName, scalingGroupName) 61 | return 0, err 62 | } 63 | 64 | return aws.Int64Value(out.LifecycleHooks[0].HeartbeatTimeout), nil 65 | } 66 | 67 | func completeLifecycleAction(client autoscalingiface.AutoScalingAPI, event LifecycleEvent, result string) error { 68 | log.Infof("%v> setting lifecycle event as completed with result: %v", event.EC2InstanceID, result) 69 | input := &autoscaling.CompleteLifecycleActionInput{ 70 | AutoScalingGroupName: aws.String(event.AutoScalingGroupName), 71 | InstanceId: aws.String(event.EC2InstanceID), 72 | LifecycleActionResult: aws.String(result), 73 | LifecycleHookName: aws.String(event.LifecycleHookName), 74 | } 75 | _, err := client.CompleteLifecycleAction(input) 76 | if err != nil { 77 | return err 78 | } 79 | return nil 80 | } 81 | 82 | func extendLifecycleAction(client autoscalingiface.AutoScalingAPI, event LifecycleEvent) error { 83 | log.Debugf("%v> extending lifecycle event", event.EC2InstanceID) 84 | input := &autoscaling.RecordLifecycleActionHeartbeatInput{ 85 | AutoScalingGroupName: aws.String(event.AutoScalingGroupName), 86 | InstanceId: aws.String(event.EC2InstanceID), 87 | LifecycleActionToken: aws.String(event.LifecycleActionToken), 88 | LifecycleHookName: aws.String(event.LifecycleHookName), 89 | } 90 | _, err := client.RecordLifecycleActionHeartbeat(input) 91 | if err != nil { 92 | return err 93 | } 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/service/autoscaling_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/service/autoscaling" 9 | "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" 10 | ) 11 | 12 | type stubAutoscaling struct { 13 | autoscalingiface.AutoScalingAPI 14 | lifecycleHooks []*autoscaling.LifecycleHook 15 | timesCalledDescribeLifecycleHooks int 16 | timesCalledRecordLifecycleActionHeartbeat int 17 | timesCalledCompleteLifecycleAction int 18 | } 19 | 20 | func (a *stubAutoscaling) DescribeLifecycleHooks(input *autoscaling.DescribeLifecycleHooksInput) (*autoscaling.DescribeLifecycleHooksOutput, error) { 21 | a.timesCalledDescribeLifecycleHooks++ 22 | return &autoscaling.DescribeLifecycleHooksOutput{LifecycleHooks: a.lifecycleHooks}, nil 23 | } 24 | 25 | func (a *stubAutoscaling) RecordLifecycleActionHeartbeat(input *autoscaling.RecordLifecycleActionHeartbeatInput) (*autoscaling.RecordLifecycleActionHeartbeatOutput, error) { 26 | a.timesCalledRecordLifecycleActionHeartbeat++ 27 | return &autoscaling.RecordLifecycleActionHeartbeatOutput{}, nil 28 | } 29 | 30 | func (a *stubAutoscaling) CompleteLifecycleAction(input *autoscaling.CompleteLifecycleActionInput) (*autoscaling.CompleteLifecycleActionOutput, error) { 31 | a.timesCalledCompleteLifecycleAction++ 32 | return &autoscaling.CompleteLifecycleActionOutput{}, nil 33 | } 34 | 35 | func (e *LifecycleEvent) _setEventCompletedAfter(value bool, seconds int64) { 36 | time.Sleep(time.Duration(seconds)*time.Second + time.Duration(500)*time.Millisecond) 37 | e.eventCompleted = value 38 | } 39 | 40 | func Test_SendHeartbeatPositive(t *testing.T) { 41 | t.Log("Test_SendHeartbeatPositive: If drain is not complete, a heartbeat should be sent") 42 | stubber := &stubAutoscaling{} 43 | event := &LifecycleEvent{ 44 | AutoScalingGroupName: "my-asg", 45 | EC2InstanceID: "i-1234567890", 46 | LifecycleActionToken: "some-token-1234", 47 | LifecycleHookName: "my-hook", 48 | drainCompleted: false, 49 | heartbeatInterval: 3, 50 | } 51 | maxTimeToProcessSeconds := int64(3600) 52 | 53 | go event._setEventCompletedAfter(true, 2) 54 | sendHeartbeat(stubber, event, maxTimeToProcessSeconds) 55 | expectedHeartbeatCalls := 3 56 | 57 | if stubber.timesCalledRecordLifecycleActionHeartbeat != expectedHeartbeatCalls { 58 | t.Fatalf("expected timesCalledRecordLifecycleActionHeartbeat: %v, got: %v", expectedHeartbeatCalls, stubber.timesCalledRecordLifecycleActionHeartbeat) 59 | } 60 | } 61 | 62 | func Test_SendHeartbeatNegative(t *testing.T) { 63 | t.Log("Test_SendHeartbeatNegative: If event is completed, heartbeat should not be sent") 64 | stubber := &stubAutoscaling{} 65 | event := &LifecycleEvent{ 66 | AutoScalingGroupName: "my-asg", 67 | EC2InstanceID: "i-1234567890", 68 | LifecycleActionToken: "some-token-1234", 69 | LifecycleHookName: "my-hook", 70 | eventCompleted: true, 71 | heartbeatInterval: 3, 72 | } 73 | maxTimeToProcessSeconds := int64(3600) 74 | 75 | sendHeartbeat(stubber, event, maxTimeToProcessSeconds) 76 | expectedHeartbeatCalls := 0 77 | 78 | if stubber.timesCalledRecordLifecycleActionHeartbeat != expectedHeartbeatCalls { 79 | t.Fatalf("expected timesCalledRecordLifecycleActionHeartbeat: %v, got: %v", expectedHeartbeatCalls, stubber.timesCalledRecordLifecycleActionHeartbeat) 80 | } 81 | } 82 | 83 | func Test_CompleteLifecycleAction(t *testing.T) { 84 | t.Log("Test_CompleteLifecycleAction: should be able to complete a lifecycle action") 85 | stubber := &stubAutoscaling{} 86 | event := &LifecycleEvent{ 87 | AutoScalingGroupName: "my-asg", 88 | EC2InstanceID: "i-1234567890", 89 | LifecycleActionToken: "some-token-1234", 90 | LifecycleHookName: "my-hook", 91 | drainCompleted: true, 92 | } 93 | 94 | completeLifecycleAction(stubber, *event, ContinueAction) 95 | completeLifecycleAction(stubber, *event, AbandonAction) 96 | expectedCalls := 2 97 | 98 | if stubber.timesCalledCompleteLifecycleAction != expectedCalls { 99 | t.Fatalf("expected timesCalledRecordLifecycleActionHeartbeat: %v, got: %v", expectedCalls, stubber.timesCalledCompleteLifecycleAction) 100 | } 101 | } 102 | 103 | func Test_GetHookHeartbeatIntervalPositive(t *testing.T) { 104 | t.Log("Test_GetHookHeartbeatIntervalPositive: should be able get a lifecycle hook's heartbeat timeout interval if it exists") 105 | stubber := &stubAutoscaling{ 106 | lifecycleHooks: []*autoscaling.LifecycleHook{ 107 | { 108 | AutoScalingGroupName: aws.String("my-asg"), 109 | HeartbeatTimeout: aws.Int64(60), 110 | }, 111 | }, 112 | } 113 | event := &LifecycleEvent{ 114 | AutoScalingGroupName: "my-asg", 115 | EC2InstanceID: "i-1234567890", 116 | LifecycleActionToken: "some-token-1234", 117 | LifecycleHookName: "my-hook", 118 | drainCompleted: true, 119 | } 120 | 121 | interval, err := getHookHeartbeatInterval(stubber, event.AutoScalingGroupName, event.LifecycleHookName) 122 | if err != nil { 123 | t.Fatalf("getHookHeartbeatInterval: expected error not to have occured, %v", err) 124 | } 125 | 126 | expectedCalls := 1 127 | expectedInterval := int64(60) 128 | 129 | if stubber.timesCalledDescribeLifecycleHooks != expectedCalls { 130 | t.Fatalf("expected timesCalledDescribeLifecycleHooks: %v, got: %v", expectedCalls, stubber.timesCalledDescribeLifecycleHooks) 131 | } 132 | 133 | if interval != expectedInterval { 134 | t.Fatalf("expected interval: %v, got: %v", expectedInterval, interval) 135 | } 136 | } 137 | 138 | func Test_GetHookHeartbeatIntervalNegative(t *testing.T) { 139 | t.Log("Test_GetHookHeartbeatIntervalNegative: should not be able get a lifecycle hook's heartbeat timeout interval if it does not exists") 140 | stubber := &stubAutoscaling{ 141 | lifecycleHooks: []*autoscaling.LifecycleHook{}, 142 | } 143 | event := &LifecycleEvent{ 144 | AutoScalingGroupName: "my-asg", 145 | EC2InstanceID: "i-1234567890", 146 | LifecycleActionToken: "some-token-1234", 147 | LifecycleHookName: "my-hook", 148 | drainCompleted: true, 149 | } 150 | 151 | interval, err := getHookHeartbeatInterval(stubber, event.AutoScalingGroupName, event.LifecycleHookName) 152 | if err == nil { 153 | t.Fatal("getHookHeartbeatInterval: expected error to have occured") 154 | } 155 | 156 | expectedCalls := 1 157 | expectedInterval := int64(0) 158 | 159 | if stubber.timesCalledDescribeLifecycleHooks != expectedCalls { 160 | t.Fatalf("expected timesCalledDescribeLifecycleHooks: %v, got: %v", expectedCalls, stubber.timesCalledDescribeLifecycleHooks) 161 | } 162 | 163 | if interval != expectedInterval { 164 | t.Fatalf("expected interval: %v, got: %v", expectedInterval, interval) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /pkg/service/deregistrator.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "github.com/aws/aws-sdk-go/aws/awserr" 5 | "github.com/aws/aws-sdk-go/service/elb" 6 | "github.com/aws/aws-sdk-go/service/elbv2" 7 | "github.com/keikoproj/lifecycle-manager/pkg/log" 8 | ) 9 | 10 | type Deregistrator struct { 11 | targetDeregisteredCount int 12 | classicDeregisteredCount int 13 | errors chan DeregistrationError 14 | } 15 | 16 | func (d *Deregistrator) AddClassicDeregistration(val int) { d.classicDeregisteredCount += val } 17 | func (d *Deregistrator) AddTargetGroupDeregistration(val int) { d.targetDeregisteredCount += val } 18 | 19 | type DeregistrationError struct { 20 | Error error 21 | Target string 22 | Instances []string 23 | Type TargetType 24 | } 25 | 26 | func (mgr *Manager) startDeregistrator(d *Deregistrator) { 27 | mgr.deregistrationMu.Lock() 28 | defer mgr.deregistrationMu.Unlock() 29 | 30 | mgr.targets.Range(func(k, v interface{}) bool { 31 | var ( 32 | targets = v.([]*Target) 33 | ) 34 | if len(targets) == 0 { 35 | return true 36 | } 37 | waitJitter(IterationJitterRangeSeconds) 38 | mgr.DeregisterTargets(targets, d) 39 | return true 40 | }) 41 | if d.targetDeregisteredCount == 0 && d.classicDeregisteredCount == 0 { 42 | log.Infof("deregistrator> no active targets for deregistration") 43 | } else { 44 | log.Infof("deregistrator> deregistered %v instances from target groups and %v instances from classic-elbs", d.targetDeregisteredCount, d.classicDeregisteredCount) 45 | } 46 | } 47 | 48 | func (m *Manager) DeregisterTargets(targets []*Target, d *Deregistrator) { 49 | var ( 50 | elbClient = m.authenticator.ELBClient 51 | elbv2Client = m.authenticator.ELBv2Client 52 | ) 53 | 54 | for _, target := range targets { 55 | mapping := m.GetTargetMapping(target.TargetId) 56 | instances := m.GetTargetInstanceIds(target.TargetId) 57 | if len(instances) == 0 || len(mapping) == 0 { 58 | continue 59 | } 60 | 61 | switch target.Type { 62 | case TargetTypeClassicELB: 63 | log.Infof("deregistrator> deregistering %+v from %v", instances, target.TargetId) 64 | err := deregisterInstances(elbClient, target.TargetId, instances) 65 | if err != nil { 66 | if awsErr, ok := err.(awserr.Error); ok { 67 | if awsErr.Code() == elb.ErrCodeAccessPointNotFoundException { 68 | log.Warnf("deregistrator> ELB %v not found, skipping", target.TargetId) 69 | continue 70 | } else if awsErr.Code() == elb.ErrCodeInvalidEndPointException { 71 | log.Warnf("deregistrator> ELB targets %v not found in %v, skipping", instances, target.TargetId) 72 | continue 73 | } 74 | } 75 | log.Errorf("deregistrator> target deregistration failed: %v", err) 76 | deregistrationErr := DeregistrationError{ 77 | Error: err, 78 | Target: target.TargetId, 79 | Instances: instances, 80 | Type: TargetTypeClassicELB, 81 | } 82 | d.errors <- deregistrationErr 83 | } 84 | d.AddClassicDeregistration(len(instances)) 85 | for _, instance := range instances { 86 | m.RemoveTargetByInstance(target.TargetId, instance) 87 | } 88 | case TargetTypeTargetGroup: 89 | log.Infof("deregistrator> deregistering %+v from %v", instances, target.TargetId) 90 | err := deregisterTargets(elbv2Client, target.TargetId, mapping) 91 | if err != nil { 92 | if awsErr, ok := err.(awserr.Error); ok { 93 | if awsErr.Code() == elbv2.ErrCodeTargetGroupNotFoundException { 94 | log.Warnf("deregistrator> target group %v not found, skipping", target.TargetId) 95 | continue 96 | } else if awsErr.Code() == elbv2.ErrCodeInvalidTargetException { 97 | log.Warnf("deregistrator> target %v not found in target group %v, skipping", instances, target.TargetId) 98 | continue 99 | } 100 | } 101 | log.Errorf("deregistrator> target deregistration failed: %v", err) 102 | deregistrationErr := DeregistrationError{ 103 | Error: err, 104 | Target: target.TargetId, 105 | Instances: instances, 106 | Type: TargetTypeTargetGroup, 107 | } 108 | d.errors <- deregistrationErr 109 | } 110 | d.AddTargetGroupDeregistration(len(instances)) 111 | for _, instance := range instances { 112 | m.RemoveTargetByInstance(target.TargetId, instance) 113 | } 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /pkg/service/elb.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/elb" 8 | "github.com/aws/aws-sdk-go/service/elb/elbiface" 9 | iebackoff "github.com/keikoproj/inverse-exp-backoff" 10 | 11 | "github.com/keikoproj/lifecycle-manager/pkg/log" 12 | ) 13 | 14 | func waitForDeregisterInstance(event *LifecycleEvent, elbClient elbiface.ELBAPI, elbName, instanceID string) error { 15 | var ( 16 | found bool 17 | ) 18 | 19 | input := &elb.DescribeInstanceHealthInput{ 20 | LoadBalancerName: aws.String(elbName), 21 | } 22 | 23 | for ieb, err := iebackoff.NewIEBackoff(WaiterMaxDelay, WaiterMinDelay, 0.5, WaiterMaxAttempts); err == nil; err = ieb.Next() { 24 | 25 | if event.eventCompleted { 26 | return errors.New("event finished execution during deregistration wait") 27 | } 28 | 29 | found = false 30 | instances, err := elbClient.DescribeInstanceHealth(input) 31 | if err != nil { 32 | return err 33 | } 34 | for _, state := range instances.InstanceStates { 35 | if aws.StringValue(state.InstanceId) == instanceID { 36 | found = true 37 | if aws.StringValue(state.State) == "OutOfService" { 38 | return nil 39 | } 40 | break 41 | } 42 | } 43 | if !found { 44 | log.Debugf("%v> instance not found in elb %v", instanceID, elbName) 45 | return nil 46 | } 47 | log.Debugf("%v> deregistration from %v pending", instanceID, elbName) 48 | } 49 | 50 | err := errors.New("wait for target deregister timed out") 51 | return err 52 | } 53 | 54 | func findInstanceInClassicBalancer(elbClient elbiface.ELBAPI, elbName, instanceID string) (bool, error) { 55 | input := &elb.DescribeInstanceHealthInput{ 56 | LoadBalancerName: aws.String(elbName), 57 | } 58 | 59 | instance, err := elbClient.DescribeInstanceHealth(input) 60 | if err != nil { 61 | log.Errorf("%v> failed finding instance in elb %v: %v", instanceID, elbName, err.Error()) 62 | return false, err 63 | } 64 | for _, state := range instance.InstanceStates { 65 | if aws.StringValue(state.InstanceId) == instanceID { 66 | return true, nil 67 | } 68 | } 69 | return false, nil 70 | } 71 | 72 | func deregisterInstances(elbClient elbiface.ELBAPI, elbName string, instances []string) error { 73 | targets := []*elb.Instance{} 74 | for _, instance := range instances { 75 | target := &elb.Instance{ 76 | InstanceId: aws.String(instance), 77 | } 78 | targets = append(targets, target) 79 | } 80 | 81 | input := &elb.DeregisterInstancesFromLoadBalancerInput{ 82 | LoadBalancerName: aws.String(elbName), 83 | Instances: targets, 84 | } 85 | 86 | _, err := elbClient.DeregisterInstancesFromLoadBalancer(input) 87 | if err != nil { 88 | return err 89 | } 90 | return nil 91 | } 92 | -------------------------------------------------------------------------------- /pkg/service/elb_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/awserr" 10 | "github.com/aws/aws-sdk-go/service/elb" 11 | "github.com/aws/aws-sdk-go/service/elb/elbiface" 12 | ) 13 | 14 | type stubELB struct { 15 | elbiface.ELBAPI 16 | instanceStates []*elb.InstanceState 17 | loadBalancerDescriptions []*elb.LoadBalancerDescription 18 | timesCalledDescribeInstanceHealth int 19 | timesCalledDeregisterInstances int 20 | timesCalledDescribeLoadBalancers int 21 | } 22 | 23 | func (e *stubELB) DescribeInstanceHealth(input *elb.DescribeInstanceHealthInput) (*elb.DescribeInstanceHealthOutput, error) { 24 | e.timesCalledDescribeInstanceHealth++ 25 | return &elb.DescribeInstanceHealthOutput{InstanceStates: e.instanceStates}, nil 26 | } 27 | 28 | func (e *stubELB) DeregisterInstancesFromLoadBalancer(input *elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error) { 29 | e.timesCalledDeregisterInstances++ 30 | return &elb.DeregisterInstancesFromLoadBalancerOutput{}, nil 31 | } 32 | 33 | func (e *stubELB) DescribeLoadBalancers(input *elb.DescribeLoadBalancersInput) (*elb.DescribeLoadBalancersOutput, error) { 34 | e.timesCalledDescribeLoadBalancers++ 35 | return &elb.DescribeLoadBalancersOutput{LoadBalancerDescriptions: e.loadBalancerDescriptions}, nil 36 | } 37 | 38 | func (e *stubELB) DescribeLoadBalancersPages(input *elb.DescribeLoadBalancersInput, callback func(*elb.DescribeLoadBalancersOutput, bool) bool) error { 39 | page, err := e.DescribeLoadBalancers(input) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | callback(page, false) 45 | 46 | return nil 47 | } 48 | 49 | type stubErrorELB struct { 50 | elbiface.ELBAPI 51 | instanceStates []*elb.InstanceState 52 | loadBalancerDescriptions []*elb.LoadBalancerDescription 53 | timesCalledDescribeInstanceHealth int 54 | timesCalledDeregisterInstances int 55 | timesCalledDescribeLoadBalancers int 56 | failHint string 57 | } 58 | 59 | func (e *stubErrorELB) DescribeInstanceHealth(input *elb.DescribeInstanceHealthInput) (*elb.DescribeInstanceHealthOutput, error) { 60 | e.timesCalledDescribeInstanceHealth++ 61 | var err error 62 | if e.failHint == elb.ErrCodeAccessPointNotFoundException { 63 | err = awserr.New(elb.ErrCodeAccessPointNotFoundException, "failed", fmt.Errorf("it failed")) 64 | } else { 65 | err = fmt.Errorf("some error, DeregisterTargets") 66 | } 67 | return &elb.DescribeInstanceHealthOutput{}, err 68 | } 69 | 70 | func (e *stubErrorELB) DeregisterInstancesFromLoadBalancer(input *elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error) { 71 | e.timesCalledDeregisterInstances++ 72 | var err error 73 | if e.failHint == elb.ErrCodeAccessPointNotFoundException { 74 | err = awserr.New(elb.ErrCodeAccessPointNotFoundException, "failed", fmt.Errorf("it failed")) 75 | } else if e.failHint == elb.ErrCodeInvalidEndPointException { 76 | err = awserr.New(elb.ErrCodeInvalidEndPointException, "failed", fmt.Errorf("it failed")) 77 | } else { 78 | err = fmt.Errorf("some other error occured") 79 | } 80 | return &elb.DeregisterInstancesFromLoadBalancerOutput{}, err 81 | } 82 | 83 | func (e *stubErrorELB) DescribeLoadBalancers(input *elb.DescribeLoadBalancersInput) (*elb.DescribeLoadBalancersOutput, error) { 84 | e.timesCalledDescribeLoadBalancers++ 85 | return &elb.DescribeLoadBalancersOutput{LoadBalancerDescriptions: e.loadBalancerDescriptions}, nil 86 | } 87 | 88 | func (e *stubErrorELB) DescribeLoadBalancersPages(input *elb.DescribeLoadBalancersInput, callback func(*elb.DescribeLoadBalancersOutput, bool) bool) error { 89 | page, err := e.DescribeLoadBalancers(input) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | callback(page, false) 95 | 96 | return nil 97 | } 98 | 99 | func Test_DeregisterInstance(t *testing.T) { 100 | t.Log("Test_DeregisterInstance: should be able to deregister an instance from classic elb") 101 | var ( 102 | stubber = &stubELB{} 103 | elbName = "some-load-balancer" 104 | instances = []string{"i-1234567890"} 105 | expectedCalls = 1 106 | ) 107 | 108 | err := deregisterInstances(stubber, elbName, instances) 109 | if err != nil { 110 | t.Fatalf("Test_DeregisterInstance: expected error not to have occured, %v", err) 111 | } 112 | 113 | if stubber.timesCalledDeregisterInstances != expectedCalls { 114 | t.Fatalf("Test_DeregisterInstance: expected timesCalledDeregisterInstances: %v, got: %v", expectedCalls, stubber.timesCalledDeregisterInstances) 115 | } 116 | } 117 | 118 | func Test_DeregisterNotFoundException(t *testing.T) { 119 | t.Log("Test_DeregisterError: should return an error when call fails") 120 | var ( 121 | stubber = &stubErrorELB{} 122 | elbName = "some-load-balancer" 123 | instances = []string{"i-1234567890"} 124 | expectedCalls = 1 125 | ) 126 | 127 | stubber.failHint = elb.ErrCodeAccessPointNotFoundException 128 | err := deregisterInstances(stubber, elbName, instances) 129 | if err == nil { 130 | t.Fatalf("Test_DeregisterInstance: expected error to have occured, got: %v", err) 131 | } 132 | 133 | if stubber.timesCalledDeregisterInstances != expectedCalls { 134 | t.Fatalf("Test_DeregisterInstance: expected timesCalledDeregisterInstances: %v, got: %v", expectedCalls, stubber.timesCalledDeregisterInstances) 135 | } 136 | } 137 | 138 | func Test_DeregisterInvalidException(t *testing.T) { 139 | t.Log("Test_DeregisterError: should return an error when call fails") 140 | var ( 141 | stubber = &stubErrorELB{} 142 | elbName = "some-load-balancer" 143 | instances = []string{"i-1234567890"} 144 | expectedCalls = 1 145 | ) 146 | 147 | stubber.failHint = elb.ErrCodeInvalidEndPointException 148 | err := deregisterInstances(stubber, elbName, instances) 149 | if err == nil { 150 | t.Fatalf("Test_DeregisterInstance: expected error to have occured, got: %v", err) 151 | } 152 | 153 | if stubber.timesCalledDeregisterInstances != expectedCalls { 154 | t.Fatalf("Test_DeregisterInstance: expected timesCalledDeregisterInstances: %v, got: %v", expectedCalls, stubber.timesCalledDeregisterInstances) 155 | } 156 | } 157 | 158 | func Test_DeregisterWaiterAbort(t *testing.T) { 159 | t.Log("Test_DeregisterWaiterAbort: should stop waiter when event is completed") 160 | var ( 161 | event = &LifecycleEvent{} 162 | elbName = "some-load-balancer" 163 | instanceID = "i-1234567890" 164 | expectedCalls = 1 165 | ) 166 | 167 | stubber := &stubELB{ 168 | instanceStates: []*elb.InstanceState{ 169 | { 170 | InstanceId: aws.String(instanceID), 171 | State: aws.String("InService"), 172 | }, 173 | }, 174 | } 175 | 176 | go _completeEventAfter(event, time.Millisecond*1500) 177 | err := waitForDeregisterInstance(event, stubber, elbName, instanceID) 178 | if err == nil { 179 | t.Fatalf("Test_DeregisterWaiterAbort: expected error to have occured, got: %v", err) 180 | } 181 | 182 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 183 | t.Fatalf("Test_DeregisterWaiterAbort: expected timesCalledDescribeInstanceHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 184 | } 185 | } 186 | 187 | func Test_DeregisterWaiterFail(t *testing.T) { 188 | t.Log("Test_DeregisterWaiterFail: should return error when call fails") 189 | var ( 190 | event = &LifecycleEvent{} 191 | elbName = "some-load-balancer" 192 | instanceID = "i-1234567890" 193 | expectedCalls = 1 194 | ) 195 | 196 | stubber := &stubErrorELB{ 197 | failHint: elb.ErrCodeAccessPointNotFoundException, 198 | } 199 | 200 | err := waitForDeregisterInstance(event, stubber, elbName, instanceID) 201 | if err == nil { 202 | t.Fatalf("Test_DeregisterWaiterFail: expected error to have occured, got: %v", err) 203 | } 204 | 205 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 206 | t.Fatalf("Test_DeregisterWaiterFail: expected timesCalledDescribeInstanceHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 207 | } 208 | } 209 | 210 | func Test_DeregisterWaiterNotFound(t *testing.T) { 211 | t.Log("Test_DeregisterWaiterNotFound: should return without error instance not found") 212 | var ( 213 | event = &LifecycleEvent{} 214 | elbName = "some-load-balancer" 215 | instanceID = "i-1234567890" 216 | expectedCalls = 1 217 | ) 218 | 219 | stubber := &stubELB{ 220 | instanceStates: []*elb.InstanceState{ 221 | { 222 | InstanceId: aws.String("some-other-instance"), 223 | State: aws.String("InService"), 224 | }, 225 | }, 226 | } 227 | 228 | err := waitForDeregisterInstance(event, stubber, elbName, instanceID) 229 | if err != nil { 230 | t.Fatalf("Test_DeregisterWaiterNotFound: expected error not to have occured, got: %v", err) 231 | } 232 | 233 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 234 | t.Fatalf("Test_DeregisterWaiterNotFound: expected timesCalledDescribeInstanceHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 235 | } 236 | } 237 | 238 | func Test_DeregisterWaiterTimeout(t *testing.T) { 239 | t.Log("Test_DeregisterWaiterTimeout: should return error when waiter times out") 240 | var ( 241 | event = &LifecycleEvent{} 242 | elbName = "some-load-balancer" 243 | instanceID = "i-1234567890" 244 | expectedCalls = 4 245 | ) 246 | 247 | stubber := &stubELB{ 248 | instanceStates: []*elb.InstanceState{ 249 | { 250 | InstanceId: aws.String(instanceID), 251 | State: aws.String("InService"), 252 | }, 253 | }, 254 | } 255 | 256 | err := waitForDeregisterInstance(event, stubber, elbName, instanceID) 257 | if err == nil { 258 | t.Fatalf("Test_DeregisterWaiterTimeout: expected error to have occured, got: %v", err) 259 | } 260 | 261 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 262 | t.Fatalf("Test_DeregisterWaiterTimeout: expected timesCalledDescribeInstanceHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 263 | } 264 | } 265 | 266 | func Test_FindInstanceInClassicBalancerPositive(t *testing.T) { 267 | t.Log("Test_FindInstanceInTargetGroupPositive: should be able to find instance in target group if it exists") 268 | var ( 269 | elbName = "some-load-balancer" 270 | instanceID = "i-1234567890" 271 | expectedCalls = 1 272 | ) 273 | stubber := &stubELB{ 274 | instanceStates: []*elb.InstanceState{ 275 | { 276 | InstanceId: aws.String(instanceID), 277 | }, 278 | }, 279 | } 280 | 281 | found, err := findInstanceInClassicBalancer(stubber, elbName, instanceID) 282 | if err != nil { 283 | t.Fatalf("Test_FindInstanceInClassicBalancerPositive: expected error not to have occured, %v", err) 284 | } 285 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 286 | t.Fatalf("expected timesCalledDescribeInstanceHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 287 | } 288 | if !found { 289 | t.Fatalf("Test_FindInstanceInClassicBalancerPositive: expected instance to be found") 290 | } 291 | } 292 | 293 | func Test_FindInstanceInClassicBalancerNegative(t *testing.T) { 294 | t.Log("Test_FindInstanceInClassicBalancerNegative: should be able to find instance in target group if it exists") 295 | var ( 296 | elbName = "some-load-balancer" 297 | instanceID = "i-1234567890" 298 | expectedCalls = 1 299 | stubber = &stubELB{} 300 | ) 301 | 302 | found, err := findInstanceInClassicBalancer(stubber, elbName, instanceID) 303 | if err != nil { 304 | t.Fatalf("Test_FindInstanceInClassicBalancerNegative: expected error not to have occured, %v", err) 305 | } 306 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 307 | t.Fatalf("expected timesCalledDescribeInstanceHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 308 | } 309 | if found { 310 | t.Fatalf("Test_FindInstanceInClassicBalancerNegative: expected instance not to be found") 311 | } 312 | } 313 | 314 | func Test_FindInstanceInClassicBalancerError(t *testing.T) { 315 | t.Log("Test_FindInstanceInClassicBalancerError: should return error if call fails") 316 | var ( 317 | elbName = "some-load-balancer" 318 | instanceID = "i-1234567890" 319 | expectedCalls = 1 320 | stubber = &stubErrorELB{} 321 | ) 322 | 323 | stubber.failHint = elb.ErrCodeAccessPointNotFoundException 324 | found, err := findInstanceInClassicBalancer(stubber, elbName, instanceID) 325 | if err == nil { 326 | t.Fatalf("Test_FindInstanceInClassicBalancerError: expected error to have occured, got: %v", err) 327 | } 328 | if stubber.timesCalledDescribeInstanceHealth != expectedCalls { 329 | t.Fatalf("expected Test_FindInstanceInClassicBalancerError: %v, got: %v", expectedCalls, stubber.timesCalledDescribeInstanceHealth) 330 | } 331 | if found { 332 | t.Fatalf("Test_FindInstanceInClassicBalancerError: expected instance not to be found") 333 | } 334 | } 335 | -------------------------------------------------------------------------------- /pkg/service/elbv2.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/elbv2" 8 | "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" 9 | iebackoff "github.com/keikoproj/inverse-exp-backoff" 10 | 11 | "github.com/keikoproj/lifecycle-manager/pkg/log" 12 | ) 13 | 14 | func waitForDeregisterTarget(event *LifecycleEvent, elbClient elbv2iface.ELBV2API, arn, instanceID string, port int64) error { 15 | var ( 16 | found bool 17 | ) 18 | 19 | input := &elbv2.DescribeTargetHealthInput{ 20 | TargetGroupArn: aws.String(arn), 21 | } 22 | 23 | for ieb, err := iebackoff.NewIEBackoff(WaiterMaxDelay, WaiterMinDelay, 0.5, WaiterMaxAttempts); err == nil; err = ieb.Next() { 24 | 25 | if event.eventCompleted { 26 | return errors.New("event finished execution during deregistration wait") 27 | } 28 | 29 | found = false 30 | targets, err := elbClient.DescribeTargetHealth(input) 31 | if err != nil { 32 | return err 33 | } 34 | for _, targetDescription := range targets.TargetHealthDescriptions { 35 | if aws.StringValue(targetDescription.Target.Id) == instanceID && aws.Int64Value(targetDescription.Target.Port) == port { 36 | found = true 37 | if aws.StringValue(targetDescription.TargetHealth.State) == elbv2.TargetHealthStateEnumUnused { 38 | return nil 39 | } 40 | break 41 | } 42 | } 43 | if !found { 44 | log.Debugf("%v> target not found in target group %v", instanceID, arn) 45 | return nil 46 | } 47 | log.Debugf("%v> deregistration from %v pending", instanceID, arn) 48 | } 49 | 50 | err := errors.New("wait for target deregister timed out") 51 | return err 52 | } 53 | 54 | func findInstanceInTargetGroup(elbClient elbv2iface.ELBV2API, arn, instanceID string) (bool, int64, error) { 55 | input := &elbv2.DescribeTargetHealthInput{ 56 | TargetGroupArn: aws.String(arn), 57 | } 58 | 59 | target, err := elbClient.DescribeTargetHealth(input) 60 | if err != nil { 61 | log.Errorf("%v> failed finding instance in target group %v: %v", instanceID, arn, err.Error()) 62 | return false, 0, err 63 | } 64 | for _, desc := range target.TargetHealthDescriptions { 65 | if aws.StringValue(desc.Target.Id) == instanceID { 66 | port := aws.Int64Value(desc.Target.Port) 67 | return true, port, nil 68 | } 69 | } 70 | return false, 0, nil 71 | } 72 | 73 | func deregisterTargets(elbClient elbv2iface.ELBV2API, arn string, mapping map[string]int64) error { 74 | 75 | targets := []*elbv2.TargetDescription{} 76 | for instance, port := range mapping { 77 | target := &elbv2.TargetDescription{ 78 | Id: aws.String(instance), 79 | Port: aws.Int64(port), 80 | } 81 | targets = append(targets, target) 82 | } 83 | 84 | input := &elbv2.DeregisterTargetsInput{ 85 | Targets: targets, 86 | TargetGroupArn: aws.String(arn), 87 | } 88 | 89 | _, err := elbClient.DeregisterTargets(input) 90 | if err != nil { 91 | return err 92 | } 93 | return nil 94 | } 95 | -------------------------------------------------------------------------------- /pkg/service/elbv2_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/awserr" 11 | "github.com/aws/aws-sdk-go/aws/request" 12 | "github.com/aws/aws-sdk-go/service/elbv2" 13 | "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" 14 | ) 15 | 16 | type stubELBv2 struct { 17 | elbv2iface.ELBV2API 18 | targetHealthDescriptions []*elbv2.TargetHealthDescription 19 | targetGroups []*elbv2.TargetGroup 20 | timesCalledDescribeTargetHealth int 21 | timesCalledDeregisterTargets int 22 | timesCalledDescribeTargetGroups int 23 | } 24 | 25 | func (e *stubELBv2) WaitUntilTargetDeregisteredWithContext(ctx context.Context, input *elbv2.DescribeTargetHealthInput, req ...request.WaiterOption) error { 26 | return nil 27 | } 28 | 29 | func (e *stubELBv2) DescribeTargetHealth(input *elbv2.DescribeTargetHealthInput) (*elbv2.DescribeTargetHealthOutput, error) { 30 | e.timesCalledDescribeTargetHealth++ 31 | return &elbv2.DescribeTargetHealthOutput{TargetHealthDescriptions: e.targetHealthDescriptions}, nil 32 | } 33 | 34 | func (e *stubELBv2) DeregisterTargets(input *elbv2.DeregisterTargetsInput) (*elbv2.DeregisterTargetsOutput, error) { 35 | e.timesCalledDeregisterTargets++ 36 | return &elbv2.DeregisterTargetsOutput{}, nil 37 | } 38 | 39 | func (e *stubELBv2) DescribeTargetGroups(input *elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) { 40 | e.timesCalledDescribeTargetGroups++ 41 | return &elbv2.DescribeTargetGroupsOutput{TargetGroups: e.targetGroups}, nil 42 | } 43 | 44 | func (e *stubELBv2) DescribeTargetGroupsPages(input *elbv2.DescribeTargetGroupsInput, callback func(*elbv2.DescribeTargetGroupsOutput, bool) bool) error { 45 | page, err := e.DescribeTargetGroups(input) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | callback(page, false) 51 | 52 | return nil 53 | } 54 | 55 | type stubErrorELBv2 struct { 56 | elbv2iface.ELBV2API 57 | targetHealthDescriptions []*elbv2.TargetHealthDescription 58 | targetGroups []*elbv2.TargetGroup 59 | timesCalledDescribeTargetHealth int 60 | timesCalledDeregisterTargets int 61 | timesCalledDescribeTargetGroups int 62 | failHint string 63 | } 64 | 65 | func (e *stubErrorELBv2) DescribeTargetHealth(input *elbv2.DescribeTargetHealthInput) (*elbv2.DescribeTargetHealthOutput, error) { 66 | e.timesCalledDescribeTargetHealth++ 67 | var err error 68 | if e.failHint == elbv2.ErrCodeTargetGroupNotFoundException { 69 | err = awserr.New(elbv2.ErrCodeTargetGroupNotFoundException, "failed", fmt.Errorf("it failed")) 70 | } else { 71 | err = fmt.Errorf("some other error occured") 72 | } 73 | return &elbv2.DescribeTargetHealthOutput{}, err 74 | } 75 | 76 | func (e *stubErrorELBv2) DeregisterTargets(input *elbv2.DeregisterTargetsInput) (*elbv2.DeregisterTargetsOutput, error) { 77 | e.timesCalledDeregisterTargets++ 78 | var err error 79 | if e.failHint == elbv2.ErrCodeTargetGroupNotFoundException { 80 | err = awserr.New(elbv2.ErrCodeTargetGroupNotFoundException, "failed", fmt.Errorf("it failed")) 81 | } else if e.failHint == elbv2.ErrCodeInvalidTargetException { 82 | err = awserr.New(elbv2.ErrCodeInvalidTargetException, "failed", fmt.Errorf("it failed")) 83 | } else { 84 | err = fmt.Errorf("some other error occured") 85 | } 86 | return &elbv2.DeregisterTargetsOutput{}, err 87 | } 88 | 89 | func (e *stubErrorELBv2) DescribeTargetGroups(input *elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) { 90 | e.timesCalledDescribeTargetGroups++ 91 | return &elbv2.DescribeTargetGroupsOutput{TargetGroups: e.targetGroups}, nil 92 | } 93 | 94 | func (e *stubErrorELBv2) DescribeTargetGroupsPages(input *elbv2.DescribeTargetGroupsInput, callback func(*elbv2.DescribeTargetGroupsOutput, bool) bool) error { 95 | page, err := e.DescribeTargetGroups(input) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | callback(page, false) 101 | 102 | return nil 103 | } 104 | 105 | func Test_DeregisterTargetWaiterAbort(t *testing.T) { 106 | t.Log("Test_DeregisterTargetWaiterAbort: should return when event is completed") 107 | var ( 108 | event = &LifecycleEvent{} 109 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 110 | instanceID = "i-1234567890" 111 | port int64 = 32334 112 | expectedCalls = 1 113 | ) 114 | 115 | stubber := &stubELBv2{ 116 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 117 | { 118 | Target: &elbv2.TargetDescription{ 119 | Id: aws.String(instanceID), 120 | Port: aws.Int64(port), 121 | }, 122 | TargetHealth: &elbv2.TargetHealth{ 123 | State: aws.String(elbv2.TargetHealthStateEnumHealthy), 124 | }, 125 | }, 126 | }, 127 | } 128 | 129 | go _completeEventAfter(event, time.Millisecond*1500) 130 | 131 | err := waitForDeregisterTarget(event, stubber, arn, instanceID, port) 132 | if err == nil { 133 | t.Fatalf("Test_DeregisterTargetWaiterAbort: expected error not have occured, %v", err) 134 | } 135 | 136 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 137 | t.Fatalf("Test_DeregisterTargetWaiterAbort: expected timesCalledDescribeTargetHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 138 | } 139 | } 140 | 141 | func Test_DeregisterTargetWaiterNotFound(t *testing.T) { 142 | t.Log("Test_DeregisterTargetWaiterNotFound: should return when instance is not found") 143 | var ( 144 | event = &LifecycleEvent{} 145 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 146 | instanceID = "i-1234567890" 147 | port int64 = 32334 148 | expectedCalls = 1 149 | ) 150 | 151 | stubber := &stubELBv2{ 152 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 153 | { 154 | Target: &elbv2.TargetDescription{ 155 | Id: aws.String("i-11111111"), 156 | Port: aws.Int64(port), 157 | }, 158 | TargetHealth: &elbv2.TargetHealth{ 159 | State: aws.String(elbv2.TargetHealthStateEnumHealthy), 160 | }, 161 | }, 162 | }, 163 | } 164 | 165 | err := waitForDeregisterTarget(event, stubber, arn, instanceID, port) 166 | if err != nil { 167 | t.Fatalf("Test_DeregisterTargetWaiterNotFound: expected error not to have occured, %v", err) 168 | } 169 | 170 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 171 | t.Fatalf("Test_DeregisterTargetWaiterNotFound: expected timesCalledDescribeTargetHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 172 | } 173 | } 174 | 175 | func Test_DeregisterTargetWaiterTimeout(t *testing.T) { 176 | t.Log("Test_DeregisterTargetWaiterTimeout: should return an error when timeout occurs") 177 | var ( 178 | event = &LifecycleEvent{} 179 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 180 | instanceID = "i-1234567890" 181 | port int64 = 32334 182 | expectedCalls = 4 183 | ) 184 | 185 | stubber := &stubELBv2{ 186 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 187 | { 188 | Target: &elbv2.TargetDescription{ 189 | Id: aws.String(instanceID), 190 | Port: aws.Int64(port), 191 | }, 192 | TargetHealth: &elbv2.TargetHealth{ 193 | State: aws.String(elbv2.TargetHealthStateEnumHealthy), 194 | }, 195 | }, 196 | }, 197 | } 198 | 199 | err := waitForDeregisterTarget(event, stubber, arn, instanceID, port) 200 | if err == nil { 201 | t.Fatalf("Test_DeregisterTargetWaiterNotFound: expected error to have occured, %v", err) 202 | } 203 | 204 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 205 | t.Fatalf("Test_DeregisterTargetWaiterNotFound: expected timesCalledDescribeTargetHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 206 | } 207 | } 208 | 209 | func Test_DeregisterTargetWaiterFail(t *testing.T) { 210 | t.Log("Test_DeregisterTargetWaiterFail: should return an error when call fails") 211 | var ( 212 | event = &LifecycleEvent{} 213 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 214 | instanceID = "i-1234567890" 215 | port int64 = 32334 216 | expectedCalls = 1 217 | ) 218 | 219 | stubber := &stubErrorELBv2{ 220 | failHint: elbv2.ErrCodeTargetGroupNotFoundException, 221 | } 222 | 223 | err := waitForDeregisterTarget(event, stubber, arn, instanceID, port) 224 | if err == nil { 225 | t.Fatalf("Test_DeregisterTargetWaiterFail: expected error not to have occured, %v", err) 226 | } 227 | 228 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 229 | t.Fatalf("Test_DeregisterTargetWaiterFail: expected timesCalledDescribeTargetHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 230 | } 231 | } 232 | 233 | func Test_DeregisterTarget(t *testing.T) { 234 | t.Log("Test_DeregisterTarget: should be able to deregister an instance") 235 | var ( 236 | stubber = &stubELBv2{} 237 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 238 | instances = map[string]int64{"i-1234567890": 32334} 239 | expectedCalls = 1 240 | ) 241 | 242 | err := deregisterTargets(stubber, arn, instances) 243 | if err != nil { 244 | t.Fatalf("Test_DeregisterInstance: expected error not to have occured, %v", err) 245 | } 246 | 247 | if stubber.timesCalledDeregisterTargets != expectedCalls { 248 | t.Fatalf("Test_DeregisterInstance: expected timesCalledDeregisterTargets: %v, got: %v", expectedCalls, stubber.timesCalledDeregisterTargets) 249 | } 250 | } 251 | 252 | func Test_DeregisterTargetNotFoundException(t *testing.T) { 253 | t.Log("Test_DeregisterTargetNotFoundException: should return error when call fails") 254 | var ( 255 | stubber = &stubErrorELBv2{} 256 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 257 | instances = map[string]int64{"i-1234567890": 32334} 258 | expectedCalls = 1 259 | ) 260 | 261 | stubber.failHint = elbv2.ErrCodeTargetGroupNotFoundException 262 | err := deregisterTargets(stubber, arn, instances) 263 | if err == nil { 264 | t.Fatalf("Test_DeregisterTargetNotFoundException: expected error not have occured, %v", err) 265 | } 266 | 267 | if stubber.timesCalledDeregisterTargets != expectedCalls { 268 | t.Fatalf("Test_DeregisterTargetNotFoundException: expected timesCalledDeregisterTargets: %v, got: %v", expectedCalls, stubber.timesCalledDeregisterTargets) 269 | } 270 | } 271 | 272 | func Test_DeregisterTargetInvalidException(t *testing.T) { 273 | t.Log("Test_DeregisterTargetInvalidException: should return error when call fails") 274 | var ( 275 | stubber = &stubErrorELBv2{} 276 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 277 | instances = map[string]int64{"i-1234567890": 32334} 278 | expectedCalls = 1 279 | ) 280 | 281 | stubber.failHint = elbv2.ErrCodeInvalidTargetException 282 | err := deregisterTargets(stubber, arn, instances) 283 | if err == nil { 284 | t.Fatalf("Test_DeregisterTargetInvalidException: expected error not have occured, %v", err) 285 | } 286 | 287 | if stubber.timesCalledDeregisterTargets != expectedCalls { 288 | t.Fatalf("Test_DeregisterTargetInvalidException: expected timesCalledDeregisterTargets: %v, got: %v", expectedCalls, stubber.timesCalledDeregisterTargets) 289 | } 290 | } 291 | 292 | func Test_FindInstanceInTargetGroupPositive(t *testing.T) { 293 | t.Log("Test_FindInstanceInTargetGroupPositive: should be able to find instance in target group if it exists") 294 | var ( 295 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 296 | instanceID = "i-1234567890" 297 | port int64 = 32334 298 | expectedCalls = 1 299 | ) 300 | stubber := &stubELBv2{ 301 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 302 | { 303 | Target: &elbv2.TargetDescription{ 304 | Id: aws.String(instanceID), 305 | Port: aws.Int64(port), 306 | }, 307 | }, 308 | }, 309 | } 310 | found, foundPort, err := findInstanceInTargetGroup(stubber, arn, instanceID) 311 | if err != nil { 312 | t.Fatalf("Test_FindInstanceInTargetGroupPositive: expected error not to have occured, %v", err) 313 | } 314 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 315 | t.Fatalf("expected timesCalledDescribeTargetHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 316 | } 317 | if !found { 318 | t.Fatalf("Test_FindInstanceInTargetGroupPositive: expected instance to be found") 319 | } 320 | if port != foundPort { 321 | t.Fatalf("Test_FindInstanceInTargetGroupPositive: expected port to be: %v got: %v", port, foundPort) 322 | } 323 | } 324 | 325 | func Test_FindInstanceInTargetGroupNegative(t *testing.T) { 326 | t.Log("Test_FindInstanceInTargetGroupNegative: should not be able to find instance in target group if it doesnt exists") 327 | var ( 328 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 329 | instanceID = "i-1234567890" 330 | port int64 = 32334 331 | expectedCalls = 1 332 | ) 333 | stubber := &stubELBv2{ 334 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 335 | { 336 | Target: &elbv2.TargetDescription{ 337 | Id: aws.String("i-222333444"), 338 | Port: aws.Int64(port), 339 | }, 340 | }, 341 | }, 342 | } 343 | found, _, err := findInstanceInTargetGroup(stubber, arn, instanceID) 344 | if err != nil { 345 | t.Fatalf("Test_FindInstanceInTargetGroupPositive: expected error not to have occured, %v", err) 346 | } 347 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 348 | t.Fatalf("expected timesCalledDescribeTargetHealth: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 349 | } 350 | if found { 351 | t.Fatalf("Test_FindInstanceInTargetGroupPositive: expected instance not to be found") 352 | } 353 | } 354 | 355 | func Test_FindInstanceInTargetGroupError(t *testing.T) { 356 | t.Log("Test_FindInstanceInTargetGroupError: should return error if call fails") 357 | var ( 358 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 359 | instanceID = "i-1234567890" 360 | expectedCalls = 1 361 | stubber = &stubErrorELBv2{} 362 | ) 363 | 364 | stubber.failHint = elbv2.ErrCodeTargetGroupNotFoundException 365 | 366 | found, _, err := findInstanceInTargetGroup(stubber, arn, instanceID) 367 | if err == nil { 368 | t.Fatalf("Test_FindInstanceInTargetGroupError: expected error to have occured, got: %v", err) 369 | } 370 | if stubber.timesCalledDescribeTargetHealth != expectedCalls { 371 | t.Fatalf("Test_FindInstanceInTargetGroupError: %v, got: %v", expectedCalls, stubber.timesCalledDescribeTargetHealth) 372 | } 373 | if found { 374 | t.Fatalf("Test_FindInstanceInTargetGroupError: expected instance not to be found") 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /pkg/service/events.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "math/rand" 8 | "time" 9 | 10 | "github.com/keikoproj/lifecycle-manager/pkg/log" 11 | 12 | v1 "k8s.io/api/core/v1" 13 | apimachinery_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 15 | "k8s.io/client-go/kubernetes" 16 | ) 17 | 18 | // EventReason defines the reason of an event 19 | type EventReason string 20 | 21 | // EventLevel defines the level of an event 22 | type EventLevel string 23 | 24 | const ( 25 | // EventLevelNormal is the level of a normal event 26 | EventLevelNormal = "Normal" 27 | // EventLevelWarning is the level of a warning event 28 | EventLevelWarning = "Warning" 29 | // EventReasonLifecycleHookReceived is the reason for a lifecycle received event 30 | EventReasonLifecycleHookReceived EventReason = "LifecycleHookReceived" 31 | // EventMessageLifecycleHookReceived is the message for a lifecycle received event 32 | EventMessageLifecycleHookReceived = "lifecycle hook for event %v was received, instance %v will begin processing" 33 | // EventReasonLifecycleHookProcessed is the reason for a lifecycle successful processing event 34 | EventReasonLifecycleHookProcessed EventReason = "LifecycleHookProcessed" 35 | //EventMessageLifecycleHookProcessed is the message for a lifecycle successful processing event 36 | EventMessageLifecycleHookProcessed = "lifecycle hook for event %v has completed processing, instance %v gracefully terminated after %vs" 37 | // EventReasonLifecycleHookFailed is the reason for a lifecycle failed event 38 | EventReasonLifecycleHookFailed EventReason = "LifecycleHookFailed" 39 | // EventMessageLifecycleHookFailed is the message for a lifecycle failed event 40 | EventMessageLifecycleHookFailed = "lifecycle hook for event %v has failed processing after %vs: %v" 41 | // EventReasonNodeDrainSucceeded is the reason for a successful drain event 42 | EventReasonNodeDrainSucceeded EventReason = "NodeDrainSucceeded" 43 | // EventMessageNodeDrainSucceeded is the message for a successful drain event 44 | EventMessageNodeDrainSucceeded = "node %v has been drained successfully as a response to a termination event" 45 | // EventReasonNodeDrainFailed is the reason for a failed drain event 46 | EventReasonNodeDrainFailed EventReason = "NodeDrainFailed" 47 | // EventMessageNodeDrainFailed is the message for a failed drain event 48 | EventMessageNodeDrainFailed = "node %v draining has failed: %v" 49 | // EventReasonNodeDeleteSucceeded is the reason for a successful node delete event 50 | EventReasonNodeDeleteSucceeded EventReason = "NodeDeleteSucceeded" 51 | // EventMessageNodeDeleteSucceeded is the message for a successful node delete event 52 | EventMessageNodeDeleteSucceeded = "node %v has been deleted successfully as a response to a termination event" 53 | // EventReasonNodeDeletenFailed is the reason for a failed node delete event 54 | EventReasonNodeDeleteFailed EventReason = "NodeDeleteFailed" 55 | // EventMessageNodeDeleteFailed is the message for a failed node delete event 56 | EventMessageNodeDeleteFailed = "node %v deletion has failed: %v" 57 | // EventReasonTargetDeregisterSucceeded is the reason for a successful target group deregister event 58 | EventReasonTargetDeregisterSucceeded EventReason = "TargetDeregisterSucceeded" 59 | // EventMessageTargetDeregisterSucceeded is the message for a successful target group deregister event 60 | EventMessageTargetDeregisterSucceeded = "target %v:%v has successfully deregistered from target group %v" 61 | // EventReasonTargetDeregisterFailed is the reason for a successful drain event 62 | EventReasonTargetDeregisterFailed EventReason = "TargetDeregisterFailed" 63 | // EventMessageTargetDeregisterFailed is the message for a successful drain event 64 | EventMessageTargetDeregisterFailed = "target %v has failed to deregistered from target group %v: %v" 65 | // EventReasonInstanceDeregisterSucceeded is the reason for a successful target group deregister event 66 | EventReasonInstanceDeregisterSucceeded EventReason = "InstanceDeregisterSucceeded" 67 | // EventMessageInstanceDeregisterSucceeded is the message for a successful target group deregister event 68 | EventMessageInstanceDeregisterSucceeded = "instance %v has successfully deregistered from classic-elb %v" 69 | // EventReasonInstanceDeregisterFailed is the reason for a successful classic elb deregister event 70 | EventReasonInstanceDeregisterFailed EventReason = "InstanceDeregisterFailed" 71 | // EventMessageInstanceDeregisterFailed is the message for a successful classic elb deregister event 72 | EventMessageInstanceDeregisterFailed = "instance %v has failed to deregister from classic-elb %v: %v" 73 | ) 74 | 75 | var ( 76 | // EventName is the default name for service events 77 | EventName = "lifecycle-manager.%v" 78 | // EventNamespace is the default namespace in which events will be published in 79 | EventNamespace = "default" 80 | 81 | // EventLevels is a map of event reasons and their event level 82 | EventLevels = map[EventReason]string{ 83 | EventReasonLifecycleHookReceived: EventLevelNormal, 84 | EventReasonLifecycleHookProcessed: EventLevelNormal, 85 | EventReasonLifecycleHookFailed: EventLevelWarning, 86 | EventReasonNodeDrainSucceeded: EventLevelNormal, 87 | EventReasonNodeDrainFailed: EventLevelWarning, 88 | EventReasonTargetDeregisterSucceeded: EventLevelNormal, 89 | EventReasonTargetDeregisterFailed: EventLevelWarning, 90 | EventReasonInstanceDeregisterSucceeded: EventLevelNormal, 91 | EventReasonInstanceDeregisterFailed: EventLevelWarning, 92 | } 93 | ) 94 | 95 | func publishKubernetesEvent(kubeClient kubernetes.Interface, event *v1.Event) { 96 | log.Debugf("publishing event: %v", event.Reason) 97 | _, err := kubeClient.CoreV1().Events(EventNamespace).Create(context.Background(), event, apimachinery_v1.CreateOptions{}) 98 | if err != nil { 99 | log.Errorf("failed to publish event: %v", err) 100 | } 101 | } 102 | 103 | func getReasonEventLevel(reason EventReason) string { 104 | if val, ok := EventLevels[reason]; ok { 105 | return val 106 | } 107 | return "Normal" 108 | } 109 | 110 | func getMessageFields(event *LifecycleEvent, details string) map[string]string { 111 | return map[string]string{ 112 | "eventID": event.RequestID, 113 | "ec2InstanceId": event.EC2InstanceID, 114 | "asgName": event.AutoScalingGroupName, 115 | "details": details, 116 | } 117 | } 118 | 119 | func newKubernetesEvent(reason EventReason, msgFields map[string]string) *v1.Event { 120 | // Marshal as JSON 121 | b, err := json.Marshal(msgFields) 122 | msgPayload := string(b) 123 | // I think it is very tough to trigger this error since json.Marshal function can return two types of errors 124 | // UnsupportedTypeError or UnsupportedValueError. Since our type is very rigid, these errors won't be triggered. 125 | if err != nil { 126 | log.Errorf("json.Marshal Failed, %s", err) 127 | // let's convert map to string since encoding as JSON is failing. At the least the information will be conveyed 128 | msgPayload = fmt.Sprintf("%v", msgFields) 129 | } 130 | 131 | eventName := fmt.Sprintf("%v-%v.%v", "lifecycle-manager", time.Now().Unix(), rand.Int()) 132 | t := metav1.Time{Time: time.Now()} 133 | event := &v1.Event{ 134 | ObjectMeta: metav1.ObjectMeta{ 135 | Name: eventName, 136 | Namespace: EventNamespace, 137 | }, 138 | InvolvedObject: v1.ObjectReference{ 139 | Kind: "LifecycleManager", 140 | Namespace: EventNamespace, 141 | }, 142 | Reason: string(reason), 143 | Message: msgPayload, 144 | Type: getReasonEventLevel(reason), 145 | Count: 1, 146 | FirstTimestamp: t, 147 | LastTimestamp: t, 148 | } 149 | return event 150 | } 151 | -------------------------------------------------------------------------------- /pkg/service/events_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "testing" 8 | 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | "k8s.io/client-go/kubernetes/fake" 11 | ) 12 | 13 | func Test_NewEvent(t *testing.T) { 14 | t.Log("Test_NewEvent: should be able to get a new kubernetes event") 15 | msg := fmt.Sprintf(EventMessageInstanceDeregisterFailed, "i-123456789012", "my-load-balancer", "some bad error occurred") 16 | msgFields := map[string]string{ 17 | "ec2InstanceId": "i-123456789012", 18 | "loadBalancer": "my-load-balancer", 19 | "error": "some bad error occurred", 20 | "details": msg, 21 | } 22 | event := newKubernetesEvent(EventReasonInstanceDeregisterFailed, msgFields) 23 | 24 | if event.Reason != string(EventReasonInstanceDeregisterFailed) { 25 | t.Fatalf("expected event.Reason to be: %v, got: %v", string(EventReasonInstanceDeregisterFailed), event.Reason) 26 | } 27 | 28 | // decode the JSON 29 | var msgPayload map[string]string 30 | err := json.Unmarshal([]byte(event.Message), &msgPayload) 31 | if err != nil { 32 | t.Fatalf("json.Unmarshal Failed, %s", err) 33 | } 34 | 35 | if msg != msgPayload["details"] { 36 | t.Fatalf("expected event.Message to be: %v, got: %v", msg, msgPayload["details"]) 37 | } 38 | 39 | } 40 | 41 | func Test_PublishEvent(t *testing.T) { 42 | t.Log("Test_PublishEvent: should be able to publish kubernetes events") 43 | kubeClient := fake.NewSimpleClientset() 44 | msg := fmt.Sprintf(EventMessageInstanceDeregisterFailed, "i-123456789012", "my-load-balancer", "some bad error occurred") 45 | msgFields := map[string]string{ 46 | "ec2InstanceId": "i-123456789012", 47 | "loadBalancer": "my-load-balancer", 48 | "error": "some bad error occurred", 49 | "details": msg, 50 | } 51 | event := newKubernetesEvent(EventReasonInstanceDeregisterFailed, msgFields) 52 | 53 | publishKubernetesEvent(kubeClient, event) 54 | expectedEvents := 1 55 | 56 | events, err := kubeClient.CoreV1().Events(EventNamespace).List(context.Background(), metav1.ListOptions{}) 57 | if err != nil { 58 | t.Fatalf("Test_PublishEvent: expected error not to have occured, %v", err) 59 | } 60 | 61 | if len(events.Items) != expectedEvents { 62 | t.Fatalf("Test_PublishEvent: expected %v events, found: %v", expectedEvents, len(events.Items)) 63 | } 64 | 65 | // decode the JSON 66 | var msgPayload map[string]string 67 | err = json.Unmarshal([]byte(event.Message), &msgPayload) 68 | if err != nil { 69 | t.Fatalf("json.Unmarshal Failed, %s", err) 70 | } 71 | if msgPayload["ec2InstanceId"] != "i-123456789012" { 72 | t.Fatalf("Expected=%s, Got=%s", "i-123456789012", msgPayload["ec2InstanceId"]) 73 | } 74 | if msg != msgPayload["details"] { 75 | t.Fatalf("expected event.Message to be: %v, got: %v", msg, msgPayload["details"]) 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /pkg/service/lifecycle.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aws/aws-sdk-go/service/sqs" 7 | v1 "k8s.io/api/core/v1" 8 | ) 9 | 10 | type LifecycleEvent struct { 11 | LifecycleHookName string `json:"LifecycleHookName"` 12 | AccountID string `json:"AccountId"` 13 | RequestID string `json:"RequestId"` 14 | LifecycleTransition string `json:"LifecycleTransition"` 15 | AutoScalingGroupName string `json:"AutoScalingGroupName"` 16 | EC2InstanceID string `json:"EC2InstanceId"` 17 | LifecycleActionToken string `json:"LifecycleActionToken"` 18 | receiptHandle string 19 | queueURL string 20 | heartbeatInterval int64 21 | referencedNode v1.Node 22 | drainCompleted bool 23 | nodeDeleted bool 24 | deregisterCompleted bool 25 | eventCompleted bool 26 | startTime time.Time 27 | message *sqs.Message 28 | } 29 | 30 | // SetMessage is a setter method for the sqs message body 31 | func (e *LifecycleEvent) SetMessage(message *sqs.Message) { e.message = message } 32 | 33 | // SetReceiptHandle is a setter method for the receipt handle of the event 34 | func (e *LifecycleEvent) SetReceiptHandle(receipt string) { e.receiptHandle = receipt } 35 | 36 | // SetQueueURL is a setter method for the url of the SQS queue 37 | func (e *LifecycleEvent) SetQueueURL(url string) { e.queueURL = url } 38 | 39 | // SetHeartbeatInterval is a setter method for heartbeat interval of the event 40 | func (e *LifecycleEvent) SetHeartbeatInterval(interval int64) { e.heartbeatInterval = interval } 41 | 42 | // SetReferencedNode is a setter method for the event referenced node 43 | func (e *LifecycleEvent) SetReferencedNode(node v1.Node) { e.referencedNode = node } 44 | 45 | // SetDrainCompleted is a setter method for status of the drain operation 46 | func (e *LifecycleEvent) SetDrainCompleted(val bool) { e.drainCompleted = val } 47 | 48 | // SetNodeDeleted is a setter method for status of the node deletion operation 49 | func (e *LifecycleEvent) SetNodeDeleted(val bool) { e.nodeDeleted = val } 50 | 51 | // SetDeregisterCompleted is a setter method for status of the drain operation 52 | func (e *LifecycleEvent) SetDeregisterCompleted(val bool) { e.deregisterCompleted = val } 53 | 54 | // SetEventCompleted is a setter method for status of the drain operation 55 | func (e *LifecycleEvent) SetEventCompleted(val bool) { e.eventCompleted = val } 56 | 57 | // SetEventTimeStarted is a setter method for the time an event started 58 | func (e *LifecycleEvent) SetEventTimeStarted(t time.Time) { e.startTime = t } 59 | -------------------------------------------------------------------------------- /pkg/service/manager.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sync" 7 | "time" 8 | 9 | "golang.org/x/sync/semaphore" 10 | 11 | "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" 12 | "github.com/aws/aws-sdk-go/service/elb/elbiface" 13 | "github.com/aws/aws-sdk-go/service/elbv2/elbv2iface" 14 | "github.com/aws/aws-sdk-go/service/sqs" 15 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 16 | "github.com/keikoproj/lifecycle-manager/pkg/log" 17 | 18 | "github.com/keikoproj/aws-sdk-go-cache/cache" 19 | 20 | "k8s.io/client-go/kubernetes" 21 | ) 22 | 23 | // Manager is the main object for lifecycle-manager and holds the state 24 | type Manager struct { 25 | eventStream chan *sqs.Message 26 | authenticator Authenticator 27 | context ManagerContext 28 | deregistrationMu sync.Mutex 29 | sync.Mutex 30 | workQueue []*LifecycleEvent 31 | targets *sync.Map 32 | metrics *MetricsServer 33 | avarageLatency float64 34 | completedEvents int 35 | rejectedEvents int 36 | failedEvents int 37 | } 38 | 39 | // ManagerContext contain the user input parameters on the current context 40 | type ManagerContext struct { 41 | CacheConfig *cache.Config 42 | KubectlLocalPath string 43 | QueueName string 44 | Region string 45 | DrainTimeoutUnknownSeconds int64 46 | DrainTimeoutSeconds int64 47 | DrainRetryIntervalSeconds int64 48 | DrainRetryAttempts uint 49 | PollingIntervalSeconds int64 50 | WithDeregister bool 51 | DeregisterTargetTypes []string 52 | MaxDrainConcurrency *semaphore.Weighted 53 | MaxTimeToProcessSeconds int64 54 | } 55 | 56 | // Authenticator holds clients for all required APIs 57 | type Authenticator struct { 58 | ScalingGroupClient autoscalingiface.AutoScalingAPI 59 | SQSClient sqsiface.SQSAPI 60 | ELBv2Client elbv2iface.ELBV2API 61 | ELBClient elbiface.ELBAPI 62 | KubernetesClient kubernetes.Interface 63 | } 64 | 65 | // ScanResult contains a list of found load balancers and target groups 66 | type ScanResult struct { 67 | ActiveLoadBalancers []string 68 | ActiveTargetGroups map[string]int64 69 | } 70 | 71 | type Waiter struct { 72 | sync.WaitGroup 73 | finished chan bool 74 | errors chan WaiterError 75 | classicWaiterCount int 76 | targetGroupWaiterCount int 77 | } 78 | 79 | func (w *Waiter) IncClassicWaiter() { w.classicWaiterCount++ } 80 | func (w *Waiter) DecClassicWaiter() { w.classicWaiterCount-- } 81 | func (w *Waiter) IncTargetGroupWaiter() { w.targetGroupWaiterCount++ } 82 | func (w *Waiter) DecTargetGroupWaiter() { w.targetGroupWaiterCount-- } 83 | 84 | type WaiterError struct { 85 | Error error 86 | Type TargetType 87 | } 88 | 89 | func New(auth Authenticator, ctx ManagerContext) *Manager { 90 | return &Manager{ 91 | eventStream: make(chan *sqs.Message, 0), 92 | workQueue: make([]*LifecycleEvent, 0), 93 | metrics: &MetricsServer{}, 94 | targets: &sync.Map{}, 95 | authenticator: auth, 96 | context: ctx, 97 | } 98 | } 99 | 100 | func (mgr *Manager) AddEvent(event *LifecycleEvent) { 101 | var ( 102 | metrics = mgr.metrics 103 | kube = mgr.authenticator.KubernetesClient 104 | ) 105 | mgr.Lock() 106 | event.SetEventTimeStarted(time.Now()) 107 | metrics.IncGauge(TerminatingInstancesCountMetric) 108 | 109 | if !mgr.EventInQueue(event) { 110 | mgr.workQueue = append(mgr.workQueue, event) 111 | } 112 | 113 | mgr.Unlock() 114 | 115 | msg := fmt.Sprintf(EventMessageLifecycleHookReceived, event.RequestID, event.EC2InstanceID) 116 | kEvent := newKubernetesEvent(EventReasonLifecycleHookReceived, getMessageFields(event, msg)) 117 | publishKubernetesEvent(kube, kEvent) 118 | } 119 | 120 | func (mgr *Manager) EventInQueue(e *LifecycleEvent) bool { 121 | for _, event := range mgr.workQueue { 122 | if event.RequestID == e.RequestID { 123 | return true 124 | } 125 | } 126 | return false 127 | } 128 | 129 | func (mgr *Manager) RemoveFromQueue(event *LifecycleEvent) { 130 | for idx, ev := range mgr.workQueue { 131 | if event.RequestID == ev.RequestID { 132 | mgr.Lock() 133 | mgr.workQueue = append(mgr.workQueue[0:idx], mgr.workQueue[idx+1:]...) 134 | mgr.Unlock() 135 | } 136 | } 137 | } 138 | 139 | func (mgr *Manager) CompleteEvent(event *LifecycleEvent) { 140 | var ( 141 | queue = mgr.authenticator.SQSClient 142 | metrics = mgr.metrics 143 | kubeClient = mgr.authenticator.KubernetesClient 144 | asgClient = mgr.authenticator.ScalingGroupClient 145 | url = event.queueURL 146 | t = time.Since(event.startTime).Seconds() 147 | ) 148 | 149 | if mgr.avarageLatency == 0 { 150 | mgr.avarageLatency = t 151 | } else { 152 | mgr.avarageLatency = (mgr.avarageLatency + t) / 2 153 | } 154 | 155 | mgr.completedEvents++ 156 | 157 | log.Infof("event %v completed processing", event.RequestID) 158 | event.SetEventCompleted(true) 159 | 160 | err := deleteMessage(queue, url, event.receiptHandle) 161 | if err != nil { 162 | log.Errorf("failed to delete message: %v", err) 163 | } 164 | 165 | err = completeLifecycleAction(asgClient, *event, ContinueAction) 166 | if err != nil { 167 | log.Errorf("failed to complete lifecycle action: %v", err) 168 | } 169 | 170 | mgr.RemoveFromQueue(event) 171 | msg := fmt.Sprintf(EventMessageLifecycleHookProcessed, event.RequestID, event.EC2InstanceID, t) 172 | kEvent := newKubernetesEvent(EventReasonLifecycleHookProcessed, getMessageFields(event, msg)) 173 | publishKubernetesEvent(kubeClient, kEvent) 174 | 175 | metrics.AddCounter(SuccessfulEventsTotalMetric, 1) 176 | metrics.DecGauge(TerminatingInstancesCountMetric) 177 | metrics.SetGauge(AverageDurationSecondsMetric, mgr.avarageLatency) 178 | log.Infof("event %v for instance %v completed after %vs", event.RequestID, event.EC2InstanceID, t) 179 | } 180 | 181 | func (mgr *Manager) FailEvent(err error, event *LifecycleEvent, abandon bool) { 182 | var ( 183 | auth = mgr.authenticator 184 | kubeClient = auth.KubernetesClient 185 | queue = auth.SQSClient 186 | metrics = mgr.metrics 187 | scalingGroupClient = auth.ScalingGroupClient 188 | url = event.queueURL 189 | t = time.Since(event.startTime).Seconds() 190 | ) 191 | log.Errorf("event %v has failed processing after %vs: %v", event.RequestID, t, err) 192 | mgr.failedEvents++ 193 | metrics.AddCounter(FailedEventsTotalMetric, 1) 194 | event.SetEventCompleted(true) 195 | 196 | msg := fmt.Sprintf(EventMessageLifecycleHookFailed, event.RequestID, t, err) 197 | kEvent := newKubernetesEvent(EventReasonLifecycleHookFailed, getMessageFields(event, msg)) 198 | publishKubernetesEvent(kubeClient, kEvent) 199 | 200 | if abandon { 201 | log.Warnf("abandoning instance %v", event.EC2InstanceID) 202 | err := completeLifecycleAction(scalingGroupClient, *event, AbandonAction) 203 | if err != nil { 204 | log.Errorf("completeLifecycleAction Failed, %s", err) 205 | } 206 | } 207 | 208 | if reflect.DeepEqual(event, LifecycleEvent{}) { 209 | log.Errorf("event failed: invalid message: %v", err) 210 | return 211 | } 212 | 213 | err = deleteMessage(queue, url, event.receiptHandle) 214 | if err != nil { 215 | log.Errorf("event failed: failed to delete message: %v", err) 216 | } 217 | 218 | } 219 | 220 | func (mgr *Manager) RejectEvent(err error, event *LifecycleEvent) { 221 | var ( 222 | metrics = mgr.metrics 223 | auth = mgr.authenticator 224 | queue = auth.SQSClient 225 | url = event.queueURL 226 | ) 227 | 228 | log.Debugf("event %v has been rejected for processing: %v", event.RequestID, err) 229 | mgr.rejectedEvents++ 230 | metrics.AddCounter(RejectedEventsTotalMetric, 1) 231 | 232 | if reflect.DeepEqual(event, LifecycleEvent{}) { 233 | log.Errorf("event failed: invalid message: %v", err) 234 | return 235 | } 236 | 237 | err = deleteMessage(queue, url, event.receiptHandle) 238 | if err != nil { 239 | log.Errorf("failed to delete message: %v", err) 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /pkg/service/metrics.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/keikoproj/lifecycle-manager/pkg/log" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promhttp" 9 | ) 10 | 11 | var ( 12 | // MetricsNamespace is the namespace of prometheus metrics 13 | MetricsNamespace = "lifecycle_manager" 14 | // MetricsPort is the port used to serve metrics 15 | MetricsPort = ":8080" 16 | // MetricsEndpoint is the endpoint to expose for metrics 17 | MetricsEndpoint = "/metrics" 18 | ) 19 | 20 | const ( 21 | ActiveGoroutinesMetric = "active_goroutines" 22 | TerminatingInstancesCountMetric = "terminating_instances_count" 23 | DrainingInstancesCountMetric = "draining_instances_count" 24 | DeregisteringInstancesCountMetric = "deregistering_instances_count" 25 | AverageDurationSecondsMetric = "average_duration_seconds" 26 | SuccessfulEventsTotalMetric = "successful_events_total" 27 | SuccessfulLBDeregisterTotalMetric = "successful_lb_deregister_total" 28 | SuccessfulNodeDrainTotalMetric = "successful_node_drain_total" 29 | SuccessfulNodeDeleteTotalMetric = "successful_node_delete_total" 30 | FailedEventsTotalMetric = "failed_events_total" 31 | FailedLBDeregisterTotalMetric = "failed_lb_deregister_total" 32 | FailedNodeDrainTotalMetric = "failed_node_drain_total" 33 | FailedNodeDeleteTotalMetric = "failed_node_delete_total" 34 | RejectedEventsTotalMetric = "rejected_events_total" 35 | ) 36 | 37 | type MetricsServer struct { 38 | Counters map[string]prometheus.Counter 39 | Gauges map[string]prometheus.Gauge 40 | } 41 | 42 | func (m *MetricsServer) Start() { 43 | m.Gauges = make(map[string]prometheus.Gauge, 0) 44 | m.Counters = make(map[string]prometheus.Counter, 0) 45 | 46 | gaugeIndex := map[string]string{ 47 | ActiveGoroutinesMetric: "indicates the current number of active goroutines.", 48 | TerminatingInstancesCountMetric: "indicates the current number of terminating instances.", 49 | DrainingInstancesCountMetric: "indicates the current number of draining instances.", 50 | DeregisteringInstancesCountMetric: "indicates the current number of deregistering instances.", 51 | AverageDurationSecondsMetric: "indicates the average duration of processing a hook in seconds.", 52 | } 53 | 54 | counterIndex := map[string]string{ 55 | SuccessfulEventsTotalMetric: "indicates the sum of all successful events.", 56 | SuccessfulLBDeregisterTotalMetric: "indicates the sum of all events that succeeded to deregister loadbalancer", 57 | SuccessfulNodeDrainTotalMetric: "indicates the sum of all events that succeeded to drain the node.", 58 | SuccessfulNodeDeleteTotalMetric: "indicates the sum of all events that succeeded to delete the node.", 59 | FailedEventsTotalMetric: "indicates the sum of all failed events.", 60 | FailedLBDeregisterTotalMetric: "indicates the sum of all events that failed to deregister loadbalancer.", 61 | FailedNodeDrainTotalMetric: "indicates the sum of all events that failed to drain the node.", 62 | FailedNodeDeleteTotalMetric: "indicates the sum of all events that failed to delete the node.", 63 | RejectedEventsTotalMetric: "indicates the sum of all rejected events.", 64 | } 65 | 66 | for gaugeName, desc := range gaugeIndex { 67 | gauge := prometheus.NewGauge( 68 | prometheus.GaugeOpts{ 69 | Namespace: MetricsNamespace, 70 | Name: string(gaugeName), 71 | Help: desc, 72 | }) 73 | m.Gauges[string(gaugeName)] = gauge 74 | } 75 | 76 | for counterName, desc := range counterIndex { 77 | counter := prometheus.NewCounter( 78 | prometheus.CounterOpts{ 79 | Namespace: MetricsNamespace, 80 | Name: counterName, 81 | Help: desc, 82 | }, 83 | ) 84 | m.Counters[counterName] = counter 85 | } 86 | 87 | http.Handle(MetricsEndpoint, promhttp.Handler()) 88 | 89 | for _, gauge := range m.Gauges { 90 | prometheus.MustRegister(gauge) 91 | } 92 | 93 | for _, counter := range m.Counters { 94 | prometheus.MustRegister(counter) 95 | } 96 | 97 | log.Fatal(http.ListenAndServe(MetricsPort, nil)) 98 | } 99 | 100 | func (m *MetricsServer) AddCounter(idx string, value float64) { 101 | if val, ok := m.Counters[idx]; ok { 102 | val.Add(value) 103 | } 104 | } 105 | 106 | func (m *MetricsServer) SetGauge(idx string, value float64) { 107 | if val, ok := m.Gauges[idx]; ok { 108 | val.Set(value) 109 | } 110 | } 111 | 112 | func (m *MetricsServer) IncGauge(idx string) { 113 | if val, ok := m.Gauges[idx]; ok { 114 | val.Inc() 115 | } 116 | } 117 | 118 | func (m *MetricsServer) DecGauge(idx string) { 119 | if val, ok := m.Gauges[idx]; ok { 120 | val.Dec() 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /pkg/service/metrics_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/sqs" 11 | ) 12 | 13 | func Test_Metrics(t *testing.T) { 14 | t.Log("Test_Metrics: should be able to start metrics server") 15 | var ( 16 | fakeQueueName = "my-queue" 17 | fakeMessageBody = "message-body" 18 | ) 19 | sqsStubber := &stubSQS{ 20 | FakeQueueName: fakeQueueName, 21 | FakeQueueMessages: []*sqs.Message{ 22 | { 23 | Body: aws.String(fakeMessageBody), 24 | }, 25 | }, 26 | } 27 | 28 | auth := Authenticator{ 29 | SQSClient: sqsStubber, 30 | } 31 | 32 | ctx := ManagerContext{ 33 | QueueName: "my-queue", 34 | Region: "us-west-2", 35 | PollingIntervalSeconds: 10, 36 | } 37 | 38 | mgr := New(auth, ctx) 39 | 40 | go mgr.metrics.Start() 41 | time.Sleep(2 * time.Second) 42 | 43 | endpoint := fmt.Sprintf("http://127.0.0.1%v%v", MetricsPort, MetricsEndpoint) 44 | resp, err := http.Get(endpoint) 45 | if err != nil { 46 | t.Fatalf("handleEvent: expected error not to have occured, %v", err) 47 | } 48 | 49 | expectedStatusCode := 200 50 | if resp.StatusCode != expectedStatusCode { 51 | t.Fatalf("expected status code: %v, got: %v", expectedStatusCode, resp.StatusCode) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /pkg/service/nodes.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "os/exec" 8 | "strings" 9 | "time" 10 | 11 | v1 "k8s.io/api/core/v1" 12 | apierrors "k8s.io/apimachinery/pkg/api/errors" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | drain "k8s.io/kubectl/pkg/drain" 15 | 16 | "github.com/keikoproj/lifecycle-manager/pkg/log" 17 | "k8s.io/client-go/kubernetes" 18 | ) 19 | 20 | func getNodeByInstance(k kubernetes.Interface, instanceID string) (v1.Node, bool) { 21 | var foundNode v1.Node 22 | nodes, err := k.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) 23 | if err != nil { 24 | log.Errorf("failed to list nodes: %v", err) 25 | return foundNode, false 26 | } 27 | 28 | for _, node := range nodes.Items { 29 | providerID := node.Spec.ProviderID 30 | splitProviderID := strings.Split(providerID, "/") 31 | foundID := splitProviderID[len(splitProviderID)-1] 32 | 33 | if instanceID == foundID { 34 | return node, true 35 | } 36 | } 37 | 38 | return foundNode, false 39 | } 40 | 41 | func getNodeByName(k kubernetes.Interface, nodeName string) (v1.Node, bool) { 42 | var foundNode v1.Node 43 | nodes, err := k.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) 44 | if err != nil { 45 | log.Errorf("failed to list nodes: %v", err) 46 | return foundNode, false 47 | } 48 | 49 | for _, node := range nodes.Items { 50 | if node.Name == nodeName { 51 | return node, true 52 | } 53 | } 54 | 55 | return foundNode, false 56 | } 57 | 58 | func isNodeStatusInCondition(node v1.Node, condition v1.ConditionStatus) bool { 59 | var ( 60 | conditions = node.Status.Conditions 61 | ) 62 | for _, c := range conditions { 63 | if c.Type == v1.NodeReady { 64 | if c.Status == condition { 65 | return true 66 | } 67 | } 68 | } 69 | return false 70 | } 71 | 72 | func drainNode(kubeClient kubernetes.Interface, node *v1.Node, timeout, retryInterval int64, retryAttempts uint) error { 73 | var err error = nil 74 | if timeout == 0 { 75 | log.Warn("skipping drain since timeout was set to 0") 76 | return nil 77 | } 78 | 79 | for retryAttempts > 0 { 80 | // create a copy of the node obj, since RunCordonOrUncordon() modifies the node obj 81 | nodeCopy := node.DeepCopy() 82 | err = drainNodeUtil(nodeCopy, int(timeout), kubeClient) 83 | if err == nil { 84 | log.Infof("drain succeeded, node %v", node.Name) 85 | return nil 86 | } 87 | log.Errorf("failed to drain node %v, error: %v", node.Name, err) 88 | retryAttempts -= 1 89 | if retryAttempts > 0 { 90 | log.Infof("retrying drain, node %v", node.Name) 91 | } 92 | } 93 | 94 | return err 95 | } 96 | 97 | func deleteNode(kubeClient kubernetes.Interface, node *v1.Node) error { 98 | err := deleteNodeUtil(node, kubeClient) 99 | if err != nil { 100 | log.Errorf("failed to delete node %v error: %v ", node.Name, err) 101 | return err 102 | } 103 | 104 | log.Info("node successfully deleted") 105 | return nil 106 | } 107 | 108 | func runCommand(call string, arg []string) (string, error) { 109 | log.Debugf("invoking >> %s %s", call, arg) 110 | out, err := exec.Command(call, arg...).CombinedOutput() 111 | if err != nil { 112 | log.Errorf("call failed with output: %s, error: %s", string(out), err) 113 | return string(out), err 114 | } 115 | log.Debugf("call succeeded with output: %s", string(out)) 116 | return string(out), err 117 | } 118 | 119 | func labelNode(kubectlPath, nodeName, labelKey, labelValue string) error { 120 | label := fmt.Sprintf("%v=%v", labelKey, labelValue) 121 | labelArgs := []string{"label", "--overwrite", "node", nodeName, label} 122 | _, err := runCommand(kubectlPath, labelArgs) 123 | if err != nil { 124 | log.Errorf("failed to label node %v", nodeName) 125 | return err 126 | } 127 | return nil 128 | } 129 | 130 | func annotateNode(kubectlPath string, nodeName string, annotations map[string]string) error { 131 | annotateArgs := []string{"annotate", "--overwrite", "node", nodeName} 132 | for k, v := range annotations { 133 | annotateArgs = append(annotateArgs, fmt.Sprintf("%v=%v", k, v)) 134 | } 135 | _, err := runCommand(kubectlPath, annotateArgs) 136 | if err != nil { 137 | log.Errorf("failed to annotate node %v", nodeName) 138 | return err 139 | } 140 | return nil 141 | } 142 | 143 | func getNodesByAnnotationKeys(kubeClient kubernetes.Interface, keys ...string) (map[string]map[string]string, error) { 144 | results := make(map[string]map[string]string, 0) 145 | 146 | nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) 147 | if err != nil { 148 | return results, err 149 | } 150 | 151 | for _, node := range nodes.Items { 152 | annotations := node.GetAnnotations() 153 | resultValues := make(map[string]string) 154 | for _, k := range keys { 155 | if v, ok := annotations[k]; ok { 156 | resultValues[k] = v 157 | } 158 | } 159 | if len(resultValues) > 0 { 160 | results[node.Name] = resultValues 161 | } 162 | } 163 | return results, nil 164 | } 165 | 166 | // drainNodeUtil cordons and drains a node. 167 | func drainNodeUtil(node *v1.Node, DrainTimeout int, client kubernetes.Interface) error { 168 | var err error = nil 169 | if client == nil { 170 | return fmt.Errorf("K8sClient not set") 171 | } 172 | 173 | if node == nil { 174 | return fmt.Errorf("node not set") 175 | } 176 | 177 | if _, ok := getNodeByName(client, node.Name); !ok { 178 | return fmt.Errorf("node not found") 179 | } 180 | 181 | helper := &drain.Helper{ 182 | Ctx: context.Background(), 183 | Client: client, 184 | Force: true, 185 | GracePeriodSeconds: -1, 186 | IgnoreAllDaemonSets: true, 187 | Out: os.Stdout, 188 | ErrOut: os.Stdout, 189 | DeleteEmptyDirData: true, 190 | Timeout: time.Duration(DrainTimeout) * time.Second, 191 | } 192 | 193 | if err = drain.RunCordonOrUncordon(helper, node, true); err != nil { 194 | if apierrors.IsNotFound(err) { 195 | return err 196 | } 197 | err = fmt.Errorf("error cordoning node: %v", err) 198 | return err 199 | } 200 | 201 | if err = drain.RunNodeDrain(helper, node.Name); err != nil { 202 | if apierrors.IsNotFound(err) { 203 | return err 204 | } 205 | return fmt.Errorf("error draining node: %v", err) 206 | } 207 | return err 208 | } 209 | 210 | func deleteNodeUtil(node *v1.Node, client kubernetes.Interface) error { 211 | 212 | var err error = nil 213 | 214 | if _, ok := getNodeByName(client, node.Name); !ok { 215 | return fmt.Errorf("node not found") 216 | } 217 | 218 | err = client.CoreV1().Nodes().Delete(context.Background(), node.Name, metav1.DeleteOptions{}) 219 | if err != nil { 220 | return fmt.Errorf("failed to delete node %q: %v", node.Name, err) 221 | } 222 | 223 | return nil 224 | } 225 | -------------------------------------------------------------------------------- /pkg/service/nodes_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "reflect" 6 | "testing" 7 | 8 | v1 "k8s.io/api/core/v1" 9 | apimachinery_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/kubernetes/fake" 12 | ) 13 | 14 | var ( 15 | stubKubectlPathSuccess = "echo" 16 | stubKubectlPathFail = "/bin/some-bad-file" 17 | ) 18 | 19 | func Test_NodeStatusPredicate(t *testing.T) { 20 | t.Log("Test_NodeStatusPredicate: should return true if node readiness is in given condition") 21 | 22 | readyNode := v1.Node{ 23 | Status: v1.NodeStatus{ 24 | Conditions: []v1.NodeCondition{ 25 | { 26 | Type: v1.NodeReady, 27 | Status: v1.ConditionTrue, 28 | }, 29 | }, 30 | }, 31 | } 32 | 33 | unknownNode := v1.Node{ 34 | Status: v1.NodeStatus{ 35 | Conditions: []v1.NodeCondition{ 36 | { 37 | Type: v1.NodeReady, 38 | Status: v1.ConditionUnknown, 39 | }, 40 | }, 41 | }, 42 | } 43 | 44 | if isNodeStatusInCondition(readyNode, v1.ConditionTrue) != true { 45 | t.Fatalf("expected isNodeStatusInCondition exists to be: %t, got: %t", true, false) 46 | } 47 | 48 | if isNodeStatusInCondition(unknownNode, v1.ConditionUnknown) != true { 49 | t.Fatalf("expected isNodeStatusInCondition exists to be: %t, got: %t", true, false) 50 | } 51 | 52 | } 53 | 54 | func Test_GetNodeByInstancePositive(t *testing.T) { 55 | t.Log("Test_GetNodeByInstancePositive: If a node exists, should be able to get it's instance ID") 56 | kubeClient := fake.NewSimpleClientset() 57 | fakeNodes := []v1.Node{ 58 | { 59 | Spec: v1.NodeSpec{ 60 | ProviderID: "aws:///us-west-2a/i-11111111111111111", 61 | }, 62 | }, 63 | { 64 | Spec: v1.NodeSpec{ 65 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 66 | }, 67 | }, 68 | } 69 | 70 | for _, node := range fakeNodes { 71 | kubeClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 72 | } 73 | 74 | _, exists := getNodeByInstance(kubeClient, "i-11111111111111111") 75 | expected := true 76 | 77 | if exists != expected { 78 | t.Fatalf("expected getNodeByInstance exists to be: %v, got: %v", expected, exists) 79 | } 80 | } 81 | 82 | func Test_GetNodeByInstanceNegative(t *testing.T) { 83 | t.Log("Test_GetNodeByInstanceNegative: If a node exists, should be able to get it's instance ID") 84 | kubeClient := fake.NewSimpleClientset() 85 | fakeNodes := []v1.Node{ 86 | { 87 | Spec: v1.NodeSpec{ 88 | ProviderID: "aws:///us-west-2a/i-11111111111111111", 89 | }, 90 | }, 91 | { 92 | Spec: v1.NodeSpec{ 93 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 94 | }, 95 | }, 96 | } 97 | 98 | for _, node := range fakeNodes { 99 | kubeClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 100 | } 101 | 102 | _, exists := getNodeByInstance(kubeClient, "i-3333333333333333") 103 | expected := false 104 | 105 | if exists != expected { 106 | t.Fatalf("expected getNodeByInstance exists to be: %v, got: %v", expected, exists) 107 | } 108 | } 109 | 110 | func Test_GetNodesByAnnotationKey(t *testing.T) { 111 | t.Log("Test_GetNodesByAnnotationKey: Get map of nodes annotation values by a key") 112 | kubeClient := fake.NewSimpleClientset() 113 | fakeNodes := []v1.Node{ 114 | { 115 | ObjectMeta: metav1.ObjectMeta{ 116 | Name: "node-1", 117 | Annotations: map[string]string{ 118 | "some-key": "some-value", 119 | "another-key": "another-value", 120 | }, 121 | }, 122 | Spec: v1.NodeSpec{ 123 | ProviderID: "aws:///us-west-2a/i-11111111111111111", 124 | }, 125 | }, 126 | { 127 | ObjectMeta: metav1.ObjectMeta{ 128 | Name: "node-2", 129 | Annotations: map[string]string{ 130 | "some-other-key": "some-value", 131 | "another-other-key": "another-value", 132 | }, 133 | }, 134 | Spec: v1.NodeSpec{ 135 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 136 | }, 137 | }, 138 | } 139 | 140 | for _, node := range fakeNodes { 141 | kubeClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 142 | } 143 | 144 | result, err := getNodesByAnnotationKeys(kubeClient, "some-key", "another-key") 145 | expected := map[string]map[string]string{ 146 | "node-1": { 147 | "some-key": "some-value", 148 | "another-key": "another-value", 149 | }, 150 | } 151 | 152 | if err != nil { 153 | t.Fatalf("getNodesByAnnotationKey: expected error not to have occured, %v", err) 154 | } 155 | 156 | if !reflect.DeepEqual(result, expected) { 157 | t.Fatalf("getNodesByAnnotationKey: expected: %v, got: %v", expected, result) 158 | } 159 | } 160 | 161 | func Test_DrainNodePositive(t *testing.T) { 162 | t.Log("Test_DrainNodePositive: If drain process is successful, process should exit successfully") 163 | kubeClient := fake.NewSimpleClientset() 164 | readyNode := &v1.Node{ 165 | Status: v1.NodeStatus{ 166 | Conditions: []v1.NodeCondition{ 167 | { 168 | Type: v1.NodeReady, 169 | Status: v1.ConditionTrue, 170 | }, 171 | }, 172 | }, 173 | } 174 | kubeClient.CoreV1().Nodes().Create(context.Background(), readyNode, apimachinery_v1.CreateOptions{}) 175 | err := drainNode(kubeClient, readyNode, 10, 0, 3) 176 | if err != nil { 177 | t.Fatalf("drainNode: expected error not to have occured, %v", err) 178 | } 179 | } 180 | 181 | func Test_DrainNodeNegative(t *testing.T) { 182 | t.Log("Test_DrainNodeNegative: node is not part of cluster, drainNode should return error") 183 | kubeClient := fake.NewSimpleClientset() 184 | unjoinedNode := &v1.Node{ 185 | Status: v1.NodeStatus{ 186 | Conditions: []v1.NodeCondition{ 187 | { 188 | Type: v1.NodeReady, 189 | Status: v1.ConditionUnknown, 190 | }, 191 | }, 192 | }, 193 | } 194 | 195 | err := drainNode(kubeClient, unjoinedNode, 10, 30, 3) 196 | if err == nil { 197 | t.Fatalf("drainNode: expected error to have occured, %v", err) 198 | } 199 | } 200 | 201 | func Test_RunCommand(t *testing.T) { 202 | t.Log("Test_DrainNodePositive: should successfully run command") 203 | _, err := runCommand("/bin/sleep", []string{"0"}) 204 | if err != nil { 205 | t.Fatalf("drainNode: expected error not to have occured, %v", err) 206 | } 207 | } 208 | 209 | func Test_LabelNodePositive(t *testing.T) { 210 | t.Log("Test_LabelNode: should not return an error if succesful") 211 | var ( 212 | nodeName = "some-node" 213 | ) 214 | 215 | err := labelNode(stubKubectlPathSuccess, nodeName, ExcludeLabelKey, ExcludeLabelValue) 216 | if err != nil { 217 | t.Fatalf("Test_LabelNode: expected error not to have occured, %v", err) 218 | } 219 | } 220 | 221 | func Test_LabelNodeNegative(t *testing.T) { 222 | t.Log("Test_LabelNode: should return an error if succesful") 223 | var ( 224 | nodeName = "some-node" 225 | ) 226 | 227 | err := labelNode(stubKubectlPathFail, nodeName, ExcludeLabelKey, ExcludeLabelValue) 228 | if err == nil { 229 | t.Fatalf("Test_LabelNode: expected error to have occured, %v", err) 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /pkg/service/server_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | "time" 8 | 9 | v1 "k8s.io/api/core/v1" 10 | apimachinery_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/service/autoscaling" 14 | "github.com/aws/aws-sdk-go/service/elb" 15 | "github.com/aws/aws-sdk-go/service/elbv2" 16 | "github.com/aws/aws-sdk-go/service/sqs" 17 | "github.com/pkg/errors" 18 | "golang.org/x/sync/semaphore" 19 | "k8s.io/client-go/kubernetes/fake" 20 | ) 21 | 22 | func init() { 23 | ThreadJitterRangeSeconds = 0 24 | IterationJitterRangeSeconds = 0 25 | WaiterMinDelay = 1 * time.Second 26 | WaiterMaxDelay = 2 * time.Second 27 | WaiterMaxAttempts = 3 28 | NodeAgeCacheTTL = 100 29 | } 30 | 31 | func _completeEventAfter(event *LifecycleEvent, t time.Duration) { 32 | time.Sleep(t) 33 | event.SetEventCompleted(true) 34 | } 35 | 36 | func _newBasicContext() ManagerContext { 37 | return ManagerContext{ 38 | KubectlLocalPath: stubKubectlPathSuccess, 39 | QueueName: "my-queue", 40 | Region: "us-west-2", 41 | DrainTimeoutSeconds: 1, 42 | DrainRetryAttempts: 3, 43 | PollingIntervalSeconds: 1, 44 | MaxDrainConcurrency: semaphore.NewWeighted(32), 45 | MaxTimeToProcessSeconds: 3600, 46 | } 47 | } 48 | 49 | func Test_RejectHandler(t *testing.T) { 50 | t.Log("Test_RejectHandler: should handle rejections") 51 | var ( 52 | sqsStubber = &stubSQS{} 53 | ) 54 | 55 | asgStubber := &stubAutoscaling{ 56 | lifecycleHooks: []*autoscaling.LifecycleHook{ 57 | { 58 | AutoScalingGroupName: aws.String("my-asg"), 59 | HeartbeatTimeout: aws.Int64(60), 60 | }, 61 | }, 62 | } 63 | 64 | auth := Authenticator{ 65 | ScalingGroupClient: asgStubber, 66 | SQSClient: sqsStubber, 67 | KubernetesClient: fake.NewSimpleClientset(), 68 | } 69 | ctx := _newBasicContext() 70 | 71 | fakeMessage := &sqs.Message{ 72 | // invalid instance id 73 | Body: aws.String(`{"LifecycleHookName":"my-hook","AccountId":"12345689012","RequestId":"63f5b5c2-58b3-0574-b7d5-b3162d0268f0","LifecycleTransition":"autoscaling:EC2_INSTANCE_TERMINATING","AutoScalingGroupName":"my-asg","Service":"AWS Auto Scaling","Time":"2019-09-27T02:39:14.183Z","EC2InstanceId":"","LifecycleActionToken":"cc34960c-1e41-4703-a665-bdb3e5b81ad3"}`), 74 | ReceiptHandle: aws.String("MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw="), 75 | } 76 | 77 | mgr := New(auth, ctx) 78 | _, err := mgr.newEvent(fakeMessage, "some-queue") 79 | if err == nil { 80 | t.Fatalf("expected rejected events: %v, got: %v", 1, mgr.rejectedEvents) 81 | } 82 | } 83 | 84 | func Test_FailHandler(t *testing.T) { 85 | t.Log("Test_FailHandler: should handle failures") 86 | var ( 87 | sqsStubber = &stubSQS{} 88 | ) 89 | 90 | asgStubber := &stubAutoscaling{ 91 | lifecycleHooks: []*autoscaling.LifecycleHook{ 92 | { 93 | AutoScalingGroupName: aws.String("my-asg"), 94 | HeartbeatTimeout: aws.Int64(60), 95 | }, 96 | }, 97 | } 98 | 99 | auth := Authenticator{ 100 | ScalingGroupClient: asgStubber, 101 | SQSClient: sqsStubber, 102 | KubernetesClient: fake.NewSimpleClientset(), 103 | } 104 | ctx := _newBasicContext() 105 | 106 | event := &LifecycleEvent{ 107 | LifecycleHookName: "my-hook", 108 | AccountID: "12345689012", 109 | RequestID: "63f5b5c2-58b3-0574-b7d5-b3162d0268f0", 110 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 111 | AutoScalingGroupName: "my-asg", 112 | EC2InstanceID: "i-123486890234", 113 | LifecycleActionToken: "cc34960c-1e41-4703-a665-bdb3e5b81ad3", 114 | receiptHandle: "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=", 115 | heartbeatInterval: 2, 116 | startTime: time.Now().Add(time.Duration(-1) * time.Second), 117 | } 118 | 119 | mgr := New(auth, ctx) 120 | err := errors.New("some error occured") 121 | mgr.FailEvent(err, event, true) 122 | 123 | expectedFailedEvents := 1 124 | if mgr.failedEvents != expectedFailedEvents { 125 | t.Fatalf("expected failed events: %v, got: %v", expectedFailedEvents, mgr.failedEvents) 126 | } 127 | 128 | expectedDeleteMessageEvents := 1 129 | if sqsStubber.timesCalledDeleteMessage != expectedDeleteMessageEvents { 130 | t.Fatalf("expected deleted events: %v, got: %v", expectedDeleteMessageEvents, sqsStubber.timesCalledDeleteMessage) 131 | } 132 | 133 | expectedEventCompleted := true 134 | if event.eventCompleted != expectedEventCompleted { 135 | t.Fatalf("expected event completed: %v, got: %v", expectedEventCompleted, event.eventCompleted) 136 | } 137 | } 138 | 139 | func Test_Process(t *testing.T) { 140 | t.Log("Test_Process: should process events") 141 | asgStubber := &stubAutoscaling{} 142 | sqsStubber := &stubSQS{} 143 | auth := Authenticator{ 144 | ScalingGroupClient: asgStubber, 145 | SQSClient: sqsStubber, 146 | KubernetesClient: fake.NewSimpleClientset(), 147 | } 148 | ctx := _newBasicContext() 149 | 150 | fakeNodes := []v1.Node{ 151 | { 152 | Spec: v1.NodeSpec{ 153 | ProviderID: "aws:///us-west-2a/i-123486890234", 154 | }, 155 | }, 156 | { 157 | Spec: v1.NodeSpec{ 158 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 159 | }, 160 | }, 161 | } 162 | 163 | for _, node := range fakeNodes { 164 | auth.KubernetesClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 165 | } 166 | 167 | event := &LifecycleEvent{ 168 | LifecycleHookName: "my-hook", 169 | AccountID: "12345689012", 170 | RequestID: "63f5b5c2-58b3-0574-b7d5-b3162d0268f0", 171 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 172 | AutoScalingGroupName: "my-asg", 173 | EC2InstanceID: "i-123486890234", 174 | LifecycleActionToken: "cc34960c-1e41-4703-a665-bdb3e5b81ad3", 175 | receiptHandle: "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=", 176 | heartbeatInterval: 2, 177 | } 178 | 179 | g := New(auth, ctx) 180 | g.Process(event) 181 | 182 | if event.drainCompleted != true { 183 | t.Fatal("handleEvent: expected drainCompleted to be true, got: false") 184 | } 185 | 186 | if asgStubber.timesCalledCompleteLifecycleAction != 1 { 187 | t.Fatalf("Process: expected timesCalledCompleteLifecycleAction to be 1, got: %v", asgStubber.timesCalledCompleteLifecycleAction) 188 | } 189 | 190 | if sqsStubber.timesCalledDeleteMessage != 1 { 191 | t.Fatalf("Process: expected timesCalledDeleteMessage to be 1, got: %v", sqsStubber.timesCalledDeleteMessage) 192 | } 193 | } 194 | 195 | func Test_HandleEvent(t *testing.T) { 196 | t.Log("Test_HandleEvent: should successfully handle events") 197 | asgStubber := &stubAutoscaling{} 198 | sqsStubber := &stubSQS{} 199 | auth := Authenticator{ 200 | ScalingGroupClient: asgStubber, 201 | SQSClient: sqsStubber, 202 | KubernetesClient: fake.NewSimpleClientset(), 203 | } 204 | ctx := _newBasicContext() 205 | 206 | fakeNodes := []v1.Node{ 207 | { 208 | Spec: v1.NodeSpec{ 209 | ProviderID: "aws:///us-west-2a/i-123486890234", 210 | }, 211 | }, 212 | { 213 | Spec: v1.NodeSpec{ 214 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 215 | }, 216 | }, 217 | } 218 | 219 | for _, node := range fakeNodes { 220 | auth.KubernetesClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 221 | } 222 | 223 | event := &LifecycleEvent{ 224 | LifecycleHookName: "my-hook", 225 | AccountID: "12345689012", 226 | RequestID: "63f5b5c2-58b3-0574-b7d5-b3162d0268f0", 227 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 228 | AutoScalingGroupName: "my-asg", 229 | EC2InstanceID: "i-123486890234", 230 | LifecycleActionToken: "cc34960c-1e41-4703-a665-bdb3e5b81ad3", 231 | receiptHandle: "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=", 232 | heartbeatInterval: 3, 233 | } 234 | 235 | g := New(auth, ctx) 236 | err := g.handleEvent(event) 237 | if err != nil { 238 | t.Fatalf("handleEvent: expected error not to have occured, %v", err) 239 | } 240 | 241 | if event.drainCompleted != true { 242 | t.Fatal("handleEvent: expected drainCompleted to be true, got: false") 243 | } 244 | } 245 | 246 | func Test_HandleEventWithDeregister(t *testing.T) { 247 | t.Log("Test_HandleEvent: should successfully handle events") 248 | var ( 249 | asgStubber = &stubAutoscaling{} 250 | sqsStubber = &stubSQS{} 251 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 252 | elbName = "my-classic-elb" 253 | instanceID = "i-123486890234" 254 | port int64 = 122233 255 | ) 256 | 257 | elbv2Stubber := &stubELBv2{ 258 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 259 | { 260 | Target: &elbv2.TargetDescription{ 261 | Id: aws.String(instanceID), 262 | Port: aws.Int64(port), 263 | }, 264 | TargetHealth: &elbv2.TargetHealth{ 265 | State: aws.String(elbv2.TargetHealthStateEnumUnused), 266 | }, 267 | }, 268 | }, 269 | targetGroups: []*elbv2.TargetGroup{ 270 | { 271 | TargetGroupArn: aws.String(arn), 272 | }, 273 | }, 274 | } 275 | 276 | elbStubber := &stubELB{ 277 | loadBalancerDescriptions: []*elb.LoadBalancerDescription{ 278 | { 279 | LoadBalancerName: aws.String(elbName), 280 | }, 281 | }, 282 | instanceStates: []*elb.InstanceState{ 283 | { 284 | InstanceId: aws.String(instanceID), 285 | State: aws.String("OutOfService"), 286 | }, 287 | }, 288 | } 289 | 290 | auth := Authenticator{ 291 | ScalingGroupClient: asgStubber, 292 | SQSClient: sqsStubber, 293 | ELBv2Client: elbv2Stubber, 294 | ELBClient: elbStubber, 295 | KubernetesClient: fake.NewSimpleClientset(), 296 | } 297 | 298 | ctx := _newBasicContext() 299 | ctx.WithDeregister = true 300 | ctx.DeregisterTargetTypes = []string{TargetTypeClassicELB.String(), TargetTypeTargetGroup.String()} 301 | 302 | fakeNodes := []v1.Node{ 303 | { 304 | Spec: v1.NodeSpec{ 305 | ProviderID: fmt.Sprintf("aws:///us-west-2a/%v", instanceID), 306 | }, 307 | }, 308 | { 309 | Spec: v1.NodeSpec{ 310 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 311 | }, 312 | }, 313 | } 314 | 315 | for _, node := range fakeNodes { 316 | auth.KubernetesClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 317 | } 318 | 319 | event := &LifecycleEvent{ 320 | LifecycleHookName: "my-hook", 321 | AccountID: "12345689012", 322 | RequestID: "63f5b5c2-58b3-0574-b7d5-b3162d0268f0", 323 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 324 | AutoScalingGroupName: "my-asg", 325 | EC2InstanceID: instanceID, 326 | LifecycleActionToken: "cc34960c-1e41-4703-a665-bdb3e5b81ad3", 327 | receiptHandle: "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=", 328 | heartbeatInterval: 3, 329 | } 330 | 331 | g := New(auth, ctx) 332 | err := g.handleEvent(event) 333 | if err != nil { 334 | t.Fatalf("handleEvent: expected error not to have occured, %v", err) 335 | } 336 | 337 | if event.drainCompleted != true { 338 | t.Fatal("handleEvent: expected drainCompleted to be true, got: false") 339 | } 340 | 341 | if event.deregisterCompleted != true { 342 | t.Fatal("handleEvent: expected deregisterCompleted to be true, got: false") 343 | } 344 | } 345 | 346 | func Test_HandleEventWithDeregisterError(t *testing.T) { 347 | t.Log("Test_HandleEvent: should successfully handle events") 348 | var ( 349 | asgStubber = &stubAutoscaling{} 350 | sqsStubber = &stubSQS{} 351 | arn = "arn:aws:elasticloadbalancing:us-west-2:0000000000:targetgroup/targetgroup-name/some-id" 352 | elbName = "my-classic-elb" 353 | instanceID = "i-123486890234" 354 | port int64 = 122233 355 | ) 356 | 357 | elbv2Stubber := &stubErrorELBv2{ 358 | targetHealthDescriptions: []*elbv2.TargetHealthDescription{ 359 | { 360 | Target: &elbv2.TargetDescription{ 361 | Id: aws.String(instanceID), 362 | Port: aws.Int64(port), 363 | }, 364 | TargetHealth: &elbv2.TargetHealth{ 365 | State: aws.String(elbv2.TargetHealthStateEnumUnused), 366 | }, 367 | }, 368 | }, 369 | targetGroups: []*elbv2.TargetGroup{ 370 | { 371 | TargetGroupArn: aws.String(arn), 372 | }, 373 | }, 374 | failHint: elb.ErrCodeAccessPointNotFoundException, 375 | } 376 | 377 | elbStubber := &stubErrorELB{ 378 | loadBalancerDescriptions: []*elb.LoadBalancerDescription{ 379 | { 380 | LoadBalancerName: aws.String(elbName), 381 | }, 382 | }, 383 | instanceStates: []*elb.InstanceState{ 384 | { 385 | InstanceId: aws.String(instanceID), 386 | State: aws.String("OutOfService"), 387 | }, 388 | }, 389 | failHint: "some-other-error", 390 | } 391 | 392 | auth := Authenticator{ 393 | ScalingGroupClient: asgStubber, 394 | SQSClient: sqsStubber, 395 | ELBv2Client: elbv2Stubber, 396 | ELBClient: elbStubber, 397 | KubernetesClient: fake.NewSimpleClientset(), 398 | } 399 | 400 | ctx := _newBasicContext() 401 | ctx.WithDeregister = true 402 | ctx.DeregisterTargetTypes = []string{TargetTypeClassicELB.String(), TargetTypeTargetGroup.String()} 403 | 404 | fakeNodes := []v1.Node{ 405 | { 406 | Spec: v1.NodeSpec{ 407 | ProviderID: fmt.Sprintf("aws:///us-west-2a/%v", instanceID), 408 | }, 409 | }, 410 | { 411 | Spec: v1.NodeSpec{ 412 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 413 | }, 414 | }, 415 | } 416 | 417 | for _, node := range fakeNodes { 418 | auth.KubernetesClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 419 | } 420 | 421 | event := &LifecycleEvent{ 422 | LifecycleHookName: "my-hook", 423 | AccountID: "12345689012", 424 | RequestID: "63f5b5c2-58b3-0574-b7d5-b3162d0268f0", 425 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 426 | AutoScalingGroupName: "my-asg", 427 | EC2InstanceID: instanceID, 428 | LifecycleActionToken: "cc34960c-1e41-4703-a665-bdb3e5b81ad3", 429 | receiptHandle: "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=", 430 | heartbeatInterval: 3, 431 | } 432 | 433 | g := New(auth, ctx) 434 | err := g.handleEvent(event) 435 | if err == nil { 436 | t.Fatalf("handleEvent: expected error but did not get an error") 437 | } 438 | } 439 | 440 | func Test_Poller(t *testing.T) { 441 | t.Log("Test_Poller: should deliver messages from sqs to channel") 442 | var ( 443 | fakeQueueName = "my-queue" 444 | fakeMessageBody = "message-body" 445 | fakeEventStream = make(chan *sqs.Message, 0) 446 | ) 447 | sqsStubber := &stubSQS{ 448 | FakeQueueName: fakeQueueName, 449 | FakeQueueMessages: []*sqs.Message{ 450 | { 451 | Body: aws.String(fakeMessageBody), 452 | }, 453 | }, 454 | } 455 | 456 | auth := Authenticator{ 457 | SQSClient: sqsStubber, 458 | } 459 | 460 | ctx := _newBasicContext() 461 | 462 | mgr := New(auth, ctx) 463 | mgr.eventStream = fakeEventStream 464 | 465 | go mgr.newPoller() 466 | time.Sleep(time.Duration(1) * time.Second) 467 | 468 | if sqsStubber.timesCalledReceiveMessage == 0 { 469 | t.Fatalf("expected timesCalledReceiveMessage: N>0, got: 0") 470 | } 471 | 472 | message := <-fakeEventStream 473 | if aws.StringValue(message.Body) != fakeMessageBody { 474 | t.Fatalf("expected message body: %v, got: %v", fakeMessageBody, message.Body) 475 | } 476 | } 477 | 478 | func Test_Worker(t *testing.T) { 479 | t.Log("Test_Worker: should start processing messages") 480 | var ( 481 | sqsStubber = &stubSQS{} 482 | ) 483 | 484 | asgStubber := &stubAutoscaling{ 485 | lifecycleHooks: []*autoscaling.LifecycleHook{ 486 | { 487 | AutoScalingGroupName: aws.String("my-asg"), 488 | HeartbeatTimeout: aws.Int64(60), 489 | }, 490 | }, 491 | } 492 | 493 | auth := Authenticator{ 494 | ScalingGroupClient: asgStubber, 495 | SQSClient: sqsStubber, 496 | KubernetesClient: fake.NewSimpleClientset(), 497 | } 498 | 499 | ctx := _newBasicContext() 500 | 501 | fakeNodes := []v1.Node{ 502 | { 503 | Spec: v1.NodeSpec{ 504 | ProviderID: "aws:///us-west-2a/i-123486890234", 505 | }, 506 | }, 507 | { 508 | Spec: v1.NodeSpec{ 509 | ProviderID: "aws:///us-west-2c/i-22222222222222222", 510 | }, 511 | }, 512 | } 513 | 514 | for _, node := range fakeNodes { 515 | auth.KubernetesClient.CoreV1().Nodes().Create(context.Background(), &node, apimachinery_v1.CreateOptions{}) 516 | } 517 | 518 | fakeMessage := &sqs.Message{ 519 | Body: aws.String(`{"LifecycleHookName":"my-hook","AccountId":"12345689012","RequestId":"63f5b5c2-58b3-0574-b7d5-b3162d0268f0","LifecycleTransition":"autoscaling:EC2_INSTANCE_TERMINATING","AutoScalingGroupName":"my-asg","Service":"AWS Auto Scaling","Time":"2019-09-27T02:39:14.183Z","EC2InstanceId":"i-123486890234","LifecycleActionToken":"cc34960c-1e41-4703-a665-bdb3e5b81ad3"}`), 520 | ReceiptHandle: aws.String("MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw="), 521 | } 522 | 523 | mgr := New(auth, ctx) 524 | event, err := mgr.newEvent(fakeMessage, "some-queue") 525 | if err != nil { 526 | t.Fatalf("failed to create event: %v", err) 527 | } 528 | 529 | mgr.Process(event) 530 | expectedCompletedEvents := 1 531 | 532 | if mgr.completedEvents != expectedCompletedEvents { 533 | t.Fatalf("expected completed events: %v, got: %v", expectedCompletedEvents, mgr.completedEvents) 534 | } 535 | 536 | } 537 | -------------------------------------------------------------------------------- /pkg/service/sqs.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/awserr" 8 | "github.com/aws/aws-sdk-go/service/sqs" 9 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 10 | "github.com/keikoproj/lifecycle-manager/pkg/log" 11 | ) 12 | 13 | func getQueueURLByName(s sqsiface.SQSAPI, name string) string { 14 | resultURL, err := s.GetQueueUrl(&sqs.GetQueueUrlInput{ 15 | QueueName: aws.String(name), 16 | }) 17 | if err != nil { 18 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sqs.ErrCodeQueueDoesNotExist { 19 | log.Fatalf("unable to find queue %v: %v", name, aerr) 20 | } 21 | log.Fatalf("unable to find queue %v: %v", name, err.Error()) 22 | } 23 | return aws.StringValue(resultURL.QueueUrl) 24 | } 25 | 26 | func serializeMessage(message *sqs.Message) ([]byte, error) { 27 | serialized, err := json.Marshal(message) 28 | if err != nil { 29 | return serialized, err 30 | } 31 | return serialized, nil 32 | } 33 | 34 | func deserializeMessage(message string) (*sqs.Message, error) { 35 | sqsMessage := &sqs.Message{} 36 | err := json.Unmarshal([]byte(message), sqsMessage) 37 | if err != nil { 38 | return sqsMessage, err 39 | } 40 | return sqsMessage, nil 41 | } 42 | func readMessage(message *sqs.Message, queueURL string) (*LifecycleEvent, error) { 43 | var ( 44 | event = &LifecycleEvent{} 45 | receipt = aws.StringValue(message.ReceiptHandle) 46 | body = aws.StringValue(message.Body) 47 | ) 48 | log.Debugf("reading message id=%v", aws.StringValue(message.MessageId)) 49 | err := json.Unmarshal([]byte(body), event) 50 | if err != nil { 51 | return event, err 52 | } 53 | event.SetReceiptHandle(receipt) 54 | event.SetQueueURL(queueURL) 55 | event.SetMessage(message) 56 | log.Debugf("unmarshalling event with message body %v", aws.StringValue(message.Body)) 57 | return event, nil 58 | } 59 | 60 | func deleteMessage(sqsClient sqsiface.SQSAPI, url, receiptHandle string) error { 61 | log.Debugf("deleting message with receipt ID %v", receiptHandle) 62 | input := &sqs.DeleteMessageInput{ 63 | QueueUrl: aws.String(url), 64 | ReceiptHandle: aws.String(receiptHandle), 65 | } 66 | _, err := sqsClient.DeleteMessage(input) 67 | if err != nil { 68 | return err 69 | } 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/service/sqs_test.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/service/sqs" 11 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 12 | ) 13 | 14 | type stubSQS struct { 15 | sqsiface.SQSAPI 16 | FakeQueueMessages []*sqs.Message 17 | FakeQueueName string 18 | timesCalledReceiveMessage int 19 | timesCalledDeleteMessage int 20 | timesCalledGetQueueUrl int 21 | } 22 | 23 | func (s *stubSQS) ReceiveMessage(input *sqs.ReceiveMessageInput) (*sqs.ReceiveMessageOutput, error) { 24 | s.timesCalledReceiveMessage++ 25 | if len(s.FakeQueueMessages) != 0 { 26 | return &sqs.ReceiveMessageOutput{Messages: s.FakeQueueMessages}, nil 27 | } 28 | return &sqs.ReceiveMessageOutput{}, nil 29 | } 30 | 31 | func (s *stubSQS) DeleteMessage(input *sqs.DeleteMessageInput) (*sqs.DeleteMessageOutput, error) { 32 | s.timesCalledDeleteMessage++ 33 | return &sqs.DeleteMessageOutput{}, nil 34 | } 35 | 36 | func (a *stubSQS) GetQueueUrl(input *sqs.GetQueueUrlInput) (*sqs.GetQueueUrlOutput, error) { 37 | a.timesCalledGetQueueUrl++ 38 | queueURL := fmt.Sprintf("https://queue.amazonaws.com/80398EXAMPLE/%v", a.FakeQueueName) 39 | 40 | if aws.StringValue(input.QueueName) == a.FakeQueueName { 41 | return &sqs.GetQueueUrlOutput{QueueUrl: aws.String(queueURL)}, nil 42 | } 43 | return &sqs.GetQueueUrlOutput{}, nil 44 | } 45 | 46 | func _quitPollerAfter(quitter chan bool, seconds int64) { 47 | time.Sleep(time.Duration(seconds)*time.Second + time.Duration(500)*time.Millisecond) 48 | quitter <- true 49 | } 50 | 51 | func Test_DesserializeMessage(t *testing.T) { 52 | t.Log("Test_DesserializeMessage: should be able to deserialize a string to sqs Message") 53 | message := `{"Attributes":{"SenderId":"ABCD:efg"},"Body":"{\"LifecycleHookName\":\"test\",\"AccountId\":\"0000000\",\"RequestId\":\"c2281dfd\",\"LifecycleTransition\":\"autoscaling:EC2_INSTANCE_TERMINATING\",\"AutoScalingGroupName\":\"some-asg\",\"Service\":\"AWS Auto Scaling\",\"Time\":\"2020-01-15T03:54:51.913Z\",\"EC2InstanceId\":\"i-0000000000\",\"LifecycleActionToken\":\"c7b2144c\"}","MD5OfBody":"123123123123","MD5OfMessageAttributes":null,"MessageAttributes":null,"MessageId":"c643f9fc","ReceiptHandle":"AQEBVkU="}` 54 | sqsMessage, err := deserializeMessage(message) 55 | if err != nil { 56 | t.Fatalf("deserializeMessage: expected error not to have occured, %v", err) 57 | } 58 | if aws.StringValue(sqsMessage.Body) == "" { 59 | t.Fatalf("deserializeMessage: expected sqsMessage.Body not to be empty") 60 | } 61 | } 62 | 63 | func Test_GetQueueURLByNamePositive(t *testing.T) { 64 | t.Log("Test_GetQueueURLByName: should be able to fetch queue URL by it's name") 65 | fakeQueueName := "my-queue" 66 | stubber := &stubSQS{ 67 | FakeQueueName: fakeQueueName, 68 | } 69 | url := getQueueURLByName(stubber, fakeQueueName) 70 | expectedURL := fmt.Sprintf("https://queue.amazonaws.com/80398EXAMPLE/%v", fakeQueueName) 71 | expectedTimesCalled := 1 72 | if url != expectedURL { 73 | t.Fatalf("expected getQueueURLByName: %v, got: %v", expectedURL, url) 74 | } 75 | 76 | if stubber.timesCalledGetQueueUrl != expectedTimesCalled { 77 | t.Fatalf("expected timesCalledGetQueueUrl: %v, got: %v", expectedTimesCalled, stubber.timesCalledGetQueueUrl) 78 | } 79 | } 80 | 81 | func Test_GetQueueURLByNameNegative(t *testing.T) { 82 | t.Log("Test_GetQueueURLByNameNegative: should return empty string if failed to find URL") 83 | fakeQueueName := "my-queue" 84 | stubber := &stubSQS{ 85 | FakeQueueName: "other-queue", 86 | } 87 | url := getQueueURLByName(stubber, fakeQueueName) 88 | expectedURL := "" 89 | expectedTimesCalled := 1 90 | if url != expectedURL { 91 | t.Fatalf("expected getQueueURLByName: %v, got: %v", expectedURL, url) 92 | } 93 | 94 | if stubber.timesCalledGetQueueUrl != expectedTimesCalled { 95 | t.Fatalf("expected timesCalledGetQueueUrl: %v, got: %v", expectedTimesCalled, stubber.timesCalledGetQueueUrl) 96 | } 97 | } 98 | 99 | func Test_DeleteMessage(t *testing.T) { 100 | t.Log("Test_DeleteMessage: should delete a message from queue") 101 | fakeQueueName := "my-queue" 102 | fakeReceiptHandle := "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=" 103 | stubber := &stubSQS{ 104 | FakeQueueName: fakeQueueName, 105 | } 106 | url := getQueueURLByName(stubber, fakeQueueName) 107 | err := deleteMessage(stubber, url, fakeReceiptHandle) 108 | if err != nil { 109 | t.Fatalf("deleteMessage: expected error not to have occured, %v", err) 110 | } 111 | expectedTimesCalled := 1 112 | 113 | if stubber.timesCalledDeleteMessage != expectedTimesCalled { 114 | t.Fatalf("expected timesCalledDeleteMessage: %v, got: %v", expectedTimesCalled, stubber.timesCalledDeleteMessage) 115 | } 116 | } 117 | 118 | func Test_ReadMessage(t *testing.T) { 119 | t.Log("Test_ReadMessage: should unmarshal a message into a LifecycleEvent") 120 | expectedLifecycleEvent := &LifecycleEvent{ 121 | LifecycleHookName: "my-hook", 122 | AccountID: "12345689012", 123 | RequestID: "63f5b5c2-58b3-0574-b7d5-b3162d0268f0", 124 | LifecycleTransition: "autoscaling:EC2_INSTANCE_TERMINATING", 125 | AutoScalingGroupName: "my-asg", 126 | EC2InstanceID: "i-123486890234", 127 | LifecycleActionToken: "cc34960c-1e41-4703-a665-bdb3e5b81ad3", 128 | receiptHandle: "MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw=", 129 | queueURL: "some-queue", 130 | } 131 | fakeMessage := &sqs.Message{ 132 | Body: aws.String(`{"LifecycleHookName":"my-hook","AccountId":"12345689012","RequestId":"63f5b5c2-58b3-0574-b7d5-b3162d0268f0","LifecycleTransition":"autoscaling:EC2_INSTANCE_TERMINATING","AutoScalingGroupName":"my-asg","Service":"AWS Auto Scaling","Time":"2019-09-27T02:39:14.183Z","EC2InstanceId":"i-123486890234","LifecycleActionToken":"cc34960c-1e41-4703-a665-bdb3e5b81ad3"}`), 133 | ReceiptHandle: aws.String("MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+Cw="), 134 | } 135 | expectedLifecycleEvent.SetMessage(fakeMessage) 136 | 137 | event, err := readMessage(fakeMessage, "some-queue") 138 | if err != nil { 139 | t.Fatalf("readMessage: expected error not to have occured, %v", err) 140 | } 141 | 142 | if !reflect.DeepEqual(expectedLifecycleEvent, event) { 143 | t.Fatalf("readMessage: expected event: %+v got: %+v", expectedLifecycleEvent, event) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /pkg/service/target.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | type TargetType string 4 | 5 | func (t TargetType) String() string { 6 | return string(t) 7 | } 8 | 9 | const ( 10 | // TargetTypeClassicELB defines a target type of classic elb 11 | TargetTypeClassicELB TargetType = "classic-elb" 12 | // TargetTypeTargetGroup defines a target type of target-group 13 | TargetTypeTargetGroup TargetType = "target-group" 14 | ) 15 | 16 | // Target defines a deregistration target 17 | type Target struct { 18 | Type TargetType 19 | TargetId string 20 | InstanceId string 21 | Port int64 22 | } 23 | 24 | func (m *Manager) NewTarget(targetId, instanceId string, port int64, targetType TargetType) *Target { 25 | return &Target{ 26 | TargetId: targetId, 27 | InstanceId: instanceId, 28 | Port: port, 29 | Type: targetType, 30 | } 31 | } 32 | 33 | // LoadTargets loads a list of targets by key 34 | func (m *Manager) LoadTargets(key interface{}) []*Target { 35 | targets := []*Target{} 36 | if val, ok := m.targets.Load(key); ok { 37 | return val.([]*Target) 38 | } 39 | m.SetTargets(key, targets) 40 | return targets 41 | } 42 | 43 | // SetTargets sets targets to a specific key 44 | func (m *Manager) SetTargets(key interface{}, targets []*Target) { 45 | m.targets.Store(key, targets) 46 | } 47 | 48 | // GetTargetInstanceIds gets instance ids for a specific key 49 | func (m *Manager) GetTargetInstanceIds(key interface{}) []string { 50 | list := []string{} 51 | for _, target := range m.LoadTargets(key) { 52 | list = append(list, target.InstanceId) 53 | } 54 | return list 55 | } 56 | 57 | // GetTargetMapping gets instanceID>port mapping for a specific key 58 | func (m *Manager) GetTargetMapping(key interface{}) map[string]int64 { 59 | mapping := map[string]int64{} 60 | for _, target := range m.LoadTargets(key) { 61 | mapping[target.InstanceId] = target.Port 62 | } 63 | return mapping 64 | } 65 | 66 | // AddTargetByInstance adds a target by it's instance ID 67 | func (m *Manager) AddTargetByInstance(key interface{}, add *Target) { 68 | targets := m.LoadTargets(key) 69 | newTargets := []*Target{} 70 | found := false 71 | for _, target := range targets { 72 | if target.InstanceId == add.InstanceId { 73 | found = true 74 | newTargets = append(newTargets, add) 75 | } else { 76 | newTargets = append(newTargets, target) 77 | } 78 | } 79 | if !found { 80 | newTargets = append(newTargets, add) 81 | } 82 | m.SetTargets(key, newTargets) 83 | } 84 | 85 | // RemoveTargetByInstance removes a target by it's instance ID 86 | func (m *Manager) RemoveTargetByInstance(key interface{}, instanceID string) { 87 | targets := m.LoadTargets(key) 88 | newTargets := []*Target{} 89 | for _, target := range targets { 90 | if target.InstanceId != instanceID { 91 | newTargets = append(newTargets, target) 92 | } 93 | } 94 | m.SetTargets(key, newTargets) 95 | } 96 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | // The git commit that was compiled. This will be filled in by the compiler. 9 | var GitCommit string 10 | 11 | // The main version number that is being run at the moment. 12 | const Version = "0.6.3" 13 | 14 | var BuildDate = "" 15 | 16 | var GoVersion = runtime.Version() 17 | 18 | var OsArch = fmt.Sprintf("%s %s", runtime.GOOS, runtime.GOARCH) 19 | --------------------------------------------------------------------------------