├── .github ├── dependabot.yml └── workflows │ ├── container-image.yml │ └── helm.yml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── chaoskube.png ├── chaoskube ├── chaoskube.go └── chaoskube_test.go ├── chart └── chaoskube │ ├── .helmignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── clusterrole.yaml │ ├── clusterrolebinding.yaml │ ├── deployment.yaml │ └── serviceaccount.yaml │ └── values.yaml ├── examples ├── deployment │ └── chaoskube.yaml ├── job │ └── chaoskube.yaml └── rbac.yaml ├── go.mod ├── go.sum ├── internal └── testutil │ └── assert.go ├── main.go ├── metrics └── metrics.go ├── notifier ├── noop.go ├── notifier.go ├── notifier_test.go ├── slack.go └── slack_test.go ├── terminator ├── delete_pod.go ├── delete_pod_test.go └── terminator.go └── util ├── util.go └── util_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: monthly 7 | time: "07:00" 8 | open-pull-requests-limit: 10 9 | - package-ecosystem: docker 10 | directory: "/" 11 | schedule: 12 | interval: monthly 13 | time: "07:00" 14 | open-pull-requests-limit: 10 15 | - package-ecosystem: github-actions 16 | directory: "/" 17 | schedule: 18 | interval: monthly 19 | time: "07:00" 20 | open-pull-requests-limit: 10 21 | -------------------------------------------------------------------------------- /.github/workflows/container-image.yml: -------------------------------------------------------------------------------- 1 | name: container-image 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | tags: 8 | - 'v*' 9 | pull_request: 10 | branches: 11 | - 'master' 12 | 13 | env: 14 | REGISTRY: ghcr.io 15 | IMAGE_NAME: ${{ github.repository }} 16 | 17 | jobs: 18 | build-and-push: 19 | runs-on: ubuntu-latest 20 | 21 | permissions: 22 | contents: read 23 | packages: write 24 | 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v4 28 | 29 | - name: Get all git tags 30 | run: git fetch --prune --unshallow --tags 31 | 32 | - name: Set up QEMU 33 | uses: docker/setup-qemu-action@v3 34 | 35 | - name: Set up Docker Buildx 36 | uses: docker/setup-buildx-action@v3 37 | 38 | - name: Login to GitHub Container Registry 39 | if: github.event_name != 'pull_request' 40 | uses: docker/login-action@v3 41 | with: 42 | registry: ghcr.io 43 | username: ${{ github.actor }} 44 | password: ${{ secrets.GITHUB_TOKEN }} 45 | 46 | - name: Docker meta 47 | uses: docker/metadata-action@v5 48 | id: meta 49 | with: 50 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 51 | tags: | 52 | type=ref,event=branch 53 | type=ref,event=pr 54 | type=semver,pattern=v{{version}} 55 | type=semver,pattern=v{{major}}.{{minor}} 56 | 57 | - name: Build and push 58 | uses: docker/build-push-action@v6 59 | with: 60 | context: . 61 | platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/ppc64le 62 | push: ${{ github.event_name != 'pull_request' }} 63 | tags: ${{ steps.meta.outputs.tags }} 64 | labels: ${{ steps.meta.outputs.labels }} 65 | -------------------------------------------------------------------------------- /.github/workflows/helm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: helm 3 | on: 4 | push: 5 | branches: 6 | - master 7 | jobs: 8 | changes: 9 | runs-on: ubuntu-latest 10 | outputs: 11 | chart: ${{ steps.filter.outputs.chart }} 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v4 15 | - uses: dorny/paths-filter@v3 16 | id: filter 17 | with: 18 | token: ${{ secrets.GITHUB_TOKEN }} 19 | filters: | 20 | chart: 21 | - 'chart/chaoskube/Chart.yaml' 22 | - 'chart/chaoskube/**/*' 23 | chart: 24 | name: release chart 25 | runs-on: ubuntu-latest 26 | needs: 27 | - changes 28 | if: | 29 | needs.changes.outputs.chart == 'true' 30 | steps: 31 | - name: Checkout master 32 | uses: actions/checkout@v4 33 | with: 34 | # Fetch entire history. Required for chart-releaser; see https://github.com/helm/chart-releaser-action/issues/13#issuecomment-602063896 35 | fetch-depth: 0 36 | - name: Configure Git 37 | run: | 38 | git config user.name "$GITHUB_ACTOR" 39 | git config user.email "$GITHUB_ACTOR@users.noreply.github.com" 40 | - name: Run chart-releaser 41 | uses: helm/chart-releaser-action@v1.7.0 42 | env: 43 | CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 44 | CR_RELEASE_NAME_TEMPLATE: "helm-chart-{{ .Version }}" 45 | with: 46 | charts_dir: chart -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | [568vq].out 12 | 13 | *.cgo1.go 14 | *.cgo2.c 15 | _cgo_defun.c 16 | _cgo_gotypes.go 17 | _cgo_export.* 18 | 19 | _testmain.go 20 | 21 | *.exe 22 | *.test 23 | *.prof 24 | 25 | /vendor 26 | /bin 27 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Updates: 2 | * [#595](https://github.com/linki/chaoskube/pull/595) Update builds to use Go `v1.22` @linki 3 | * [#568](https://github.com/linki/chaoskube/pull/568) Update builds to use Go `v1.21` @linki 4 | * [#528](https://github.com/linki/chaoskube/pull/528) Update builds to use Go `v1.20` @linki 5 | * [#505](https://github.com/linki/chaoskube/pull/505) Update builds to use Go `v1.19` @linki 6 | 7 | ## v0.22.0 - 2021-11-05 8 | 9 | Features: 10 | * [#325](https://github.com/linki/chaoskube/pull/325) Add maximum runtime flag @IoannisMatzaris 11 | * [#295](https://github.com/linki/chaoskube/pull/295) Add a Helm chart to deploy chaoskube @ghouscht 12 | 13 | Updates: 14 | * [#374](https://github.com/linki/chaoskube/pull/374) Add kubeinvaders as related project in the docs @lucky-sideburn 15 | * [#341](https://github.com/linki/chaoskube/pull/341) Add namespace metadata to deleted pods metric @linki 16 | * [#323](https://github.com/linki/chaoskube/pull/323) Add missing max-kill flag to the docs @KeisukeYamashita 17 | * Update several dependencies, such as Kubernetes to v0.20.x. 18 | 19 | ## v0.21.0 - 2020-09-28 20 | 21 | Features: 22 | * [#224](https://github.com/linki/chaoskube/pull/224) Allow to filter by OwnerReference's Kind @qlikcoe 23 | 24 | Updates: 25 | * [#248](https://github.com/linki/chaoskube/pull/248) Added test & build to Makefile @el10savio 26 | 27 | ## v0.20.0 - 2020-07-03 28 | 29 | Updates: 30 | * [#197](https://github.com/linki/chaoskube/pull/197) [#203](https://github.com/linki/chaoskube/pull/203) Fix a bug that caused chaoskube to always kill the same pod of a replicated group of pods @HaveFun83 @linki 31 | 32 | ## v0.19.0 - 2020-04-02 33 | 34 | Updates: 35 | * [#192](https://github.com/linki/chaoskube/pull/192) Use `context.Context` to cancel in-flight requests @linki 36 | * [#191](https://github.com/linki/chaoskube/pull/191) Update client-go to `v0.18.0` @linki 37 | * [#180](https://github.com/linki/chaoskube/pull/180) Update builds to use Go `v1.14` @linki 38 | 39 | ## v0.18.0 - 2020-02-03 40 | 41 | Updates: 42 | * [#170](https://github.com/linki/chaoskube/pull/170) Add slack webhook flag to documentation @Clivern 43 | * [#169](https://github.com/linki/chaoskube/pull/169) Update client-go to v0.17.0 @linki 44 | * [#167](https://github.com/linki/chaoskube/pull/167) Add Makefile and prettify test output @linki 45 | * [#166](https://github.com/linki/chaoskube/pull/166) Update klog to v1.0.0 @linki 46 | * [#164](https://github.com/linki/chaoskube/pull/164) Update Helm's Quickstart Guide link in README @SergioSV96 47 | 48 | ## v0.17.0 - 2019-12-09 49 | 50 | Features: 51 | * [#158](https://github.com/linki/chaoskube/pull/158) Support for sending Slack notifications @GaruGaru 52 | 53 | ## v0.16.0 - 2019-11-08 54 | 55 | Features: 56 | * [#154](https://github.com/linki/chaoskube/pull/154) Add support for terminating multiple pods per iteration @pims 57 | 58 | Updates: 59 | * [#156](https://github.com/linki/chaoskube/pull/156) Remove incomplete snippet from the readme and point to examples @jan-warchol 60 | * [#153](https://github.com/linki/chaoskube/pull/153) Don't attempt to terminate `Terminating` pods @pims 61 | * [#148](https://github.com/linki/chaoskube/pull/148) Update builds to use Go `v1.13` @linki 62 | * [#140](https://github.com/linki/chaoskube/pull/140) Update Docker images to use alpine `3.10` @linki 63 | 64 | ## v0.15.1 - 2019-08-09 65 | 66 | Updates: 67 | * [#137](https://github.com/linki/chaoskube/pull/137) [#138](https://github.com/linki/chaoskube/pull/138) Avoid writing logs to the container filesystem @linki 68 | 69 | ## v0.15.0 - 2019-07-30 70 | 71 | Features: 72 | * [#130](https://github.com/linki/chaoskube/pull/130) Add `--log-caller` flag that adds file name and line to the log output @linki 73 | 74 | Updates: 75 | * [#129](https://github.com/linki/chaoskube/pull/129) Update client-go to `v12` for Kubernetes `v1.14` @linki 76 | * [#126](https://github.com/linki/chaoskube/pull/126) Update builds to use Go `v1.12` and Go Modules @linki 77 | 78 | ## v0.14.0 - 2019-05-20 79 | 80 | Features: 81 | * [#121](https://github.com/linki/chaoskube/pull/121) Add include and exclude regular expression filters for pod names @dansimone 82 | 83 | ## v0.13.0 - 2019-01-30 84 | 85 | Features: 86 | * [#120](https://github.com/linki/chaoskube/pull/120) Adding JSON as additional log format @syedimam0012 87 | 88 | ## v0.12.1 - 2019-01-20 89 | 90 | Updates: 91 | * [#119](https://github.com/linki/chaoskube/pull/119) Add logo for chaoskube @linki 92 | * [#118](https://github.com/linki/chaoskube/pull/118) [#81](https://github.com/linki/chaoskube/pull/81) Add Dockerfile for `arm32v6` and `arm64v8` @toolboc 93 | * [#117](https://github.com/linki/chaoskube/pull/117) [#104](https://github.com/linki/chaoskube/pull/104) Abstract termination strategy in order to add more means of killing pods @jakewins @linki 94 | 95 | ## v0.12.0 - 2019-01-08 96 | 97 | Features: 98 | * [#116](https://github.com/linki/chaoskube/pull/116) Add several useful Prometheus metrics @ruehowl @shaikatz 99 | 100 | Updates: 101 | * [#115](https://github.com/linki/chaoskube/pull/115) Replace event related code with Kubernetes's `EventRecorder` @linki 102 | * [#114](https://github.com/linki/chaoskube/pull/114) Document major difference to `kube-monkey` @prageethw 103 | * [#113](https://github.com/linki/chaoskube/pull/113) Update dependencies to match Kubernetes v1.12 API @linki 104 | * [#112](https://github.com/linki/chaoskube/pull/112) Update docker image to alpine v3.8 and go v1.11 @linki 105 | 106 | ## v0.11.0 - 2018-10-09 107 | 108 | Features: 109 | * [#110](https://github.com/linki/chaoskube/pull/110) Add option to define grace period given to pods @palmerabollo 110 | * [#105](https://github.com/linki/chaoskube/pull/105) Implement event creation after terminating a pod @djboris9 111 | 112 | Updates: 113 | * [#107](https://github.com/linki/chaoskube/pull/107) Replace `glog` with a `noop` logger to allow for read-only filesystem @linki 114 | 115 | ## v0.10.0 - 2018-08-06 116 | 117 | Features: 118 | * [#97](https://github.com/linki/chaoskube/pull/97) Expose basic metrics via Prometheus @bakins 119 | * [#94](https://github.com/linki/chaoskube/pull/94) Add a health check endpoint @bakins 120 | * [#86](https://github.com/linki/chaoskube/pull/86) Add a flag to exclude Pods under a certain age @bakins 121 | * [#84](https://github.com/linki/chaoskube/pull/84) Exclude Pods that are not in phase `Running` @linki 122 | * [#60](https://github.com/linki/chaoskube/pull/60) Add a Dockerfile for building images for `ppc64le` @hchenxa 123 | 124 | Updates: 125 | * [#96](https://github.com/linki/chaoskube/pull/96) Use versioned functions of `client-go` @linki 126 | * [#95](https://github.com/linki/chaoskube/pull/95) Handle signals to enable more graceful shutdown @linki 127 | * [#89](https://github.com/linki/chaoskube/pull/89) Run `chaoskube` as `nobody` by default @bavarianbidi 128 | * [#77](https://github.com/linki/chaoskube/pull/77) Use [roveralls](https://github.com/lawrencewoodman/roveralls) to improve coverage results @linki 129 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # builder image 2 | FROM golang:1.24.2-alpine3.21 AS builder 3 | 4 | ENV CGO_ENABLED=0 5 | RUN apk --no-cache add alpine-sdk 6 | WORKDIR /go/src/github.com/linki/chaoskube 7 | COPY . . 8 | RUN GODEBUG=randseednop=0 go test -v ./... 9 | RUN go build -o /usr/local/bin/chaoskube -v \ 10 | -ldflags "-X main.version=$(git describe --tags --always --dirty) -w -s" 11 | RUN /usr/local/bin/chaoskube --version 12 | 13 | # final image 14 | FROM alpine:3.21.3 15 | 16 | RUN apk --no-cache add ca-certificates dumb-init tzdata 17 | COPY --from=builder /usr/local/bin/chaoskube /usr/local/bin/chaoskube 18 | 19 | USER 65534 20 | ENTRYPOINT ["dumb-init", "--", "/usr/local/bin/chaoskube"] 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Martin Linkhorst 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TPARSE := $(shell tparse --version 2>/dev/null) 2 | 3 | check: 4 | ifdef TPARSE 5 | GODEBUG=randseednop=0 go test ./... -race -cover -json | tparse -all 6 | else 7 | GODEBUG=randseednop=0 go test ./... -race -cover 8 | endif 9 | 10 | test: check 11 | 12 | build: 13 | go build -o bin/chaoskube -v 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # chaoskube 2 | [![GitHub release](https://img.shields.io/github/release/linki/chaoskube.svg)](https://github.com/linki/chaoskube/releases) 3 | [![go-doc](https://godoc.org/github.com/linki/chaoskube/chaoskube?status.svg)](https://godoc.org/github.com/linki/chaoskube/chaoskube) 4 | 5 | `chaoskube` periodically kills random pods in your Kubernetes cluster. 6 | 7 |

chaoskube

8 | 9 | ## Why 10 | 11 | Test how your system behaves under arbitrary pod failures. 12 | 13 | ## Example 14 | 15 | Running it will kill a pod in any namespace every 10 minutes by default. 16 | 17 | ```console 18 | $ chaoskube 19 | INFO[0000] starting up dryRun=true interval=10m0s version=v0.21.0 20 | INFO[0000] connecting to cluster master="https://kube.you.me" serverVersion=v1.10.5+coreos.0 21 | INFO[0000] setting pod filter annotations= labels= minimumAge=0s namespaces= 22 | INFO[0000] setting quiet times daysOfYear="[]" timesOfDay="[]" weekdays="[]" 23 | INFO[0000] setting timezone location=UTC name=UTC offset=0 24 | INFO[0001] terminating pod name=kube-dns-v20-6ikos namespace=kube-system 25 | INFO[0601] terminating pod name=nginx-701339712-u4fr3 namespace=chaoskube 26 | INFO[1201] terminating pod name=kube-proxy-gke-earthcoin-pool-3-5ee87f80-n72s namespace=kube-system 27 | INFO[1802] terminating pod name=nginx-701339712-bfh2y namespace=chaoskube 28 | INFO[2402] terminating pod name=heapster-v1.2.0-1107848163-bhtcw namespace=kube-system 29 | INFO[3003] terminating pod name=l7-default-backend-v1.0-o2hc9 namespace=kube-system 30 | INFO[3603] terminating pod name=heapster-v1.2.0-1107848163-jlfcd namespace=kube-system 31 | INFO[4203] terminating pod name=nginx-701339712-bfh2y namespace=chaoskube 32 | INFO[4804] terminating pod name=nginx-701339712-51nt8 namespace=chaoskube 33 | ... 34 | ``` 35 | 36 | `chaoskube` allows to filter target pods [by namespaces, labels, annotations and age](#filtering-targets) as well as [exclude certain weekdays, times of day and days of a year](#limit-the-chaos) from chaos. 37 | 38 | ## How 39 | 40 | ### Helm 41 | 42 | You can install `chaoskube` with [`Helm`](https://github.com/kubernetes/helm). Follow [Helm's Quickstart Guide](https://helm.sh/docs/intro/quickstart/) and then install the `chaoskube` chart. 43 | 44 | ```console 45 | $ helm repo add chaoskube https://linki.github.io/chaoskube/ 46 | $ helm install chaoskube chaoskube/chaoskube --atomic --namespace=chaoskube --create-namespace 47 | ``` 48 | 49 | Refer to [chaoskube on kubeapps.com](https://kubeapps.com/charts/stable/chaoskube) to learn how to configure it and to find other useful Helm charts. 50 | 51 | ### Raw manifest 52 | 53 | Refer to [example manifest](./examples/). Be sure to give chaoskube appropriate 54 | permissions using provided ClusterRole. 55 | 56 | ### Configuration 57 | 58 | By default `chaoskube` will be friendly and not kill anything. When you validated your target cluster you may disable dry-run mode by passing the flag `--no-dry-run`. You can also specify a more aggressive interval and other supported flags for your deployment. 59 | 60 | If you're running in a Kubernetes cluster and want to target the same cluster then this is all you need to do. 61 | 62 | If you want to target a different cluster or want to run it locally specify your cluster via the `--master` flag or provide a valid kubeconfig via the `--kubeconfig` flag. By default, it uses your standard kubeconfig path in your home. That means, whatever is the current context in there will be targeted. 63 | 64 | If you want to increase or decrease the amount of chaos change the interval between killings with the `--interval` flag. Alternatively, you can increase the number of replicas of your `chaoskube` deployment. 65 | 66 | Remember that `chaoskube` by default kills any pod in all your namespaces, including system pods and itself. 67 | 68 | `chaoskube` provides a simple HTTP endpoint that can be used to check that it is running. This can be used for [Kubernetes liveness and readiness probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/). By default, this listens on port 8080. To disable, pass `--metrics-address=""` to `chaoskube`. 69 | 70 | ## Filtering targets 71 | 72 | However, you can limit the search space of `chaoskube` by providing label, annotation, and namespace selectors, pod name include/exclude patterns, as well as a minimum age setting. 73 | 74 | ```console 75 | $ chaoskube --labels 'app=mate,chaos,stage!=production' 76 | ... 77 | INFO[0000] setting pod filter labels="app=mate,chaos,stage!=production" 78 | ``` 79 | 80 | This selects all pods that have the label `app` set to `mate`, the label `chaos` set to anything and the label `stage` not set to `production` or unset. 81 | 82 | You can filter target pods by namespace selector as well. 83 | 84 | ```console 85 | $ chaoskube --namespaces 'default,testing,staging' 86 | ... 87 | INFO[0000] setting pod filter namespaces="default,staging,testing" 88 | ``` 89 | 90 | This will filter for pods in the three namespaces `default`, `staging` and `testing`. 91 | 92 | Namespaces can additionally be filtered by a namespace label selector. 93 | 94 | ```console 95 | $ chaoskube --namespace-labels='!integration' 96 | ... 97 | INFO[0000] setting pod filter namespaceLabels="!integration" 98 | ``` 99 | 100 | This will exclude all pods from namespaces with the label `integration`. 101 | 102 | You can filter target pods by [OwnerReference's](https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#OwnerReference) kind selector. 103 | 104 | ```console 105 | $ chaoskube --kinds '!DaemonSet,!StatefulSet' 106 | ... 107 | INFO[0000] setting pod filter kinds="!DaemonSet,!StatefulSet" 108 | ``` 109 | 110 | This will exclude any `DaemonSet` and `StatefulSet` pods. 111 | 112 | ```console 113 | $ chaoskube --kinds 'DaemonSet' 114 | ... 115 | INFO[0000] setting pod filter kinds="DaemonSet" 116 | ``` 117 | 118 | This will only include any `DaemonSet` pods. 119 | 120 | Please note: any `include` filter will automatically exclude all the pods with no OwnerReference defined. 121 | 122 | You can filter pods by name: 123 | 124 | ```console 125 | $ chaoskube --included-pod-names 'foo|bar' --excluded-pod-names 'prod' 126 | ... 127 | INFO[0000] setting pod filter excludedPodNames=prod includedPodNames="foo|bar" 128 | ``` 129 | 130 | This will cause only pods whose name contains 'foo' or 'bar' and does _not_ contain 'prod' to be targeted. 131 | 132 | You can also exclude namespaces and mix and match with the label and annotation selectors. 133 | 134 | ```console 135 | $ chaoskube \ 136 | --labels 'app=mate,chaos,stage!=production' \ 137 | --annotations '!scheduler.alpha.kubernetes.io/critical-pod' \ 138 | --namespaces '!kube-system,!production' 139 | ... 140 | INFO[0000] setting pod filter annotations="!scheduler.alpha.kubernetes.io/critical-pod" labels="app=mate,chaos,stage!=production" namespaces="!kube-system,!production" 141 | ``` 142 | 143 | This further limits the search space of the above label selector by also excluding any pods in the `kube-system` and `production` namespaces as well as ignore all pods that are marked as critical. 144 | 145 | The annotation selector can also be used to run `chaoskube` as a cluster addon and allow pods to opt-in to being terminated as you see fit. For example, you could run `chaoskube` like this: 146 | 147 | ```console 148 | $ chaoskube --annotations 'chaos.alpha.kubernetes.io/enabled=true' --debug 149 | ... 150 | INFO[0000] setting pod filter annotations="chaos.alpha.kubernetes.io/enabled=true" 151 | DEBU[0000] found candidates count=0 152 | DEBU[0000] no victim found 153 | ``` 154 | 155 | Unless you already use that annotation somewhere, this will initially ignore all of your pods (you can see the number of candidates in debug mode). You could then selectively opt-in individual deployments to chaos mode by annotating their pods with `chaos.alpha.kubernetes.io/enabled=true`. 156 | 157 | ```yaml 158 | apiVersion: apps/v1 159 | kind: Deployment 160 | metadata: 161 | name: my-app 162 | spec: 163 | replicas: 3 164 | template: 165 | metadata: 166 | annotations: 167 | chaos.alpha.kubernetes.io/enabled: "true" 168 | spec: 169 | ... 170 | ``` 171 | 172 | You can exclude pods that have recently started by using the `--minimum-age` flag. 173 | 174 | ```console 175 | $ chaoskube --minimum-age 6h 176 | ... 177 | INFO[0000] setting pod filter minimumAge=6h0m0s 178 | ``` 179 | 180 | ## Limit the Chaos 181 | 182 | You can limit the time when chaos is introduced by weekdays, time periods of a day, day of a year or all of them together. 183 | 184 | Add a comma-separated list of abbreviated weekdays via the `--excluded-weekdays` options, a comma-separated list of time periods via the `--excluded-times-of-day` option and/or a comma-separated list of days of a year via the `--excluded-days-of-year` option and specify a `--timezone` by which to interpret them. 185 | 186 | ```console 187 | $ chaoskube \ 188 | --excluded-weekdays=Sat,Sun \ 189 | --excluded-times-of-day=22:00-08:00,11:00-13:00 \ 190 | --excluded-days-of-year=Apr1,Dec24 \ 191 | --timezone=Europe/Berlin 192 | ... 193 | INFO[0000] setting quiet times daysOfYear="[Apr 1 Dec24]" timesOfDay="[22:00-08:00 11:00-13:00]" weekdays="[Saturday Sunday]" 194 | INFO[0000] setting timezone location=Europe/Berlin name=CET offset=1 195 | ``` 196 | 197 | Use `UTC`, `Local` or pick a timezone name from the [(IANA) tz database](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). If you're testing `chaoskube` from your local machine then `Local` makes the most sense. Once you deploy `chaoskube` to your cluster you should deploy it with a specific timezone, e.g. where most of your team members are living, so that both your team and `chaoskube` have a common understanding when a particular weekday begins and ends, for instance. If your team is spread across multiple time zones it's probably best to pick `UTC` which is also the default. Picking the wrong timezone shifts the meaning of a particular weekday by a couple of hours between you and the server. 198 | 199 | ## Flags 200 | | Option | Environment | Description | Default | 201 | | -------------------------- | ---------------------------------- | -------------------------------------------------------------------- | -------------------------- | 202 | | `--interval` | `CHAOSKUBE_INTERVAL` | interval between pod terminations | 10m | 203 | | `--labels` | `CHAOSKUBE_LABELS` | label selector to filter pods by | (matches everything) | 204 | | `--annotations` | `CHAOSKUBE_ANNOTATIONS` | annotation selector to filter pods by | (matches everything) | 205 | | `--kinds` | `CHAOSKUBE_KINDS` | owner's kind selector to filter pods by | (all kinds) | 206 | | `--namespaces` | `CHAOSKUBE_NAMESPACES` | namespace selector to filter pods by | (all namespaces) | 207 | | `--namespace-labels` | `CHAOSKUBE_NAMESPACE_LABELS` | label selector to filter namespaces and its pods by | (all namespaces) | 208 | | `--included-pod-names` | `CHAOSKUBE_INCLUDED_POD_NAMES` | regular expression pattern for pod names to include | (all included) | 209 | | `--excluded-pod-names` | `CHAOSKUBE_EXCLUDED_POD_NAMES` | regular expression pattern for pod names to exclude | (none excluded) | 210 | | `--excluded-weekdays` | `CHAOSKUBE_EXCLUDED_WEEKDAYS` | weekdays when chaos is to be suspended, e.g. "Sat,Sun" | (no weekday excluded) | 211 | | `--excluded-times-of-day` | `CHAOSKUBE_EXCLUDED_TIMES_OF_DAY` | times of day when chaos is to be suspended, e.g. "22:00-08:00" | (no times of day excluded) | 212 | | `--excluded-days-of-year` | `CHAOSKUBE_EXCLUDED_DAYS_OF_YEAR` | days of a year when chaos is to be suspended, e.g. "Apr1,Dec24" | (no days of year excluded) | 213 | | `--timezone` | `CHAOSKUBE_TIMEZONE` | timezone from tz database, e.g. "America/New_York", "UTC" or "Local" | (UTC) | 214 | | `--max-runtime` | `CHAOSKUBE_MAX_RUNTIME` | Maximum runtime before chaoskube exits | -1s (infinite time) | 215 | | `--max-kill` | `CHAOSKUBE_MAX_KILL` | Specifies the maximum number of pods to be terminated per interval | 1 | 216 | | `--minimum-age` | `CHAOSKUBE_MINIMUM_AGE` | Minimum age to filter pods by | 0s (matches every pod) | 217 | | `--dry-run` | `CHAOSKUBE_DRY_RUN` | don't kill pods, only log what would have been done | true | 218 | | `--log-format` | `CHAOSKUBE_LOG_FORMAT` | specify the format of the log messages. Options are text and json | text | 219 | | `--log-caller` | `CHAOSKUBE_LOG_CALLER` | include the calling function name and location in the log messages | false | 220 | | `--slack-webhook` | `CHAOSKUBE_SLACK_WEBHOOK` | The address of the slack webhook for notifications | disabled | 221 | | `--client-namespace-scope` | `CHAOSKUBE_CLIENT_NAMESPACE_SCOPE` | Scope Kubernetes API calls to the given namespace | (all namespaces) | 222 | 223 | ## Related work 224 | 225 | There are several other projects that allow you to create some chaos in your Kubernetes cluster. 226 | 227 | * [kube-monkey](https://github.com/asobti/kube-monkey) is a sophisticated pod-based chaos monkey for Kubernetes. Each morning it compiles a schedule of pod terminations that should happen throughout the day. It allows to specify a mean time between failures on a per-pod basis, a feature that `chaoskube` [lacks](https://github.com/linki/chaoskube/issues/20). It can also be made aware of groups of pods forming an application so that it can treat them specially, e.g. kill all pods of an application at once. `kube-mokey` allows filtering targets globally via configuration options as well allows pods to opt-in to chaos via annotations,it allows individual apps to opt-in in their own unique way, as an example, app-a can request to kill him each week day one pod, while app-b which more couragues can request to kill 50% of pods. It understands a similar [configuration file](https://github.com/asobti/kube-monkey/blob/069e6fa9dc54ff9c83ac044b2d653f83e9dbdb5a/examples/configmap.yaml) used by Netflix's ChaosMonkey. 228 | * [PowerfulSeal](https://github.com/bloomberg/powerfulseal) is indeed a powerful tool to trouble your Kubernetes setup. Besides killing pods it can also take out your Cloud VMs or kill your Docker daemon. It has a vast number of [configuration options](https://github.com/bloomberg/powerfulseal/blob/1.1.1/tests/policy/example_config.yml) to define what can be killed and when. It also has an interactive mode that allows you to kill pods easily. 229 | * [fabric8's chaos monkey](https://fabric8.io/guide/chaosMonkey.html): A chaos monkey that comes bundled as an app with [fabric8's](https://fabric8.io/) Kubernetes platform. It can be deployed via a UI and reports any actions taken as a chat message and/or desktop notification. It can be configured with an interval and a pod name pattern that possible targets must match. 230 | * [k8aos](https://github.com/AlexsJones/k8aos): An interactive tool that can issue [a series of random pod deletions](https://github.com/AlexsJones/k8aos/blob/0dd0e1876a3d10b558d661bed7a28f79439b489e/core/mischief.go#L41-L51) across an entire Kubernetes cluster or scoped to a namespace. 231 | * [pod-reaper](https://github.com/target/pod-reaper) kills pods based on an interval and a configurable chaos chance. It allows to specify possible target pods via a label selector and namespace. It has the ability successfully shutdown itself after a while and therefore might be suited to work well with Kubernetes Job objects. It can also be configured to kill every pod that has been running for longer than a configurable duration. 232 | * [kubernetes-pod-chaos-monkey](https://github.com/jnewland/kubernetes-pod-chaos-monkey): A very simple random pod killer using `kubectl` written in a [couple lines of bash](https://github.com/jnewland/kubernetes-pod-chaos-monkey/blob/master/chaos.sh). Given a namespace and an interval it kills a random pod in that namespace at each interval. Pretty much like `chaoskube` worked in the beginning. 233 | * [kubeinvaders](https://github.com/lucky-sideburn/KubeInvaders) gamified chaos engineering tool for Kubernetes. It is like Space Invaders but the aliens are pods or worker nodes. 234 | 235 | ## Acknowledgements 236 | 237 | This project wouldn't be where it is with the ideas and help of several awesome contributors: 238 | * Thanks to [@twildeboer](https://github.com/twildeboer) and [@klautcomputing](https://github.com/klautcomputing) who sparked the idea of limiting chaos during certain times, such as [business hours](https://github.com/linki/chaoskube/issues/35) or [holidays](https://github.com/linki/chaoskube/issues/48) as well as the first implementations of this feature in [#54](https://github.com/linki/chaoskube/pull/54) and [#55](https://github.com/linki/chaoskube/pull/55). 239 | * Thanks to [@klautcomputing](https://github.com/klautcomputing) for the first attempt to solve the missing [percentage feature](https://github.com/linki/chaoskube/pull/47) as well as for providing [the RBAC config](https://github.com/linki/chaoskube/pull/30) files. 240 | * Thanks to [@j0sh3rs](https://github.com/j0sh3rs) for bringing [the Helm chart](https://hub.kubeapps.com/charts/stable/chaoskube) to the latest version. 241 | * Thanks to [@klautcomputing](https://github.com/klautcomputing), [@grosser](https://github.com/grosser), [@twz123](https://github.com/twz123), [@hchenxa](https://github.com/hchenxa) and [@bavarianbidi](https://github.com/bavarianbidi) for improvements to the Dockerfile and docs in [#31](https://github.com/linki/chaoskube/pull/31), [#40](https://github.com/linki/chaoskube/pull/40) and [#58](https://github.com/linki/chaoskube/pull/58). 242 | * Thanks to [@bakins](https://github.com/bakins) for adding the minimum age filter in [#86](https://github.com/linki/chaoskube/pull/86). 243 | * Thanks to [@bakins](https://github.com/bakins) for adding a health check and Prometheus metrics in [#94](https://github.com/linki/chaoskube/pull/94) and [#97](https://github.com/linki/chaoskube/pull/97). 244 | 245 | ## Contributing 246 | 247 | Feel free to create issues or submit pull requests. 248 | -------------------------------------------------------------------------------- /chaoskube.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/linki/chaoskube/235da9660d88a3c2b2a70d7ef2eae834050200bb/chaoskube.png -------------------------------------------------------------------------------- /chaoskube/chaoskube.go: -------------------------------------------------------------------------------- 1 | package chaoskube 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "regexp" 8 | "time" 9 | 10 | multierror "github.com/hashicorp/go-multierror" 11 | 12 | log "github.com/sirupsen/logrus" 13 | 14 | v1 "k8s.io/api/core/v1" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/apimachinery/pkg/labels" 17 | "k8s.io/apimachinery/pkg/selection" 18 | "k8s.io/apimachinery/pkg/types" 19 | "k8s.io/client-go/kubernetes" 20 | "k8s.io/client-go/kubernetes/scheme" 21 | typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" 22 | "k8s.io/client-go/tools/record" 23 | "k8s.io/client-go/tools/reference" 24 | 25 | "github.com/linki/chaoskube/metrics" 26 | "github.com/linki/chaoskube/notifier" 27 | "github.com/linki/chaoskube/terminator" 28 | "github.com/linki/chaoskube/util" 29 | ) 30 | 31 | // Chaoskube represents an instance of chaoskube 32 | type Chaoskube struct { 33 | // a kubernetes client object 34 | Client kubernetes.Interface 35 | // a label selector which restricts the pods to choose from 36 | Labels labels.Selector 37 | // an annotation selector which restricts the pods to choose from 38 | Annotations labels.Selector 39 | // a kind label selector which restricts the kinds to choose from 40 | Kinds labels.Selector 41 | // a namespace selector which restricts the pods to choose from 42 | Namespaces labels.Selector 43 | // a namespace label selector which restricts the namespaces to choose from 44 | NamespaceLabels labels.Selector 45 | // a regular expression for pod names to include 46 | IncludedPodNames *regexp.Regexp 47 | // a regular expression for pod names to exclude 48 | ExcludedPodNames *regexp.Regexp 49 | // a list of weekdays when termination is suspended 50 | ExcludedWeekdays []time.Weekday 51 | // a list of time periods of a day when termination is suspended 52 | ExcludedTimesOfDay []util.TimePeriod 53 | // a list of days of a year when termination is suspended 54 | ExcludedDaysOfYear []time.Time 55 | // the timezone to apply when detecting the current weekday 56 | Timezone *time.Location 57 | // minimum age of pods to consider 58 | MinimumAge time.Duration 59 | // an instance of logrus.StdLogger to write log messages to 60 | Logger log.FieldLogger 61 | // a terminator that terminates victim pods 62 | Terminator terminator.Terminator 63 | // dry run will not allow any pod terminations 64 | DryRun bool 65 | // grace period to terminate the pods 66 | GracePeriod time.Duration 67 | // event recorder allows to publish events to Kubernetes 68 | EventRecorder record.EventRecorder 69 | // a function to retrieve the current time 70 | Now func() time.Time 71 | 72 | MaxKill int 73 | // chaos events notifier 74 | Notifier notifier.Notifier 75 | // namespace scope for the Kubernetes client 76 | ClientNamespaceScope string 77 | } 78 | 79 | var ( 80 | // errPodNotFound is returned when no victim could be found 81 | errPodNotFound = errors.New("pod not found") 82 | // msgVictimNotFound is the log message when no victim was found 83 | msgVictimNotFound = "no victim found" 84 | // msgWeekdayExcluded is the log message when termination is suspended due to the weekday filter 85 | msgWeekdayExcluded = "weekday excluded" 86 | // msgTimeOfDayExcluded is the log message when termination is suspended due to the time of day filter 87 | msgTimeOfDayExcluded = "time of day excluded" 88 | // msgDayOfYearExcluded is the log message when termination is suspended due to the day of year filter 89 | msgDayOfYearExcluded = "day of year excluded" 90 | ) 91 | 92 | // New returns a new instance of Chaoskube. It expects: 93 | // * a Kubernetes client to connect to a Kubernetes API 94 | // * label, annotation and/or namespace selectors to reduce the amount of possible target pods 95 | // * a list of weekdays, times of day and/or days of a year when chaos mode is disabled 96 | // * a time zone to apply to the aforementioned time-based filters 97 | // * a logger implementing logrus.FieldLogger to send log output to 98 | // * what specific terminator to use to imbue chaos on victim pods 99 | // * whether to enable/disable dry-run mode 100 | func New(client kubernetes.Interface, labels, annotations, kinds, namespaces, namespaceLabels labels.Selector, includedPodNames, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, terminator terminator.Terminator, maxKill int, notifier notifier.Notifier, clientNamespaceScope string) *Chaoskube { 101 | broadcaster := record.NewBroadcaster() 102 | broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: client.CoreV1().Events(clientNamespaceScope)}) 103 | recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "chaoskube"}) 104 | 105 | return &Chaoskube{ 106 | Client: client, 107 | Labels: labels, 108 | Annotations: annotations, 109 | Kinds: kinds, 110 | Namespaces: namespaces, 111 | NamespaceLabels: namespaceLabels, 112 | IncludedPodNames: includedPodNames, 113 | ExcludedPodNames: excludedPodNames, 114 | ExcludedWeekdays: excludedWeekdays, 115 | ExcludedTimesOfDay: excludedTimesOfDay, 116 | ExcludedDaysOfYear: excludedDaysOfYear, 117 | Timezone: timezone, 118 | MinimumAge: minimumAge, 119 | Logger: logger, 120 | DryRun: dryRun, 121 | Terminator: terminator, 122 | EventRecorder: recorder, 123 | Now: time.Now, 124 | MaxKill: maxKill, 125 | Notifier: notifier, 126 | ClientNamespaceScope: clientNamespaceScope, 127 | } 128 | } 129 | 130 | // Run continuously picks and terminates a victim pod at a given interval 131 | // described by channel next. It returns when the given context is canceled. 132 | func (c *Chaoskube) Run(ctx context.Context, next <-chan time.Time) { 133 | for { 134 | if err := c.TerminateVictims(ctx); err != nil { 135 | c.Logger.WithField("err", err).Error("failed to terminate victim") 136 | metrics.ErrorsTotal.Inc() 137 | } 138 | 139 | c.Logger.Debug("sleeping...") 140 | metrics.IntervalsTotal.Inc() 141 | select { 142 | case <-next: 143 | case <-ctx.Done(): 144 | return 145 | } 146 | } 147 | } 148 | 149 | // TerminateVictims picks and deletes a victim. 150 | // It respects the configured excluded weekdays, times of day and days of a year filters. 151 | func (c *Chaoskube) TerminateVictims(ctx context.Context) error { 152 | now := c.Now().In(c.Timezone) 153 | 154 | for _, wd := range c.ExcludedWeekdays { 155 | if wd == now.Weekday() { 156 | c.Logger.WithField("weekday", now.Weekday()).Debug(msgWeekdayExcluded) 157 | return nil 158 | } 159 | } 160 | 161 | for _, tp := range c.ExcludedTimesOfDay { 162 | if tp.Includes(now) { 163 | c.Logger.WithField("timeOfDay", now.Format(util.Kitchen24)).Debug(msgTimeOfDayExcluded) 164 | return nil 165 | } 166 | } 167 | 168 | for _, d := range c.ExcludedDaysOfYear { 169 | if d.Day() == now.Day() && d.Month() == now.Month() { 170 | c.Logger.WithField("dayOfYear", now.Format(util.YearDay)).Debug(msgDayOfYearExcluded) 171 | return nil 172 | } 173 | } 174 | 175 | victims, err := c.Victims(ctx) 176 | if err == errPodNotFound { 177 | c.Logger.Debug(msgVictimNotFound) 178 | return nil 179 | } 180 | if err != nil { 181 | return err 182 | } 183 | 184 | var result *multierror.Error 185 | for _, victim := range victims { 186 | err = c.DeletePod(ctx, victim) 187 | result = multierror.Append(result, err) 188 | } 189 | 190 | return result.ErrorOrNil() 191 | } 192 | 193 | // Victims returns up to N pods as configured by MaxKill flag 194 | func (c *Chaoskube) Victims(ctx context.Context) ([]v1.Pod, error) { 195 | pods, err := c.Candidates(ctx) 196 | if err != nil { 197 | return []v1.Pod{}, err 198 | } 199 | 200 | c.Logger.WithField("count", len(pods)).Debug("found candidates") 201 | 202 | if len(pods) == 0 { 203 | return []v1.Pod{}, errPodNotFound 204 | } 205 | 206 | pods = util.RandomPodSubSlice(pods, c.MaxKill) 207 | 208 | c.Logger.WithField("count", len(pods)).Debug("found victims") 209 | return pods, nil 210 | } 211 | 212 | // Candidates returns the list of pods that are available for termination. 213 | // It returns all pods that match the configured label, annotation and namespace selectors. 214 | func (c *Chaoskube) Candidates(ctx context.Context) ([]v1.Pod, error) { 215 | listOptions := metav1.ListOptions{LabelSelector: c.Labels.String()} 216 | 217 | podList, err := c.Client.CoreV1().Pods(c.ClientNamespaceScope).List(ctx, listOptions) 218 | if err != nil { 219 | return nil, err 220 | } 221 | 222 | pods, err := filterByNamespaces(podList.Items, c.Namespaces) 223 | if err != nil { 224 | return nil, err 225 | } 226 | 227 | pods, err = filterPodsByNamespaceLabels(ctx, pods, c.NamespaceLabels, c.Client) 228 | if err != nil { 229 | return nil, err 230 | } 231 | 232 | pods, err = filterByKinds(pods, c.Kinds) 233 | if err != nil { 234 | return nil, err 235 | } 236 | 237 | pods = filterByAnnotations(pods, c.Annotations) 238 | pods = filterByPhase(pods, v1.PodRunning) 239 | pods = filterTerminatingPods(pods) 240 | pods = filterByMinimumAge(pods, c.MinimumAge, c.Now()) 241 | pods = filterByPodName(pods, c.IncludedPodNames, c.ExcludedPodNames) 242 | pods = filterByOwnerReference(pods) 243 | 244 | return pods, nil 245 | } 246 | 247 | // DeletePod deletes the given pod with the selected terminator. 248 | // It will not delete the pod if dry-run mode is enabled. 249 | func (c *Chaoskube) DeletePod(ctx context.Context, victim v1.Pod) error { 250 | c.Logger.WithFields(log.Fields{ 251 | "namespace": victim.Namespace, 252 | "name": victim.Name, 253 | }).Info("terminating pod") 254 | 255 | // return early if we're running in dryRun mode. 256 | if c.DryRun { 257 | return nil 258 | } 259 | 260 | start := time.Now() 261 | err := c.Terminator.Terminate(ctx, victim) 262 | metrics.TerminationDurationSeconds.Observe(time.Since(start).Seconds()) 263 | if err != nil { 264 | return err 265 | } 266 | 267 | metrics.PodsDeletedTotal.WithLabelValues(victim.Namespace).Inc() 268 | 269 | ref, err := reference.GetReference(scheme.Scheme, &victim) 270 | if err != nil { 271 | return err 272 | } 273 | 274 | c.EventRecorder.Event(ref, v1.EventTypeNormal, "Killing", "Pod was terminated by chaoskube to introduce chaos.") 275 | 276 | if err := c.Notifier.NotifyPodTermination(victim); err != nil { 277 | c.Logger.WithField("err", err).Warn("failed to notify pod termination") 278 | } 279 | 280 | return nil 281 | } 282 | 283 | // filterByKinds filters a list of pods by a given kind selector. 284 | func filterByKinds(pods []v1.Pod, kinds labels.Selector) ([]v1.Pod, error) { 285 | // empty filter returns original list 286 | if kinds.Empty() { 287 | return pods, nil 288 | } 289 | 290 | // split requirements into including and excluding groups 291 | reqs, _ := kinds.Requirements() 292 | reqIncl := []labels.Requirement{} 293 | reqExcl := []labels.Requirement{} 294 | 295 | for _, req := range reqs { 296 | switch req.Operator() { 297 | case selection.Exists: 298 | reqIncl = append(reqIncl, req) 299 | case selection.DoesNotExist: 300 | reqExcl = append(reqExcl, req) 301 | default: 302 | return nil, fmt.Errorf("unsupported operator: %s", req.Operator()) 303 | } 304 | } 305 | 306 | filteredList := []v1.Pod{} 307 | 308 | for _, pod := range pods { 309 | // if there aren't any including requirements, we're in by default 310 | included := len(reqIncl) == 0 311 | 312 | // Check owner reference 313 | for _, ref := range pod.GetOwnerReferences() { 314 | // convert the pod's owner kind to an equivalent label selector 315 | selector := labels.Set{ref.Kind: ""} 316 | 317 | // include pod if one including requirement matches 318 | for _, req := range reqIncl { 319 | if req.Matches(selector) { 320 | included = true 321 | break 322 | } 323 | } 324 | 325 | // exclude pod if it is filtered out by at least one excluding requirement 326 | for _, req := range reqExcl { 327 | if !req.Matches(selector) { 328 | included = false 329 | break 330 | } 331 | } 332 | } 333 | 334 | if included { 335 | filteredList = append(filteredList, pod) 336 | } 337 | } 338 | 339 | return filteredList, nil 340 | } 341 | 342 | // filterByNamespaces filters a list of pods by a given namespace selector. 343 | func filterByNamespaces(pods []v1.Pod, namespaces labels.Selector) ([]v1.Pod, error) { 344 | // empty filter returns original list 345 | if namespaces.Empty() { 346 | return pods, nil 347 | } 348 | 349 | // split requirements into including and excluding groups 350 | reqs, _ := namespaces.Requirements() 351 | reqIncl := []labels.Requirement{} 352 | reqExcl := []labels.Requirement{} 353 | 354 | for _, req := range reqs { 355 | switch req.Operator() { 356 | case selection.Exists: 357 | reqIncl = append(reqIncl, req) 358 | case selection.DoesNotExist: 359 | reqExcl = append(reqExcl, req) 360 | default: 361 | return nil, fmt.Errorf("unsupported operator: %s", req.Operator()) 362 | } 363 | } 364 | 365 | filteredList := []v1.Pod{} 366 | 367 | for _, pod := range pods { 368 | // if there aren't any including requirements, we're in by default 369 | included := len(reqIncl) == 0 370 | 371 | // convert the pod's namespace to an equivalent label selector 372 | selector := labels.Set{pod.Namespace: ""} 373 | 374 | // include pod if one including requirement matches 375 | for _, req := range reqIncl { 376 | if req.Matches(selector) { 377 | included = true 378 | break 379 | } 380 | } 381 | 382 | // exclude pod if it is filtered out by at least one excluding requirement 383 | for _, req := range reqExcl { 384 | if !req.Matches(selector) { 385 | included = false 386 | break 387 | } 388 | } 389 | 390 | if included { 391 | filteredList = append(filteredList, pod) 392 | } 393 | } 394 | 395 | return filteredList, nil 396 | } 397 | 398 | // filterPodsByNamespaceLabels filters a list of pods by a given label selector on their namespace. 399 | func filterPodsByNamespaceLabels(ctx context.Context, pods []v1.Pod, labels labels.Selector, client kubernetes.Interface) ([]v1.Pod, error) { 400 | // empty filter returns original list 401 | if labels.Empty() { 402 | return pods, nil 403 | } 404 | 405 | // find all namespaces matching the label selector 406 | listOptions := metav1.ListOptions{LabelSelector: labels.String()} 407 | 408 | namespaces, err := client.CoreV1().Namespaces().List(ctx, listOptions) 409 | if err != nil { 410 | return nil, err 411 | } 412 | 413 | filteredList := []v1.Pod{} 414 | 415 | for _, pod := range pods { 416 | for _, namespace := range namespaces.Items { 417 | // include pod if its in one of the matched namespaces 418 | if pod.Namespace == namespace.Name { 419 | filteredList = append(filteredList, pod) 420 | } 421 | } 422 | } 423 | 424 | return filteredList, nil 425 | } 426 | 427 | // filterByAnnotations filters a list of pods by a given annotation selector. 428 | func filterByAnnotations(pods []v1.Pod, annotations labels.Selector) []v1.Pod { 429 | // empty filter returns original list 430 | if annotations.Empty() { 431 | return pods 432 | } 433 | 434 | filteredList := []v1.Pod{} 435 | 436 | for _, pod := range pods { 437 | // convert the pod's annotations to an equivalent label selector 438 | selector := labels.Set(pod.Annotations) 439 | 440 | // include pod if its annotations match the selector 441 | if annotations.Matches(selector) { 442 | filteredList = append(filteredList, pod) 443 | } 444 | } 445 | 446 | return filteredList 447 | } 448 | 449 | // filterByPhase filters a list of pods by a given PodPhase, e.g. Running. 450 | func filterByPhase(pods []v1.Pod, phase v1.PodPhase) []v1.Pod { 451 | filteredList := []v1.Pod{} 452 | 453 | for _, pod := range pods { 454 | if pod.Status.Phase == phase { 455 | filteredList = append(filteredList, pod) 456 | } 457 | } 458 | 459 | return filteredList 460 | } 461 | 462 | // filterTerminatingPods removes pod which have a non nil DeletionTimestamp 463 | func filterTerminatingPods(pods []v1.Pod) []v1.Pod { 464 | filteredList := []v1.Pod{} 465 | for _, pod := range pods { 466 | if pod.DeletionTimestamp != nil { 467 | continue 468 | } 469 | filteredList = append(filteredList, pod) 470 | } 471 | return filteredList 472 | } 473 | 474 | // filterByMinimumAge filters pods by creation time. Only pods 475 | // older than minimumAge are returned 476 | func filterByMinimumAge(pods []v1.Pod, minimumAge time.Duration, now time.Time) []v1.Pod { 477 | if minimumAge <= time.Duration(0) { 478 | return pods 479 | } 480 | 481 | creationTime := now.Add(-minimumAge) 482 | 483 | filteredList := []v1.Pod{} 484 | 485 | for _, pod := range pods { 486 | if pod.ObjectMeta.CreationTimestamp.Time.Before(creationTime) { 487 | filteredList = append(filteredList, pod) 488 | } 489 | } 490 | 491 | return filteredList 492 | } 493 | 494 | // filterByPodName filters pods by name. Only pods matching the includedPodNames and not 495 | // matching the excludedPodNames are returned 496 | func filterByPodName(pods []v1.Pod, includedPodNames, excludedPodNames *regexp.Regexp) []v1.Pod { 497 | // return early if neither included nor excluded regular expressions are given 498 | if includedPodNames == nil && excludedPodNames == nil { 499 | return pods 500 | } 501 | 502 | filteredList := []v1.Pod{} 503 | 504 | for _, pod := range pods { 505 | include := includedPodNames == nil || includedPodNames.String() == "" || includedPodNames.MatchString(pod.Name) 506 | exclude := excludedPodNames != nil && excludedPodNames.String() != "" && excludedPodNames.MatchString(pod.Name) 507 | 508 | if include && !exclude { 509 | filteredList = append(filteredList, pod) 510 | } 511 | } 512 | 513 | return filteredList 514 | } 515 | 516 | func filterByOwnerReference(pods []v1.Pod) []v1.Pod { 517 | owners := make(map[types.UID][]v1.Pod) 518 | filteredList := []v1.Pod{} 519 | for _, pod := range pods { 520 | // Don't filter out pods with no owner reference 521 | if len(pod.GetOwnerReferences()) == 0 { 522 | filteredList = append(filteredList, pod) 523 | continue 524 | } 525 | 526 | // Group remaining pods by their owner reference 527 | for _, ref := range pod.GetOwnerReferences() { 528 | owners[ref.UID] = append(owners[ref.UID], pod) 529 | } 530 | } 531 | 532 | // For each owner reference select a random pod from its group 533 | for _, pods := range owners { 534 | filteredList = append(filteredList, util.RandomPodSubSlice(pods, 1)...) 535 | } 536 | 537 | return filteredList 538 | } 539 | -------------------------------------------------------------------------------- /chaoskube/chaoskube_test.go: -------------------------------------------------------------------------------- 1 | package chaoskube 2 | 3 | import ( 4 | "context" 5 | "math/rand" 6 | "regexp" 7 | "sort" 8 | "testing" 9 | "time" 10 | 11 | log "github.com/sirupsen/logrus" 12 | "github.com/sirupsen/logrus/hooks/test" 13 | 14 | v1 "k8s.io/api/core/v1" 15 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 16 | "k8s.io/apimachinery/pkg/labels" 17 | "k8s.io/client-go/kubernetes" 18 | "k8s.io/client-go/kubernetes/fake" 19 | 20 | "github.com/linki/chaoskube/internal/testutil" 21 | "github.com/linki/chaoskube/notifier" 22 | "github.com/linki/chaoskube/terminator" 23 | "github.com/linki/chaoskube/util" 24 | 25 | "github.com/stretchr/testify/suite" 26 | ) 27 | 28 | type Suite struct { 29 | testutil.TestSuite 30 | } 31 | 32 | // podInfo holds information used to create a v1.Pod 33 | type podInfo struct { 34 | Namespace string 35 | Name string 36 | } 37 | 38 | var ( 39 | logger, logOutput = test.NewNullLogger() 40 | testNotifier = ¬ifier.Noop{} 41 | ) 42 | 43 | func (suite *Suite) SetupTest() { 44 | logger.SetLevel(log.DebugLevel) 45 | logOutput.Reset() 46 | } 47 | 48 | // TestNew tests that arguments are passed to the new instance correctly 49 | func (suite *Suite) TestNew() { 50 | var ( 51 | client = fake.NewSimpleClientset() 52 | labelSelector, _ = labels.Parse("foo=bar") 53 | annotations, _ = labels.Parse("baz=waldo") 54 | kinds, _ = labels.Parse("job") 55 | namespaces, _ = labels.Parse("qux") 56 | namespaceLabels, _ = labels.Parse("taz=wubble") 57 | includedPodNames = regexp.MustCompile("foo") 58 | excludedPodNames = regexp.MustCompile("bar") 59 | excludedWeekdays = []time.Weekday{time.Friday} 60 | excludedTimesOfDay = []util.TimePeriod{util.TimePeriod{}} 61 | excludedDaysOfYear = []time.Time{time.Now()} 62 | minimumAge = time.Duration(42) 63 | dryRun = true 64 | terminator = terminator.NewDeletePodTerminator(client, logger, 10*time.Second) 65 | maxKill = 1 66 | notifier = testNotifier 67 | ) 68 | 69 | chaoskube := New( 70 | client, 71 | labelSelector, 72 | annotations, 73 | kinds, 74 | namespaces, 75 | namespaceLabels, 76 | includedPodNames, 77 | excludedPodNames, 78 | excludedWeekdays, 79 | excludedTimesOfDay, 80 | excludedDaysOfYear, 81 | time.UTC, 82 | minimumAge, 83 | logger, 84 | dryRun, 85 | terminator, 86 | maxKill, 87 | notifier, 88 | v1.NamespaceAll, 89 | ) 90 | suite.Require().NotNil(chaoskube) 91 | 92 | suite.Equal(client, chaoskube.Client) 93 | suite.Equal("foo=bar", chaoskube.Labels.String()) 94 | suite.Equal("baz=waldo", chaoskube.Annotations.String()) 95 | suite.Equal("job", chaoskube.Kinds.String()) 96 | suite.Equal("qux", chaoskube.Namespaces.String()) 97 | suite.Equal("taz=wubble", chaoskube.NamespaceLabels.String()) 98 | suite.Equal("foo", chaoskube.IncludedPodNames.String()) 99 | suite.Equal("bar", chaoskube.ExcludedPodNames.String()) 100 | suite.Equal(excludedWeekdays, chaoskube.ExcludedWeekdays) 101 | suite.Equal(excludedTimesOfDay, chaoskube.ExcludedTimesOfDay) 102 | suite.Equal(excludedDaysOfYear, chaoskube.ExcludedDaysOfYear) 103 | suite.Equal(time.UTC, chaoskube.Timezone) 104 | suite.Equal(minimumAge, chaoskube.MinimumAge) 105 | suite.Equal(logger, chaoskube.Logger) 106 | suite.Equal(dryRun, chaoskube.DryRun) 107 | suite.Equal(terminator, chaoskube.Terminator) 108 | } 109 | 110 | // TestRunContextCanceled tests that a canceled context will exit the Run function. 111 | func (suite *Suite) TestRunContextCanceled() { 112 | chaoskube := suite.setup( 113 | labels.Everything(), 114 | labels.Everything(), 115 | labels.Everything(), 116 | labels.Everything(), 117 | labels.Everything(), 118 | ®exp.Regexp{}, 119 | ®exp.Regexp{}, 120 | []time.Weekday{}, 121 | []util.TimePeriod{}, 122 | []time.Time{}, 123 | time.UTC, 124 | time.Duration(0), 125 | false, 126 | 10, 127 | 1, 128 | v1.NamespaceAll, 129 | ) 130 | 131 | ctx, cancel := context.WithCancel(context.Background()) 132 | cancel() 133 | 134 | chaoskube.Run(ctx, nil) 135 | } 136 | 137 | // TestCandidates tests that the various pod filters are applied correctly. 138 | func (suite *Suite) TestCandidates() { 139 | foo := map[string]string{"namespace": "default", "name": "foo"} 140 | bar := map[string]string{"namespace": "testing", "name": "bar"} 141 | 142 | for _, tt := range []struct { 143 | labelSelector string 144 | annotationSelector string 145 | namespaceSelector string 146 | pods []map[string]string 147 | }{ 148 | {"", "", "", []map[string]string{foo, bar}}, 149 | {"app=foo", "", "", []map[string]string{foo}}, 150 | {"app!=foo", "", "", []map[string]string{bar}}, 151 | {"", "chaos=foo", "", []map[string]string{foo}}, 152 | {"", "chaos!=foo", "", []map[string]string{bar}}, 153 | {"", "", "default", []map[string]string{foo}}, 154 | {"", "", "default,testing", []map[string]string{foo, bar}}, 155 | {"", "", "!testing", []map[string]string{foo}}, 156 | {"", "", "!default,!testing", []map[string]string{}}, 157 | {"", "", "default,!testing", []map[string]string{foo}}, 158 | {"", "", "default,!default", []map[string]string{}}, 159 | } { 160 | labelSelector, err := labels.Parse(tt.labelSelector) 161 | suite.Require().NoError(err) 162 | 163 | annotationSelector, err := labels.Parse(tt.annotationSelector) 164 | suite.Require().NoError(err) 165 | 166 | namespaceSelector, err := labels.Parse(tt.namespaceSelector) 167 | suite.Require().NoError(err) 168 | 169 | chaoskube := suite.setupWithPods( 170 | labelSelector, 171 | annotationSelector, 172 | labels.Everything(), 173 | namespaceSelector, 174 | labels.Everything(), 175 | nil, 176 | nil, 177 | []time.Weekday{}, 178 | []util.TimePeriod{}, 179 | []time.Time{}, 180 | time.UTC, 181 | time.Duration(0), 182 | false, 183 | 10, 184 | v1.NamespaceAll, 185 | ) 186 | 187 | suite.assertCandidates(chaoskube, tt.pods) 188 | } 189 | } 190 | 191 | // TestCandidatesNamespaceLabels tests that the label selector for namespaces works correctly. 192 | func (suite *Suite) TestCandidatesNamespaceLabels() { 193 | foo := map[string]string{"namespace": "default", "name": "foo"} 194 | bar := map[string]string{"namespace": "testing", "name": "bar"} 195 | 196 | for _, tt := range []struct { 197 | labels string 198 | pods []map[string]string 199 | }{ 200 | {"", []map[string]string{foo, bar}}, 201 | {"env", []map[string]string{foo, bar}}, 202 | {"!env", []map[string]string{}}, 203 | {"env=default", []map[string]string{foo}}, 204 | {"env=testing", []map[string]string{bar}}, 205 | {"env!=default", []map[string]string{bar}}, 206 | {"env!=testing", []map[string]string{foo}}, 207 | {"env!=default,env!=testing", []map[string]string{}}, 208 | {"env=default,env!=testing", []map[string]string{foo}}, 209 | {"env=default,env!=default", []map[string]string{}}, 210 | {"nomatch", []map[string]string{}}, 211 | } { 212 | namespaceLabels, err := labels.Parse(tt.labels) 213 | suite.Require().NoError(err) 214 | 215 | chaoskube := suite.setupWithPods( 216 | labels.Everything(), 217 | labels.Everything(), 218 | labels.Everything(), 219 | labels.Everything(), 220 | namespaceLabels, 221 | nil, 222 | nil, 223 | []time.Weekday{}, 224 | []util.TimePeriod{}, 225 | []time.Time{}, 226 | time.UTC, 227 | time.Duration(0), 228 | false, 229 | 10, 230 | v1.NamespaceAll, 231 | ) 232 | 233 | suite.assertCandidates(chaoskube, tt.pods) 234 | } 235 | } 236 | 237 | func (suite *Suite) TestCandidatesClientNamespaceScope() { 238 | foo := map[string]string{"namespace": "default", "name": "foo"} 239 | bar := map[string]string{"namespace": "testing", "name": "bar"} 240 | 241 | for _, tt := range []struct { 242 | clientNamespaceScope string 243 | pods []map[string]string 244 | }{ 245 | {v1.NamespaceAll, []map[string]string{foo, bar}}, 246 | {"default", []map[string]string{foo}}, 247 | {"testing", []map[string]string{bar}}, 248 | } { 249 | chaoskube := suite.setupWithPods( 250 | labels.Everything(), 251 | labels.Everything(), 252 | labels.Everything(), 253 | labels.Everything(), 254 | labels.Everything(), 255 | nil, 256 | nil, 257 | []time.Weekday{}, 258 | []util.TimePeriod{}, 259 | []time.Time{}, 260 | time.UTC, 261 | time.Duration(0), 262 | false, 263 | 10, 264 | tt.clientNamespaceScope, 265 | ) 266 | 267 | suite.assertCandidates(chaoskube, tt.pods) 268 | } 269 | } 270 | 271 | // TestCandidatesPodNameRegexp tests that the included and excluded pod name regular expressions 272 | // are applied correctly. 273 | func (suite *Suite) TestCandidatesPodNameRegexp() { 274 | foo := map[string]string{"namespace": "default", "name": "foo"} 275 | bar := map[string]string{"namespace": "testing", "name": "bar"} 276 | 277 | for _, tt := range []struct { 278 | includedPodNames *regexp.Regexp 279 | excludedPodNames *regexp.Regexp 280 | pods []map[string]string 281 | }{ 282 | // no included nor excluded regular expressions given 283 | {nil, nil, []map[string]string{foo, bar}}, 284 | // either included or excluded regular expression given 285 | {regexp.MustCompile("fo.*"), nil, []map[string]string{foo}}, 286 | {nil, regexp.MustCompile("fo.*"), []map[string]string{bar}}, 287 | // either included or excluded regular expression is empty 288 | {regexp.MustCompile("fo.*"), regexp.MustCompile(""), []map[string]string{foo}}, 289 | {regexp.MustCompile(""), regexp.MustCompile("fo.*"), []map[string]string{bar}}, 290 | // both included and excluded regular expressions are considered 291 | {regexp.MustCompile("fo.*"), regexp.MustCompile("f.*"), []map[string]string{}}, 292 | } { 293 | chaoskube := suite.setupWithPods( 294 | labels.Everything(), 295 | labels.Everything(), 296 | labels.Everything(), 297 | labels.Everything(), 298 | labels.Everything(), 299 | tt.includedPodNames, 300 | tt.excludedPodNames, 301 | []time.Weekday{}, 302 | []util.TimePeriod{}, 303 | []time.Time{}, 304 | time.UTC, 305 | time.Duration(0), 306 | false, 307 | 10, 308 | v1.NamespaceAll, 309 | ) 310 | 311 | suite.assertCandidates(chaoskube, tt.pods) 312 | } 313 | } 314 | 315 | // TestVictim tests that a random victim is chosen from selected candidates. 316 | func (suite *Suite) TestVictim() { 317 | foo := map[string]string{"namespace": "default", "name": "foo"} 318 | bar := map[string]string{"namespace": "testing", "name": "bar"} 319 | 320 | for _, tt := range []struct { 321 | seed int64 322 | labelSelector string 323 | victim map[string]string 324 | }{ 325 | {1000, "", foo}, 326 | {2000, "", bar}, 327 | {2000, "app=foo", foo}, 328 | } { 329 | rand.Seed(tt.seed) 330 | 331 | labelSelector, err := labels.Parse(tt.labelSelector) 332 | suite.Require().NoError(err) 333 | 334 | chaoskube := suite.setupWithPods( 335 | labelSelector, 336 | labels.Everything(), 337 | labels.Everything(), 338 | labels.Everything(), 339 | labels.Everything(), 340 | ®exp.Regexp{}, 341 | ®exp.Regexp{}, 342 | []time.Weekday{}, 343 | []util.TimePeriod{}, 344 | []time.Time{}, 345 | time.UTC, 346 | time.Duration(0), 347 | false, 348 | 10, 349 | v1.NamespaceAll, 350 | ) 351 | 352 | suite.assertVictim(chaoskube, tt.victim) 353 | } 354 | } 355 | 356 | // TestVictims tests that a random subset of pods is chosen from selected candidates 357 | func (suite *Suite) TestVictims() { 358 | 359 | podsInfo := []podInfo{ 360 | {"default", "foo"}, 361 | {"testing", "bar"}, 362 | {"test", "baz"}, 363 | } 364 | 365 | t := func(p podInfo) map[string]string { 366 | return map[string]string{"namespace": p.Namespace, "name": p.Name} 367 | } 368 | 369 | foo := t(podsInfo[0]) 370 | bar := t(podsInfo[1]) 371 | baz := t(podsInfo[2]) 372 | 373 | rand.Seed(2) // yields order of bar, baz, foo 374 | 375 | for _, tt := range []struct { 376 | labelSelector string 377 | victims []map[string]string 378 | maxKill int 379 | }{ 380 | {"", []map[string]string{baz}, 1}, 381 | {"", []map[string]string{baz, bar}, 2}, 382 | {"app=foo", []map[string]string{foo}, 2}, 383 | } { 384 | 385 | labelSelector, err := labels.Parse(tt.labelSelector) 386 | suite.Require().NoError(err) 387 | 388 | chaoskube := suite.setup( 389 | labelSelector, 390 | labels.Everything(), 391 | labels.Everything(), 392 | labels.Everything(), 393 | labels.Everything(), 394 | ®exp.Regexp{}, 395 | ®exp.Regexp{}, 396 | []time.Weekday{}, 397 | []util.TimePeriod{}, 398 | []time.Time{}, 399 | time.UTC, 400 | time.Duration(0), 401 | false, 402 | 10, 403 | tt.maxKill, 404 | v1.NamespaceAll, 405 | ) 406 | suite.createPods(chaoskube.Client, podsInfo) 407 | 408 | suite.assertVictims(chaoskube, tt.victims) 409 | } 410 | } 411 | 412 | // TestNoVictimReturnsError tests that on missing victim it returns a known error 413 | func (suite *Suite) TestNoVictimReturnsError() { 414 | chaoskube := suite.setup( 415 | labels.Everything(), 416 | labels.Everything(), 417 | labels.Everything(), 418 | labels.Everything(), 419 | labels.Everything(), 420 | ®exp.Regexp{}, 421 | ®exp.Regexp{}, 422 | []time.Weekday{}, 423 | []util.TimePeriod{}, 424 | []time.Time{}, 425 | time.UTC, 426 | time.Duration(0), 427 | false, 428 | 10, 429 | 1, 430 | v1.NamespaceAll, 431 | ) 432 | 433 | _, err := chaoskube.Victims(context.Background()) 434 | suite.Equal(err, errPodNotFound) 435 | suite.EqualError(err, "pod not found") 436 | } 437 | 438 | // TestDeletePod tests that a given pod is deleted and dryRun is respected. 439 | func (suite *Suite) TestDeletePod() { 440 | foo := map[string]string{"namespace": "default", "name": "foo"} 441 | bar := map[string]string{"namespace": "testing", "name": "bar"} 442 | 443 | for _, tt := range []struct { 444 | dryRun bool 445 | remainingPods []map[string]string 446 | }{ 447 | {false, []map[string]string{bar}}, 448 | {true, []map[string]string{foo, bar}}, 449 | } { 450 | chaoskube := suite.setupWithPods( 451 | labels.Everything(), 452 | labels.Everything(), 453 | labels.Everything(), 454 | labels.Everything(), 455 | labels.Everything(), 456 | ®exp.Regexp{}, 457 | ®exp.Regexp{}, 458 | []time.Weekday{}, 459 | []util.TimePeriod{}, 460 | []time.Time{}, 461 | time.UTC, 462 | time.Duration(0), 463 | tt.dryRun, 464 | 10, 465 | v1.NamespaceAll, 466 | ) 467 | 468 | victim := util.NewPod("default", "foo", v1.PodRunning) 469 | 470 | err := chaoskube.DeletePod(context.Background(), victim) 471 | suite.Require().NoError(err) 472 | 473 | suite.AssertLog(logOutput, log.InfoLevel, "terminating pod", log.Fields{"namespace": "default", "name": "foo"}) 474 | suite.assertCandidates(chaoskube, tt.remainingPods) 475 | } 476 | } 477 | 478 | // TestDeletePodNotFound tests missing target pod will return an error. 479 | func (suite *Suite) TestDeletePodNotFound() { 480 | chaoskube := suite.setup( 481 | labels.Everything(), 482 | labels.Everything(), 483 | labels.Everything(), 484 | labels.Everything(), 485 | labels.Everything(), 486 | ®exp.Regexp{}, 487 | ®exp.Regexp{}, 488 | []time.Weekday{}, 489 | []util.TimePeriod{}, 490 | []time.Time{}, 491 | time.UTC, 492 | time.Duration(0), 493 | false, 494 | 10, 495 | 1, 496 | v1.NamespaceAll, 497 | ) 498 | 499 | victim := util.NewPod("default", "foo", v1.PodRunning) 500 | 501 | err := chaoskube.DeletePod(context.Background(), victim) 502 | suite.EqualError(err, `pods "foo" not found`) 503 | } 504 | 505 | func (suite *Suite) TestTerminateVictim() { 506 | midnight := util.NewTimePeriod( 507 | ThankGodItsFriday{}.Now().Add(-16*time.Hour), 508 | ThankGodItsFriday{}.Now().Add(-14*time.Hour), 509 | ) 510 | morning := util.NewTimePeriod( 511 | ThankGodItsFriday{}.Now().Add(-7*time.Hour), 512 | ThankGodItsFriday{}.Now().Add(-6*time.Hour), 513 | ) 514 | afternoon := util.NewTimePeriod( 515 | ThankGodItsFriday{}.Now().Add(-1*time.Hour), 516 | ThankGodItsFriday{}.Now().Add(+1*time.Hour), 517 | ) 518 | 519 | australia, err := time.LoadLocation("Australia/Brisbane") 520 | suite.Require().NoError(err) 521 | 522 | for _, tt := range []struct { 523 | excludedWeekdays []time.Weekday 524 | excludedTimesOfDay []util.TimePeriod 525 | excludedDaysOfYear []time.Time 526 | now func() time.Time 527 | timezone *time.Location 528 | remainingPodCount int 529 | }{ 530 | // no time is excluded, one pod should be killed 531 | { 532 | []time.Weekday{}, 533 | []util.TimePeriod{}, 534 | []time.Time{}, 535 | ThankGodItsFriday{}.Now, 536 | time.UTC, 537 | 1, 538 | }, 539 | // current weekday is excluded, no pod should be killed 540 | { 541 | []time.Weekday{time.Friday}, 542 | []util.TimePeriod{}, 543 | []time.Time{}, 544 | ThankGodItsFriday{}.Now, 545 | time.UTC, 546 | 2, 547 | }, 548 | // current time of day is excluded, no pod should be killed 549 | { 550 | []time.Weekday{}, 551 | []util.TimePeriod{afternoon}, 552 | []time.Time{}, 553 | ThankGodItsFriday{}.Now, 554 | time.UTC, 555 | 2, 556 | }, 557 | // one day after an excluded weekday, one pod should be killed 558 | { 559 | []time.Weekday{time.Friday}, 560 | []util.TimePeriod{}, 561 | []time.Time{}, 562 | func() time.Time { return ThankGodItsFriday{}.Now().Add(24 * time.Hour) }, 563 | time.UTC, 564 | 1, 565 | }, 566 | // seven days after an excluded weekday, no pod should be killed 567 | { 568 | []time.Weekday{time.Friday}, 569 | []util.TimePeriod{}, 570 | []time.Time{}, 571 | func() time.Time { return ThankGodItsFriday{}.Now().Add(7 * 24 * time.Hour) }, 572 | time.UTC, 573 | 2, 574 | }, 575 | // one hour after an excluded time period, one pod should be killed 576 | { 577 | []time.Weekday{}, 578 | []util.TimePeriod{afternoon}, 579 | []time.Time{}, 580 | func() time.Time { return ThankGodItsFriday{}.Now().Add(+2 * time.Hour) }, 581 | time.UTC, 582 | 1, 583 | }, 584 | // twenty four hours after an excluded time period, no pod should be killed 585 | { 586 | []time.Weekday{}, 587 | []util.TimePeriod{afternoon}, 588 | []time.Time{}, 589 | func() time.Time { return ThankGodItsFriday{}.Now().Add(+24 * time.Hour) }, 590 | time.UTC, 591 | 2, 592 | }, 593 | // current weekday is excluded but we are in another time zone, one pod should be killed 594 | { 595 | []time.Weekday{time.Friday}, 596 | []util.TimePeriod{}, 597 | []time.Time{}, 598 | ThankGodItsFriday{}.Now, 599 | australia, 600 | 1, 601 | }, 602 | // current time period is excluded but we are in another time zone, one pod should be killed 603 | { 604 | []time.Weekday{}, 605 | []util.TimePeriod{afternoon}, 606 | []time.Time{}, 607 | ThankGodItsFriday{}.Now, 608 | australia, 609 | 1, 610 | }, 611 | // one out of two excluded weeksdays match, no pod should be killed 612 | { 613 | []time.Weekday{time.Monday, time.Friday}, 614 | []util.TimePeriod{}, 615 | []time.Time{}, 616 | ThankGodItsFriday{}.Now, 617 | time.UTC, 618 | 2, 619 | }, 620 | // one out of two excluded time periods match, no pod should be killed 621 | { 622 | []time.Weekday{}, 623 | []util.TimePeriod{morning, afternoon}, 624 | []time.Time{}, 625 | ThankGodItsFriday{}.Now, 626 | time.UTC, 627 | 2, 628 | }, 629 | // we're inside an excluded time period across days, no pod should be killed 630 | { 631 | []time.Weekday{}, 632 | []util.TimePeriod{midnight}, 633 | []time.Time{}, 634 | func() time.Time { return ThankGodItsFriday{}.Now().Add(-15 * time.Hour) }, 635 | time.UTC, 636 | 2, 637 | }, 638 | // we're before an excluded time period across days, one pod should be killed 639 | { 640 | []time.Weekday{}, 641 | []util.TimePeriod{midnight}, 642 | []time.Time{}, 643 | func() time.Time { return ThankGodItsFriday{}.Now().Add(-17 * time.Hour) }, 644 | time.UTC, 645 | 1, 646 | }, 647 | // we're after an excluded time period across days, one pod should be killed 648 | { 649 | []time.Weekday{}, 650 | []util.TimePeriod{midnight}, 651 | []time.Time{}, 652 | func() time.Time { return ThankGodItsFriday{}.Now().Add(-13 * time.Hour) }, 653 | time.UTC, 654 | 1, 655 | }, 656 | // this day of year is excluded, no pod should be killed 657 | { 658 | []time.Weekday{}, 659 | []util.TimePeriod{}, 660 | []time.Time{ 661 | ThankGodItsFriday{}.Now(), // today 662 | }, 663 | func() time.Time { return ThankGodItsFriday{}.Now() }, 664 | time.UTC, 665 | 2, 666 | }, 667 | // this day of year in year 0 is excluded, no pod should be killed 668 | { 669 | []time.Weekday{}, 670 | []util.TimePeriod{}, 671 | []time.Time{ 672 | time.Date(0, 9, 24, 0, 00, 00, 00, time.UTC), // same year day 673 | }, 674 | func() time.Time { return ThankGodItsFriday{}.Now() }, 675 | time.UTC, 676 | 2, 677 | }, 678 | // matching works fine even when multiple days-of-year are provided, no pod should be killed 679 | { 680 | []time.Weekday{}, 681 | []util.TimePeriod{}, 682 | []time.Time{ 683 | time.Date(0, 9, 25, 10, 00, 00, 00, time.UTC), // different year day 684 | time.Date(0, 9, 24, 10, 00, 00, 00, time.UTC), // same year day 685 | }, 686 | func() time.Time { return ThankGodItsFriday{}.Now() }, 687 | time.UTC, 688 | 2, 689 | }, 690 | // there is an excluded day of year but it's not today, one pod should be killed 691 | { 692 | []time.Weekday{}, 693 | []util.TimePeriod{}, 694 | []time.Time{ 695 | time.Date(0, 9, 25, 10, 00, 00, 00, time.UTC), // different year day 696 | }, 697 | func() time.Time { return ThankGodItsFriday{}.Now() }, 698 | time.UTC, 699 | 1, 700 | }, 701 | // there is an excluded day of year but the month is different, one pod should be killed 702 | { 703 | []time.Weekday{}, 704 | []util.TimePeriod{}, 705 | []time.Time{ 706 | time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC), // different year day 707 | }, 708 | func() time.Time { return ThankGodItsFriday{}.Now() }, 709 | time.UTC, 710 | 1, 711 | }, 712 | } { 713 | chaoskube := suite.setupWithPods( 714 | labels.Everything(), 715 | labels.Everything(), 716 | labels.Everything(), 717 | labels.Everything(), 718 | labels.Everything(), 719 | ®exp.Regexp{}, 720 | ®exp.Regexp{}, 721 | tt.excludedWeekdays, 722 | tt.excludedTimesOfDay, 723 | tt.excludedDaysOfYear, 724 | tt.timezone, 725 | time.Duration(0), 726 | false, 727 | 10, 728 | v1.NamespaceAll, 729 | ) 730 | chaoskube.Now = tt.now 731 | 732 | err := chaoskube.TerminateVictims(context.Background()) 733 | suite.Require().NoError(err) 734 | 735 | pods, err := chaoskube.Candidates(context.Background()) 736 | suite.Require().NoError(err) 737 | 738 | suite.Len(pods, tt.remainingPodCount) 739 | } 740 | } 741 | 742 | // TestTerminateNoVictimLogsInfo tests that missing victim prints a log message 743 | func (suite *Suite) TestTerminateNoVictimLogsInfo() { 744 | chaoskube := suite.setup( 745 | labels.Everything(), 746 | labels.Everything(), 747 | labels.Everything(), 748 | labels.Everything(), 749 | labels.Everything(), 750 | ®exp.Regexp{}, 751 | ®exp.Regexp{}, 752 | []time.Weekday{}, 753 | []util.TimePeriod{}, 754 | []time.Time{}, 755 | time.UTC, 756 | time.Duration(0), 757 | false, 758 | 10, 759 | 1, 760 | v1.NamespaceAll, 761 | ) 762 | 763 | err := chaoskube.TerminateVictims(context.Background()) 764 | suite.Require().NoError(err) 765 | 766 | suite.AssertLog(logOutput, log.DebugLevel, msgVictimNotFound, log.Fields{}) 767 | } 768 | 769 | // helper functions 770 | 771 | func (suite *Suite) assertCandidates(chaoskube *Chaoskube, expected []map[string]string) { 772 | pods, err := chaoskube.Candidates(context.Background()) 773 | suite.Require().NoError(err) 774 | 775 | suite.AssertPods(pods, expected) 776 | } 777 | 778 | func (suite *Suite) assertVictims(chaoskube *Chaoskube, expected []map[string]string) { 779 | victims, err := chaoskube.Victims(context.Background()) 780 | suite.Require().NoError(err) 781 | 782 | for i, victim := range victims { 783 | suite.AssertPod(victim, expected[i]) 784 | } 785 | } 786 | 787 | func (suite *Suite) assertVictim(chaoskube *Chaoskube, expected map[string]string) { 788 | suite.assertVictims(chaoskube, []map[string]string{expected}) 789 | } 790 | 791 | func (suite *Suite) assertNotified(notifier *notifier.Noop) { 792 | suite.Assert().Greater(notifier.Calls, 0) 793 | } 794 | 795 | func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, clientNamespaceScope string) *Chaoskube { 796 | chaoskube := suite.setup( 797 | labelSelector, 798 | annotations, 799 | kinds, 800 | namespaces, 801 | namespaceLabels, 802 | includedPodNames, 803 | excludedPodNames, 804 | excludedWeekdays, 805 | excludedTimesOfDay, 806 | excludedDaysOfYear, 807 | timezone, 808 | minimumAge, 809 | dryRun, 810 | gracePeriod, 811 | 1, 812 | clientNamespaceScope, 813 | ) 814 | 815 | for _, namespace := range []v1.Namespace{ 816 | util.NewNamespace("default"), 817 | util.NewNamespace("testing"), 818 | } { 819 | _, err := chaoskube.Client.CoreV1().Namespaces().Create(context.Background(), &namespace, metav1.CreateOptions{}) 820 | suite.Require().NoError(err) 821 | } 822 | 823 | pods := []v1.Pod{ 824 | util.NewPod("default", "foo", v1.PodRunning), 825 | util.NewPod("testing", "bar", v1.PodRunning), 826 | util.NewPod("testing", "baz", v1.PodPending), // Non-running pods are ignored 827 | } 828 | 829 | for _, pod := range pods { 830 | _, err := chaoskube.Client.CoreV1().Pods(pod.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) 831 | suite.Require().NoError(err) 832 | } 833 | 834 | return chaoskube 835 | } 836 | 837 | func (suite *Suite) createPods(client kubernetes.Interface, podsInfo []podInfo) { 838 | for _, p := range podsInfo { 839 | namespace := util.NewNamespace(p.Namespace) 840 | _, err := client.CoreV1().Namespaces().Create(context.Background(), &namespace, metav1.CreateOptions{}) 841 | suite.Require().NoError(err) 842 | pod := util.NewPod(p.Namespace, p.Name, v1.PodRunning) 843 | _, err = client.CoreV1().Pods(p.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) 844 | suite.Require().NoError(err) 845 | } 846 | } 847 | 848 | func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, kinds labels.Selector, namespaces labels.Selector, namespaceLabels labels.Selector, includedPodNames *regexp.Regexp, excludedPodNames *regexp.Regexp, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, gracePeriod time.Duration, maxKill int, clientNamespaceScope string) *Chaoskube { 849 | logOutput.Reset() 850 | 851 | client := fake.NewSimpleClientset() 852 | nullLogger, _ := test.NewNullLogger() 853 | 854 | return New( 855 | client, 856 | labelSelector, 857 | annotations, 858 | kinds, 859 | namespaces, 860 | namespaceLabels, 861 | includedPodNames, 862 | excludedPodNames, 863 | excludedWeekdays, 864 | excludedTimesOfDay, 865 | excludedDaysOfYear, 866 | timezone, 867 | minimumAge, 868 | logger, 869 | dryRun, 870 | terminator.NewDeletePodTerminator(client, nullLogger, gracePeriod), 871 | maxKill, 872 | testNotifier, 873 | clientNamespaceScope, 874 | ) 875 | } 876 | 877 | func TestSuite(t *testing.T) { 878 | suite.Run(t, new(Suite)) 879 | } 880 | 881 | // ThankGodItsFriday is a helper struct that contains a Now() function that always returns a Friday. 882 | type ThankGodItsFriday struct{} 883 | 884 | // Now returns a particular Friday. 885 | func (t ThankGodItsFriday) Now() time.Time { 886 | blackFriday, _ := time.Parse(time.RFC1123, "Fri, 24 Sep 1869 15:04:05 UTC") 887 | return blackFriday 888 | } 889 | 890 | func (suite *Suite) TestMinimumAge() { 891 | type pod struct { 892 | name string 893 | namespace string 894 | creationTime time.Time 895 | } 896 | 897 | for _, tt := range []struct { 898 | minimumAge time.Duration 899 | now func() time.Time 900 | pods []pod 901 | candidates int 902 | }{ 903 | // no minimum age set 904 | { 905 | time.Duration(0), 906 | func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, 907 | []pod{ 908 | { 909 | name: "test1", 910 | namespace: "test", 911 | creationTime: time.Date(0, 10, 24, 9, 00, 00, 00, time.UTC), 912 | }, 913 | }, 914 | 1, 915 | }, 916 | // minimum age set, but pod is too young 917 | { 918 | time.Hour * 1, 919 | func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, 920 | []pod{ 921 | { 922 | name: "test1", 923 | namespace: "test", 924 | creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC), 925 | }, 926 | }, 927 | 0, 928 | }, 929 | // one pod is too young, one matches 930 | { 931 | time.Hour * 1, 932 | func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, 933 | []pod{ 934 | // too young 935 | { 936 | name: "test1", 937 | namespace: "test", 938 | creationTime: time.Date(0, 10, 24, 9, 30, 00, 00, time.UTC), 939 | }, 940 | // matches 941 | { 942 | name: "test2", 943 | namespace: "test", 944 | creationTime: time.Date(0, 10, 23, 8, 00, 00, 00, time.UTC), 945 | }, 946 | }, 947 | 1, 948 | }, 949 | // exact time - should not match 950 | { 951 | time.Hour * 1, 952 | func() time.Time { return time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC) }, 953 | []pod{ 954 | { 955 | name: "test1", 956 | namespace: "test", 957 | creationTime: time.Date(0, 10, 24, 10, 00, 00, 00, time.UTC), 958 | }, 959 | }, 960 | 0, 961 | }, 962 | } { 963 | chaoskube := suite.setup( 964 | labels.Everything(), 965 | labels.Everything(), 966 | labels.Everything(), 967 | labels.Everything(), 968 | labels.Everything(), 969 | ®exp.Regexp{}, 970 | ®exp.Regexp{}, 971 | []time.Weekday{}, 972 | []util.TimePeriod{}, 973 | []time.Time{}, 974 | time.UTC, 975 | tt.minimumAge, 976 | false, 977 | 10, 978 | 1, 979 | v1.NamespaceAll, 980 | ) 981 | chaoskube.Now = tt.now 982 | 983 | for _, p := range tt.pods { 984 | pod := util.NewPod(p.namespace, p.name, v1.PodRunning) 985 | pod.ObjectMeta.CreationTimestamp = metav1.Time{Time: p.creationTime} 986 | _, err := chaoskube.Client.CoreV1().Pods(pod.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) 987 | suite.Require().NoError(err) 988 | } 989 | 990 | pods, err := chaoskube.Candidates(context.Background()) 991 | suite.Require().NoError(err) 992 | 993 | suite.Len(pods, tt.candidates) 994 | } 995 | } 996 | 997 | func (suite *Suite) TestFilterDeletedPods() { 998 | deletedPod := util.NewPod("default", "deleted", v1.PodRunning) 999 | now := metav1.NewTime(time.Now()) 1000 | deletedPod.SetDeletionTimestamp(&now) 1001 | 1002 | runningPod := util.NewPod("default", "running", v1.PodRunning) 1003 | 1004 | pods := []v1.Pod{runningPod, deletedPod} 1005 | 1006 | filtered := filterTerminatingPods(pods) 1007 | suite.Equal(len(filtered), 1) 1008 | suite.Equal(pods[0].Name, "running") 1009 | } 1010 | 1011 | func (suite *Suite) TestFilterByKinds() { 1012 | foo := util.NewPodWithOwner("default", "foo", v1.PodRunning, "parent-1") 1013 | foo1 := util.NewPodWithOwner("default", "foo-1", v1.PodRunning, "parent-2") 1014 | bar := util.NewPodWithOwner("default", "bar", v1.PodRunning, "other-parent") 1015 | baz := util.NewPod("default", "baz", v1.PodRunning) 1016 | baz1 := util.NewPod("default", "baz-1", v1.PodRunning) 1017 | 1018 | for _, tt := range []struct { 1019 | name string 1020 | kinds string 1021 | pods []v1.Pod 1022 | expected []v1.Pod 1023 | }{ 1024 | { 1025 | name: "2 pods, one with owner ref", 1026 | kinds: "testkind", 1027 | pods: []v1.Pod{foo, baz}, 1028 | expected: []v1.Pod{foo}, 1029 | }, 1030 | { 1031 | name: "5 pods, 3 with owner ref", 1032 | kinds: "!testkind", 1033 | pods: []v1.Pod{foo, foo1, baz, bar, baz1}, 1034 | expected: []v1.Pod{baz, baz1}, 1035 | }, 1036 | { 1037 | name: "3 pods with owner ref, different kind", 1038 | kinds: "!testkind", 1039 | pods: []v1.Pod{foo, foo1, bar}, 1040 | expected: []v1.Pod{}, 1041 | }, 1042 | { 1043 | name: "3 pods with owner ref, different kind", 1044 | kinds: "!testkind,!job", 1045 | pods: []v1.Pod{foo, baz}, 1046 | expected: []v1.Pod{baz}, 1047 | }, 1048 | { 1049 | name: "3 pods with owner ref, different kind", 1050 | kinds: "testkind,job", 1051 | pods: []v1.Pod{foo, foo1, bar, baz}, 1052 | expected: []v1.Pod{foo, foo1, bar}, 1053 | }, 1054 | { 1055 | name: "3 pods with owner ref, different kind", 1056 | kinds: "!testkind,job", 1057 | pods: []v1.Pod{foo, foo1, bar, baz}, 1058 | expected: []v1.Pod{}, 1059 | }, 1060 | { 1061 | name: "3 pods with owner ref, different kind", 1062 | kinds: "testkind,!job", 1063 | pods: []v1.Pod{foo, foo1, bar, baz}, 1064 | expected: []v1.Pod{foo, foo1, bar}, 1065 | }, 1066 | { 1067 | name: "3 pods with owner ref, different kind", 1068 | kinds: "job", 1069 | pods: []v1.Pod{foo, foo1, bar, baz}, 1070 | expected: []v1.Pod{}, 1071 | }, 1072 | } { 1073 | kindsSelector, err := labels.Parse(tt.kinds) 1074 | suite.Require().NoError(err) 1075 | 1076 | results, err := filterByKinds(tt.pods, kindsSelector) 1077 | suite.Require().Len(results, len(tt.expected)) 1078 | suite.Require().NoError(err) 1079 | 1080 | for i, result := range results { 1081 | suite.Assert().Equal(tt.expected[i], result, tt.name) 1082 | } 1083 | } 1084 | } 1085 | 1086 | func (suite *Suite) TestFilterByOwnerReference() { 1087 | foo := util.NewPodWithOwner("default", "foo", v1.PodRunning, "parent") 1088 | foo1 := util.NewPodWithOwner("default", "foo-1", v1.PodRunning, "parent") 1089 | bar := util.NewPodWithOwner("default", "bar", v1.PodRunning, "other-parent") 1090 | baz := util.NewPod("default", "baz", v1.PodRunning) 1091 | baz1 := util.NewPod("default", "baz-1", v1.PodRunning) 1092 | 1093 | for _, tt := range []struct { 1094 | seed int64 1095 | name string 1096 | pods []v1.Pod 1097 | expected []v1.Pod 1098 | }{ 1099 | { 1100 | seed: 1000, 1101 | name: "2 pods, same parent, pick first", 1102 | pods: []v1.Pod{foo, foo1}, 1103 | expected: []v1.Pod{foo}, 1104 | }, 1105 | { 1106 | seed: 2000, 1107 | name: "2 pods, same parent, pick second", 1108 | pods: []v1.Pod{foo, foo1}, 1109 | expected: []v1.Pod{foo1}, 1110 | }, 1111 | { 1112 | seed: 1000, 1113 | name: "2 pods, different parents, pick both", 1114 | pods: []v1.Pod{bar, foo}, 1115 | expected: []v1.Pod{bar, foo}, 1116 | }, 1117 | { 1118 | seed: 1000, 1119 | name: "2 pods, one without and one with parent, pick both", 1120 | pods: []v1.Pod{baz, foo}, 1121 | expected: []v1.Pod{baz, foo}, 1122 | }, 1123 | { 1124 | seed: 1000, 1125 | name: "2 pods, no parents, pick both", 1126 | pods: []v1.Pod{baz, baz1}, 1127 | expected: []v1.Pod{baz, baz1}, 1128 | }, 1129 | } { 1130 | rand.Seed(tt.seed) 1131 | 1132 | results := filterByOwnerReference(tt.pods) 1133 | suite.Require().Len(results, len(tt.expected)) 1134 | 1135 | // ensure returned pods are ordered by name 1136 | // to make the following assertion work correctly 1137 | sort.Slice(results, func(i, j int) bool { 1138 | return results[i].Name < results[j].Name 1139 | }) 1140 | for i, result := range results { 1141 | suite.Assert().Equal(tt.expected[i], result, tt.name) 1142 | } 1143 | } 1144 | } 1145 | 1146 | func (suite *Suite) TestNotifierCall() { 1147 | chaoskube := suite.setupWithPods( 1148 | labels.Everything(), 1149 | labels.Everything(), 1150 | labels.Everything(), 1151 | labels.Everything(), 1152 | labels.Everything(), 1153 | ®exp.Regexp{}, 1154 | ®exp.Regexp{}, 1155 | []time.Weekday{}, 1156 | []util.TimePeriod{}, 1157 | []time.Time{}, 1158 | time.UTC, 1159 | time.Duration(0), 1160 | false, 1161 | 10, 1162 | v1.NamespaceAll, 1163 | ) 1164 | 1165 | victim := util.NewPod("default", "foo", v1.PodRunning) 1166 | err := chaoskube.DeletePod(context.Background(), victim) 1167 | 1168 | suite.Require().NoError(err) 1169 | suite.assertNotified(testNotifier) 1170 | } 1171 | -------------------------------------------------------------------------------- /chart/chaoskube/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /chart/chaoskube/Chart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v2 3 | name: chaoskube 4 | type: application 5 | description: Chaoskube periodically kills random pods in your Kubernetes cluster. 6 | icon: https://raw.githubusercontent.com/linki/chaoskube/master/chaoskube.png 7 | home: https://github.com/linki/chaoskube 8 | keywords: 9 | - chaos-monkey 10 | - chaos-engineering 11 | maintainers: 12 | - name: Martin Linkhorst 13 | url: https://github.com/linki 14 | - name: Thomas Gosteli 15 | url: https://github.com/ghouscht 16 | version: 0.4.0 17 | appVersion: 0.32.0 18 | -------------------------------------------------------------------------------- /chart/chaoskube/README.md: -------------------------------------------------------------------------------- 1 | # chaoskube Helm Chart 2 | 3 | chaoskube periodically kills random pods in your Kubernetes cluster. 4 | 5 | ## Installation 6 | 7 | ```console 8 | $ helm repo add chaoskube https://linki.github.io/chaoskube/ 9 | $ helm install chaoskube chaoskube/chaoskube --atomic --namespace=chaoskube --create-namespace 10 | ``` 11 | 12 | ## Example Helm values 13 | 14 | Basic configuration with `3` replicas and minimum resources assigned that will take out any pod it can find (including the other chaoskube pods): 15 | 16 | ```yaml 17 | chaoskube: 18 | args: 19 | no-dry-run: "" 20 | replicaCount: 3 21 | resources: 22 | limits: 23 | cpu: 15m 24 | memory: 32Mi 25 | requests: 26 | cpu: 15m 27 | memory: 32Mi 28 | ``` 29 | 30 | More advance configuration that limits based on several factors like time, day of the week, and date: 31 | 32 | ```yaml 33 | chaoskube: 34 | args: 35 | # kill a pod every 10 minutes 36 | interval: "10m" 37 | # only target pods in the test environment 38 | labels: "environment=test" 39 | # only consider pods with this annotation 40 | annotations: "chaos.alpha.kubernetes.io/enabled=true" 41 | # exclude all DaemonSet pods 42 | kinds: "!DaemonSet" 43 | # exclude all pods in the kube-system namespace 44 | namespaces: "!kube-system" 45 | # don't kill anything on weekends 46 | excluded-weekdays: "Sat,Sun" 47 | # don't kill anything during the night or at lunchtime 48 | excluded-times-of-day: "22:00-08:00,11:00-13:00" 49 | # don't kill anything as a joke or on christmas eve 50 | excluded-days-of-year: "Apr1,Dec24" 51 | # let's make sure we all agree on what the above times mean 52 | timezone: "UTC" 53 | # exclude all pods that haven't been running for at least one hour 54 | #minimum-age: "1h" 55 | # terminate pods for real: this disables dry-run mode which is on by default 56 | no-dry-run: "" 57 | replicaCount: 3 58 | resources: 59 | limits: 60 | cpu: 15m 61 | memory: 32Mi 62 | requests: 63 | cpu: 15m 64 | memory: 32Mi 65 | ``` 66 | -------------------------------------------------------------------------------- /chart/chaoskube/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | Chaoskube successfully installed in namespace {{ .Release.Namespace }}. 2 | Please visit https://github.com/linki/chaoskube for further information. 3 | -------------------------------------------------------------------------------- /chart/chaoskube/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "chaoskube.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "chaoskube.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "chaoskube.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "chaoskube.labels" -}} 37 | helm.sh/chart: {{ include "chaoskube.chart" . }} 38 | {{ include "chaoskube.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "chaoskube.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "chaoskube.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "chaoskube.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "chaoskube.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | -------------------------------------------------------------------------------- /chart/chaoskube/templates/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: {{ include "chaoskube.fullname" . }} 6 | rules: 7 | - apiGroups: [""] 8 | resources: ["pods"] 9 | verbs: ["list", "delete"] 10 | - apiGroups: [""] 11 | resources: ["events"] 12 | verbs: ["create"] 13 | -------------------------------------------------------------------------------- /chart/chaoskube/templates/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRoleBinding 4 | metadata: 5 | name: {{ include "chaoskube.fullname" . }} 6 | roleRef: 7 | apiGroup: rbac.authorization.k8s.io 8 | kind: ClusterRole 9 | name: {{ include "chaoskube.fullname" . }} 10 | subjects: 11 | - kind: ServiceAccount 12 | name: {{ include "chaoskube.serviceAccountName" . }} 13 | namespace: {{ .Release.Namespace }} 14 | -------------------------------------------------------------------------------- /chart/chaoskube/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: {{ include "chaoskube.fullname" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "chaoskube.labels" . | nindent 4 }} 9 | {{- if .Values.additionalLabels }} 10 | {{- range $key, $value := .Values.additionalLabels }} 11 | {{ $key }}: {{ $value | quote }} 12 | {{- end }} 13 | {{- end }} 14 | spec: 15 | strategy: 16 | type: Recreate 17 | replicas: {{ .Values.replicaCount }} 18 | selector: 19 | matchLabels: 20 | {{- include "chaoskube.selectorLabels" . | nindent 6 }} 21 | template: 22 | metadata: 23 | {{- with .Values.podAnnotations }} 24 | annotations: 25 | {{- toYaml . | nindent 8 }} 26 | {{- end }} 27 | labels: 28 | {{- include "chaoskube.selectorLabels" . | nindent 8 }} 29 | {{- if .Values.additionalLabels }} 30 | {{- range $key, $value := .Values.additionalLabels }} 31 | {{ $key }}: {{ $value | quote }} 32 | {{- end }} 33 | {{- end }} 34 | spec: 35 | tolerations: 36 | {{- toYaml .Values.tolerations | nindent 8 }} 37 | serviceAccountName: {{ include "chaoskube.serviceAccountName" . }} 38 | containers: 39 | - name: {{ .Chart.Name }} 40 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "v%s" .Chart.AppVersion) }}" 41 | imagePullPolicy: {{ .Values.image.pullPolicy }} 42 | {{- if .Values.chaoskube.env }} 43 | env: 44 | {{ toYaml .Values.chaoskube.env | indent 8 }} 45 | {{- end }} 46 | {{- if .Values.chaoskube.envFromConfigMapRefs }} 47 | envFrom: 48 | {{- range .Values.chaoskube.envFromConfigMapRefs }} 49 | - configMapRef: 50 | name: {{ . }} 51 | {{- end }} 52 | {{- end }} 53 | {{- with .Values.chaoskube.args }} 54 | args: 55 | {{- range $key, $value := . }} 56 | {{- if $value }} 57 | - --{{ $key }}={{ $value }} 58 | {{- else }} 59 | - --{{ $key }} 60 | {{- end }} 61 | {{- end }} 62 | {{- end }} 63 | securityContext: 64 | {{- toYaml .Values.podSecurityContext | nindent 10 }} 65 | {{- with .Values.resources}} 66 | resources: 67 | {{- toYaml . | nindent 10 }} 68 | {{- end }} 69 | -------------------------------------------------------------------------------- /chart/chaoskube/templates/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ include "chaoskube.serviceAccountName" . }} 6 | namespace: {{ .Release.Namespace }} 7 | labels: 8 | {{- include "chaoskube.labels" . | nindent 4 }} 9 | {{- with .Values.serviceAccount.annotations }} 10 | annotations: 11 | {{- toYaml . | nindent 4 }} 12 | {{- end }} 13 | -------------------------------------------------------------------------------- /chart/chaoskube/values.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | # replicaCount configures the number of replicas to run 3 | replicaCount: 1 4 | 5 | # image specifies image location, tag and pullPolicy 6 | image: 7 | repository: ghcr.io/linki/chaoskube 8 | pullPolicy: IfNotPresent 9 | # Overrides the image tag whose default is the chart appVersion. 10 | tag: "" 11 | 12 | # chaoskube is used to configure chaoskube 13 | chaoskube: 14 | env: {} 15 | envFromConfigMapRefs: [] 16 | # - 'configmap-a' 17 | args: {} 18 | ###### 19 | # Example configuration, uncomment and adjust to your needs. 20 | # Be sure to read: https://github.com/linki/chaoskube#flags 21 | ###### 22 | # kill a pod every 10 minutes 23 | #interval: "10m" 24 | # only target pods in the test environment 25 | #labels: "environment=test" 26 | # only consider pods with this annotation 27 | #annotations: "chaos.alpha.kubernetes.io/enabled=true" 28 | # exclude all DaemonSet pods 29 | #kinds: "!DaemonSet" 30 | # exclude all pods in the kube-system namespace 31 | #namespaces: "!kube-system" 32 | # don't kill anything on weekends 33 | #excluded-weekdays: "Sat,Sun" 34 | # don't kill anything during the night or at lunchtime 35 | #excluded-times-of-day: "22:00-08:00,11:00-13:00" 36 | # don't kill anything as a joke or on christmas eve 37 | #excluded-days-of-year: "Apr1,Dec24" 38 | # let's make sure we all agree on what the above times mean 39 | #timezone: "UTC" 40 | # exclude all pods that haven't been running for at least one hour 41 | #minimum-age: "1h" 42 | # terminate pods for real: this disables dry-run mode which is on by default 43 | #no-dry-run: "" 44 | 45 | # serviceAccount can be used to customize the service account which will be crated and used by chaoskube 46 | serviceAccount: 47 | create: true 48 | name: "" 49 | annotations: {} 50 | 51 | # podAnnotations can be used to add additional annotations to the pod 52 | podAnnotations: {} 53 | 54 | # additionalLabels can be used to add additional labels to the pod 55 | additionalLabels: {} 56 | 57 | # podSecurityContext is used to customize the security context of the pod 58 | podSecurityContext: 59 | runAsNonRoot: true 60 | runAsUser: 65534 61 | readOnlyRootFilesystem: true 62 | capabilities: 63 | drop: ["ALL"] 64 | 65 | # resources can be used to limit pod compute resources (limits/requests) 66 | resources: {} 67 | # limits: 68 | # cpu: 100m 69 | # memory: 128Mi 70 | # requests: 71 | # cpu: 10m 72 | # memory: 16Mi 73 | 74 | # tolerations can be used to configure tolerations for the pods 75 | tolerations: [] 76 | -------------------------------------------------------------------------------- /examples/deployment/chaoskube.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: chaoskube 5 | labels: 6 | app: chaoskube 7 | spec: 8 | strategy: 9 | type: Recreate 10 | selector: 11 | matchLabels: 12 | app: chaoskube 13 | template: 14 | metadata: 15 | labels: 16 | app: chaoskube 17 | spec: 18 | serviceAccountName: chaoskube 19 | containers: 20 | - name: chaoskube 21 | image: ghcr.io/linki/chaoskube:v0.29.0 22 | args: 23 | # kill a pod every 10 minutes 24 | - --interval=10m 25 | # only target pods in the test environment 26 | - --labels=environment=test 27 | # only consider pods with this annotation 28 | - --annotations=chaos.alpha.kubernetes.io/enabled=true 29 | # exclude all DaemonSet pods 30 | - --kinds=!DaemonSet 31 | # exclude all pods in the kube-system namespace 32 | - --namespaces=!kube-system 33 | # don't kill anything on weekends 34 | - --excluded-weekdays=Sat,Sun 35 | # don't kill anything during the night or at lunchtime 36 | - --excluded-times-of-day=22:00-08:00,11:00-13:00 37 | # don't kill anything as a joke or on christmas eve 38 | - --excluded-days-of-year=Apr1,Dec24 39 | # let's make sure we all agree on what the above times mean 40 | - --timezone=UTC 41 | # exclude all pods that haven't been running for at least one hour 42 | - --minimum-age=1h 43 | # terminate pods for real: this disables dry-run mode which is on by default 44 | - --no-dry-run 45 | securityContext: 46 | runAsNonRoot: true 47 | runAsUser: 65534 48 | readOnlyRootFilesystem: true 49 | capabilities: 50 | drop: ["ALL"] 51 | -------------------------------------------------------------------------------- /examples/job/chaoskube.yaml: -------------------------------------------------------------------------------- 1 | # chaoskube can stop after some time. This can be done 2 | # if it is deployed as a Job and having --max-runtime set as well 3 | apiVersion: batch/v1 4 | kind: Job 5 | metadata: 6 | name: chaoskube 7 | labels: 8 | app: chaoskube 9 | spec: 10 | template: 11 | metadata: 12 | labels: 13 | app: chaoskube 14 | spec: 15 | restartPolicy: Never 16 | serviceAccountName: chaoskube 17 | containers: 18 | - name: chaoskube 19 | image: ghcr.io/linki/chaoskube:v0.29.0 20 | args: 21 | # kill a pod every 10 minutes 22 | - --interval=10m 23 | # only target pods in the test environment 24 | - --labels=environment=test 25 | # only consider pods with this annotation 26 | - --annotations=chaos.alpha.kubernetes.io/enabled=true 27 | # exclude all DaemonSet pods 28 | - --kinds=!DaemonSet 29 | # exclude all pods in the kube-system namespace 30 | - --namespaces=!kube-system 31 | # don't kill anything on weekends 32 | - --excluded-weekdays=Sat,Sun 33 | # don't kill anything during the night or at lunchtime 34 | - --excluded-times-of-day=22:00-08:00,11:00-13:00 35 | # don't kill anything as a joke or on christmas eve 36 | - --excluded-days-of-year=Apr1,Dec24 37 | # let's make sure we all agree on what the above times mean 38 | - --timezone=UTC 39 | # exclude all pods that haven't been running for at least one hour 40 | - --minimum-age=1h 41 | # terminate pods for real: this disables dry-run mode which is on by default 42 | - --no-dry-run 43 | # if set, chaoskube will exit after the max runtime 44 | - --max-runtime=3600s 45 | securityContext: 46 | runAsNonRoot: true 47 | runAsUser: 65534 48 | readOnlyRootFilesystem: true 49 | capabilities: 50 | drop: ["ALL"] 51 | -------------------------------------------------------------------------------- /examples/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: chaoskube 5 | rules: 6 | - apiGroups: [""] 7 | resources: ["pods"] 8 | verbs: ["list", "delete"] 9 | - apiGroups: [""] 10 | resources: ["events"] 11 | verbs: ["create"] 12 | --- 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | kind: ClusterRoleBinding 15 | metadata: 16 | name: chaoskube 17 | roleRef: 18 | apiGroup: rbac.authorization.k8s.io 19 | kind: ClusterRole 20 | name: chaoskube 21 | subjects: 22 | - kind: ServiceAccount 23 | name: chaoskube 24 | namespace: default 25 | --- 26 | apiVersion: v1 27 | kind: ServiceAccount 28 | metadata: 29 | name: chaoskube 30 | labels: 31 | app: chaoskube 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/linki/chaoskube 2 | 3 | go 1.24.2 4 | 5 | require ( 6 | github.com/alecthomas/kingpin/v2 v2.4.0 7 | github.com/hashicorp/go-multierror v1.1.1 8 | github.com/prometheus/client_golang v1.22.0 9 | github.com/sirupsen/logrus v1.9.3 10 | github.com/stretchr/testify v1.10.0 11 | k8s.io/api v0.33.0 12 | k8s.io/apimachinery v0.33.0 13 | k8s.io/client-go v0.33.0 14 | k8s.io/klog v1.0.0 15 | ) 16 | 17 | require ( 18 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 19 | github.com/beorn7/perks v1.0.1 // indirect 20 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 21 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 22 | github.com/emicklei/go-restful/v3 v3.11.0 // indirect 23 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 24 | github.com/go-logr/logr v1.4.2 // indirect 25 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 26 | github.com/go-openapi/jsonreference v0.20.2 // indirect 27 | github.com/go-openapi/swag v0.23.0 // indirect 28 | github.com/gogo/protobuf v1.3.2 // indirect 29 | github.com/google/gnostic-models v0.6.9 // indirect 30 | github.com/google/go-cmp v0.7.0 // indirect 31 | github.com/google/uuid v1.6.0 // indirect 32 | github.com/hashicorp/errwrap v1.0.0 // indirect 33 | github.com/josharian/intern v1.0.0 // indirect 34 | github.com/json-iterator/go v1.1.12 // indirect 35 | github.com/mailru/easyjson v0.7.7 // indirect 36 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 37 | github.com/modern-go/reflect2 v1.0.2 // indirect 38 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 39 | github.com/pkg/errors v0.9.1 // indirect 40 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 41 | github.com/prometheus/client_model v0.6.1 // indirect 42 | github.com/prometheus/common v0.62.0 // indirect 43 | github.com/prometheus/procfs v0.15.1 // indirect 44 | github.com/spf13/pflag v1.0.5 // indirect 45 | github.com/x448/float16 v0.8.4 // indirect 46 | github.com/xhit/go-str2duration/v2 v2.1.0 // indirect 47 | golang.org/x/net v0.38.0 // indirect 48 | golang.org/x/oauth2 v0.27.0 // indirect 49 | golang.org/x/sys v0.31.0 // indirect 50 | golang.org/x/term v0.30.0 // indirect 51 | golang.org/x/text v0.23.0 // indirect 52 | golang.org/x/time v0.9.0 // indirect 53 | google.golang.org/protobuf v1.36.5 // indirect 54 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 55 | gopkg.in/inf.v0 v0.9.1 // indirect 56 | gopkg.in/yaml.v3 v3.0.1 // indirect 57 | k8s.io/klog/v2 v2.130.1 // indirect 58 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect 59 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect 60 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 61 | sigs.k8s.io/randfill v1.0.0 // indirect 62 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect 63 | sigs.k8s.io/yaml v1.4.0 // indirect 64 | ) 65 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= 2 | github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= 3 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= 4 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 5 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 6 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 7 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 8 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 10 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 13 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= 15 | github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= 16 | github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= 17 | github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= 18 | github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= 19 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 20 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 21 | github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= 22 | github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= 23 | github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= 24 | github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= 25 | github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= 26 | github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= 27 | github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= 28 | github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= 29 | github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= 30 | github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= 31 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 32 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 33 | github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= 34 | github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= 35 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 36 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 37 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 38 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 39 | github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= 40 | github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= 41 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 42 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 43 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 44 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 45 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 46 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 47 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 48 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 49 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 50 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 51 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 52 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 53 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 54 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 55 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 56 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 57 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 58 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 59 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 60 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 61 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 62 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 63 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 64 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 65 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 66 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 67 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 68 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 69 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 70 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 71 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 72 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 73 | github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= 74 | github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= 75 | github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= 76 | github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= 77 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 78 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 79 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 80 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 81 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 82 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 83 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 84 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 85 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 86 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 87 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 88 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 89 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 90 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 91 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 92 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 93 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 94 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 95 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 96 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 97 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 98 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 99 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 100 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 101 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 102 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 103 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 104 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 105 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 106 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 107 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 108 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 109 | github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= 110 | github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= 111 | github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= 112 | github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= 113 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 114 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 115 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 116 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 117 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 118 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 119 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 120 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 121 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 122 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 123 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 124 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 125 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 126 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 127 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 128 | golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= 129 | golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= 130 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 131 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 132 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 133 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 134 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 135 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 136 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 137 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 138 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 139 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 140 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 141 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 142 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 143 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 144 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 145 | golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= 146 | golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 147 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 148 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 149 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 150 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 151 | golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= 152 | golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= 153 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 154 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 155 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 156 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 157 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 158 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 159 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 160 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 161 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 162 | gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= 163 | gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= 164 | gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= 165 | gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= 166 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 167 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 168 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 169 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 170 | k8s.io/api v0.33.0 h1:yTgZVn1XEe6opVpP1FylmNrIFWuDqe2H0V8CT5gxfIU= 171 | k8s.io/api v0.33.0/go.mod h1:CTO61ECK/KU7haa3qq8sarQ0biLq2ju405IZAd9zsiM= 172 | k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= 173 | k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= 174 | k8s.io/client-go v0.33.0 h1:UASR0sAYVUzs2kYuKn/ZakZlcs2bEHaizrrHUZg0G98= 175 | k8s.io/client-go v0.33.0/go.mod h1:kGkd+l/gNGg8GYWAPr0xF1rRKvVWvzh9vmZAMXtaKOg= 176 | k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= 177 | k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= 178 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 179 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 180 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= 181 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= 182 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= 183 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 184 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= 185 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= 186 | sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= 187 | sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= 188 | sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= 189 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0 h1:IUA9nvMmnKWcj5jl84xn+T5MnlZKThmUW1TdblaLVAc= 190 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0/go.mod h1:dDy58f92j70zLsuZVuUX5Wp9vtxXpaZnkPGWeqDfCps= 191 | sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= 192 | sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= 193 | -------------------------------------------------------------------------------- /internal/testutil/assert.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | log "github.com/sirupsen/logrus" 5 | "github.com/sirupsen/logrus/hooks/test" 6 | 7 | "k8s.io/api/core/v1" 8 | 9 | "github.com/stretchr/testify/suite" 10 | ) 11 | 12 | type TestSuite struct { 13 | suite.Suite 14 | } 15 | 16 | func (suite *TestSuite) AssertPods(pods []v1.Pod, expected []map[string]string) { 17 | suite.Require().Len(pods, len(expected)) 18 | 19 | for i, pod := range pods { 20 | suite.AssertPod(pod, expected[i]) 21 | } 22 | } 23 | 24 | func (suite *TestSuite) AssertPod(pod v1.Pod, expected map[string]string) { 25 | suite.Equal(expected["namespace"], pod.Namespace) 26 | suite.Equal(expected["name"], pod.Name) 27 | } 28 | 29 | func (suite *TestSuite) AssertLog(output *test.Hook, level log.Level, msg string, fields log.Fields) { 30 | suite.Require().NotEmpty(output.Entries) 31 | 32 | lastEntry := output.LastEntry() 33 | suite.Equal(level, lastEntry.Level) 34 | suite.Equal(msg, lastEntry.Message) 35 | for k := range fields { 36 | suite.Equal(fields[k], lastEntry.Data[k]) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "math/rand" 8 | "net/http" 9 | _ "net/http/pprof" 10 | "os" 11 | "os/signal" 12 | "path" 13 | "regexp" 14 | "runtime" 15 | "syscall" 16 | "time" 17 | 18 | "github.com/alecthomas/kingpin/v2" 19 | "github.com/prometheus/client_golang/prometheus/promhttp" 20 | log "github.com/sirupsen/logrus" 21 | 22 | v1 "k8s.io/api/core/v1" 23 | "k8s.io/apimachinery/pkg/labels" 24 | "k8s.io/client-go/kubernetes" 25 | _ "k8s.io/client-go/plugin/pkg/client/auth" 26 | "k8s.io/client-go/tools/clientcmd" 27 | "k8s.io/klog" 28 | 29 | "github.com/linki/chaoskube/chaoskube" 30 | "github.com/linki/chaoskube/notifier" 31 | "github.com/linki/chaoskube/terminator" 32 | "github.com/linki/chaoskube/util" 33 | ) 34 | 35 | const envVarPrefix = "CHAOSKUBE_" 36 | 37 | var version = "undefined" 38 | 39 | var ( 40 | labelString string 41 | annString string 42 | kindsString string 43 | nsString string 44 | nsLabelString string 45 | includedPodNames *regexp.Regexp 46 | excludedPodNames *regexp.Regexp 47 | excludedWeekdays string 48 | excludedTimesOfDay string 49 | excludedDaysOfYear string 50 | timezone string 51 | minimumAge time.Duration 52 | maxRuntime time.Duration 53 | maxKill int 54 | master string 55 | kubeconfig string 56 | interval time.Duration 57 | dryRun bool 58 | debug bool 59 | metricsAddress string 60 | gracePeriod time.Duration 61 | logFormat string 62 | logCaller bool 63 | slackWebhook string 64 | clientNamespaceScope string 65 | ) 66 | 67 | func cliEnvVar(name string) string { 68 | return envVarPrefix + name 69 | } 70 | 71 | func init() { 72 | rand.Seed(time.Now().UTC().UnixNano()) 73 | klog.SetOutput(io.Discard) 74 | 75 | kingpin.Flag("labels", "A set of labels to restrict the list of affected pods. Defaults to everything.").Envar(cliEnvVar("LABELS")).StringVar(&labelString) 76 | kingpin.Flag("annotations", "A set of annotations to restrict the list of affected pods. Defaults to everything.").Envar(cliEnvVar("ANNOTATIONS")).StringVar(&annString) 77 | kingpin.Flag("kinds", "A set of kinds to restrict the list of affected pods. Defaults to everything.").Envar(cliEnvVar("KINDS")).StringVar(&kindsString) 78 | kingpin.Flag("namespaces", "A set of namespaces to restrict the list of affected pods. Defaults to everything.").Envar(cliEnvVar("NAMESPACES")).StringVar(&nsString) 79 | kingpin.Flag("namespace-labels", "A set of labels to restrict the list of affected namespaces. Defaults to everything.").Envar(cliEnvVar("NAMESPACE_LABELS")).StringVar(&nsLabelString) 80 | kingpin.Flag("included-pod-names", "Regular expression that defines which pods to include. All included by default.").Envar(cliEnvVar("INCLUDED_POD_NAMES")).RegexpVar(&includedPodNames) 81 | kingpin.Flag("excluded-pod-names", "Regular expression that defines which pods to exclude. None excluded by default.").Envar(cliEnvVar("EXCLUDED_POD_NAMES")).RegexpVar(&excludedPodNames) 82 | kingpin.Flag("excluded-weekdays", "A list of weekdays when termination is suspended, e.g. Sat,Sun").Envar(cliEnvVar("EXCLUDED_WEEKDAYS")).StringVar(&excludedWeekdays) 83 | kingpin.Flag("excluded-times-of-day", "A list of time periods of a day when termination is suspended, e.g. 22:00-08:00").Envar(cliEnvVar("EXCLUDED_TIMES_OF_DAY")).StringVar(&excludedTimesOfDay) 84 | kingpin.Flag("excluded-days-of-year", "A list of days of a year when termination is suspended, e.g. Apr1,Dec24").Envar(cliEnvVar("EXCLUDED_DAYS_OF_YEAR")).StringVar(&excludedDaysOfYear) 85 | kingpin.Flag("timezone", "The timezone by which to interpret the excluded weekdays and times of day, e.g. UTC, Local, Europe/Berlin. Defaults to UTC.").Envar(cliEnvVar("TIMEZONE")).Default("UTC").StringVar(&timezone) 86 | kingpin.Flag("minimum-age", "Minimum age of pods to consider for termination").Envar(cliEnvVar("MINIMUM_AGE")).Default("0s").DurationVar(&minimumAge) 87 | kingpin.Flag("max-runtime", "Maximum runtime before chaoskube exits").Envar(cliEnvVar("MAX_RUNTIME")).Default("-1s").DurationVar(&maxRuntime) 88 | kingpin.Flag("max-kill", "Specifies the maximum number of pods to be terminated per interval.").Envar(cliEnvVar("MAX_KILL")).Default("1").IntVar(&maxKill) 89 | kingpin.Flag("master", "The address of the Kubernetes cluster to target").Envar(cliEnvVar("MASTER")).StringVar(&master) 90 | kingpin.Flag("kubeconfig", "Path to a kubeconfig file").Envar(cliEnvVar("KUBECONFIG")).StringVar(&kubeconfig) 91 | kingpin.Flag("interval", "Interval between Pod terminations").Envar(cliEnvVar("INTERVAL")).Default("10m").DurationVar(&interval) 92 | kingpin.Flag("dry-run", "Don't actually kill any pod. Turned on by default. Turn off with `--no-dry-run`.").Envar(cliEnvVar("DRY_RUN")).Default("true").BoolVar(&dryRun) 93 | kingpin.Flag("debug", "Enable debug logging.").Envar(cliEnvVar("DEBUG")).BoolVar(&debug) 94 | kingpin.Flag("metrics-address", "Listening address for metrics handler").Envar(cliEnvVar("METRICS_ADDRESS")).Default(":8080").StringVar(&metricsAddress) 95 | kingpin.Flag("grace-period", "Grace period to terminate Pods. Negative values will use the Pod's grace period.").Envar(cliEnvVar("GRACE_PERIOD")).Default("-1s").DurationVar(&gracePeriod) 96 | kingpin.Flag("log-format", "Specify the format of the log messages. Options are text and json. Defaults to text.").Envar(cliEnvVar("LOG_FORMAT")).Default("text").EnumVar(&logFormat, "text", "json") 97 | kingpin.Flag("log-caller", "Include the calling function name and location in the log messages.").Envar(cliEnvVar("LOG_CALLER")).BoolVar(&logCaller) 98 | kingpin.Flag("slack-webhook", "The address of the slack webhook for notifications").Envar(cliEnvVar("SLACK_WEBHOOK")).StringVar(&slackWebhook) 99 | kingpin.Flag("client-namespace-scope", "Scope Kubernetes API calls to the given namespace. Defaults to v1.NamespaceAll which requires global read permission.").Envar(cliEnvVar("CLIENT_NAMESPACE_SCOPE")).Default(v1.NamespaceAll).StringVar(&clientNamespaceScope) 100 | } 101 | 102 | func main() { 103 | kingpin.Version(version) 104 | kingpin.Parse() 105 | 106 | if debug { 107 | log.SetLevel(log.DebugLevel) 108 | } 109 | 110 | switch logFormat { 111 | case "json": 112 | log.SetFormatter(&log.JSONFormatter{CallerPrettyfier: prettifyCaller}) 113 | default: 114 | log.SetFormatter(&log.TextFormatter{CallerPrettyfier: prettifyCaller}) 115 | } 116 | 117 | log.SetReportCaller(logCaller) 118 | 119 | log.WithFields(log.Fields{ 120 | "labels": labelString, 121 | "annotations": annString, 122 | "kinds": kindsString, 123 | "namespaces": nsString, 124 | "namespaceLabels": nsLabelString, 125 | "includedPodNames": includedPodNames, 126 | "excludedPodNames": excludedPodNames, 127 | "excludedWeekdays": excludedWeekdays, 128 | "excludedTimesOfDay": excludedTimesOfDay, 129 | "excludedDaysOfYear": excludedDaysOfYear, 130 | "timezone": timezone, 131 | "minimumAge": minimumAge, 132 | "maxRuntime": maxRuntime, 133 | "maxKill": maxKill, 134 | "master": master, 135 | "kubeconfig": kubeconfig, 136 | "interval": interval, 137 | "dryRun": dryRun, 138 | "debug": debug, 139 | "metricsAddress": metricsAddress, 140 | "gracePeriod": gracePeriod, 141 | "logFormat": logFormat, 142 | "slackWebhook": slackWebhook, 143 | "clientNamespaceScope": clientNamespaceScope, 144 | }).Debug("reading config") 145 | 146 | log.WithFields(log.Fields{ 147 | "version": version, 148 | "dryRun": dryRun, 149 | "interval": interval, 150 | "maxRuntime": maxRuntime, 151 | }).Info("starting up") 152 | 153 | client, err := newClient() 154 | if err != nil { 155 | log.WithField("err", err).Fatal("failed to connect to cluster") 156 | } 157 | 158 | var ( 159 | labelSelector = parseSelector(labelString) 160 | annotations = parseSelector(annString) 161 | kinds = parseSelector(kindsString) 162 | namespaces = parseSelector(nsString) 163 | namespaceLabels = parseSelector(nsLabelString) 164 | ) 165 | 166 | log.WithFields(log.Fields{ 167 | "labels": labelSelector.String(), 168 | "annotations": annotations.String(), 169 | "kinds": kinds.String(), 170 | "namespaces": namespaces.String(), 171 | "namespaceLabels": namespaceLabels.String(), 172 | "includedPodNames": includedPodNames, 173 | "excludedPodNames": excludedPodNames, 174 | "minimumAge": minimumAge, 175 | "maxKill": maxKill, 176 | }).Info("setting pod filter") 177 | 178 | parsedWeekdays := util.ParseWeekdays(excludedWeekdays) 179 | parsedTimesOfDay, err := util.ParseTimePeriods(excludedTimesOfDay) 180 | if err != nil { 181 | log.WithFields(log.Fields{ 182 | "timesOfDay": excludedTimesOfDay, 183 | "err": err, 184 | }).Fatal("failed to parse times of day") 185 | } 186 | parsedDaysOfYear, err := util.ParseDays(excludedDaysOfYear) 187 | if err != nil { 188 | log.WithFields(log.Fields{ 189 | "daysOfYear": excludedDaysOfYear, 190 | "err": err, 191 | }).Fatal("failed to parse days of year") 192 | } 193 | 194 | log.WithFields(log.Fields{ 195 | "weekdays": parsedWeekdays, 196 | "timesOfDay": excludedTimesOfDay, 197 | "daysOfYear": util.FormatDays(parsedDaysOfYear), 198 | }).Info("setting quiet times") 199 | 200 | parsedTimezone, err := time.LoadLocation(timezone) 201 | if err != nil { 202 | log.WithFields(log.Fields{ 203 | "timeZone": timezone, 204 | "err": err, 205 | }).Fatal("failed to detect time zone") 206 | } 207 | timezoneName, offset := time.Now().In(parsedTimezone).Zone() 208 | 209 | log.WithFields(log.Fields{ 210 | "name": timezoneName, 211 | "location": parsedTimezone, 212 | "offset": offset / int(time.Hour/time.Second), 213 | }).Info("setting timezone") 214 | 215 | notifiers := createNotifier() 216 | 217 | chaoskube := chaoskube.New( 218 | client, 219 | labelSelector, 220 | annotations, 221 | kinds, 222 | namespaces, 223 | namespaceLabels, 224 | includedPodNames, 225 | excludedPodNames, 226 | parsedWeekdays, 227 | parsedTimesOfDay, 228 | parsedDaysOfYear, 229 | parsedTimezone, 230 | minimumAge, 231 | log.StandardLogger(), 232 | dryRun, 233 | terminator.NewDeletePodTerminator(client, log.StandardLogger(), gracePeriod), 234 | maxKill, 235 | notifiers, 236 | clientNamespaceScope, 237 | ) 238 | 239 | if metricsAddress != "" { 240 | go serveMetrics() 241 | } 242 | 243 | done := make(chan os.Signal, 1) 244 | signal.Notify(done, syscall.SIGINT, syscall.SIGTERM) 245 | 246 | ctx, cancel := context.WithCancel(context.Background()) 247 | 248 | if maxRuntime > -1 { 249 | ctx, cancel = context.WithTimeout(ctx, maxRuntime) 250 | } 251 | 252 | defer cancel() 253 | 254 | go func() { 255 | <-done 256 | cancel() 257 | }() 258 | 259 | ticker := time.NewTicker(interval) 260 | defer ticker.Stop() 261 | 262 | chaoskube.Run(ctx, ticker.C) 263 | } 264 | 265 | func newClient() (*kubernetes.Clientset, error) { 266 | if kubeconfig == "" { 267 | if _, err := os.Stat(clientcmd.RecommendedHomeFile); err == nil { 268 | kubeconfig = clientcmd.RecommendedHomeFile 269 | } 270 | } 271 | 272 | log.WithFields(log.Fields{ 273 | "kubeconfig": kubeconfig, 274 | "master": master, 275 | }).Debug("using cluster config") 276 | 277 | config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig) 278 | if err != nil { 279 | return nil, err 280 | } 281 | 282 | client, err := kubernetes.NewForConfig(config) 283 | if err != nil { 284 | return nil, err 285 | } 286 | 287 | serverVersion, err := client.Discovery().ServerVersion() 288 | if err != nil { 289 | return nil, err 290 | } 291 | 292 | log.WithFields(log.Fields{ 293 | "master": config.Host, 294 | "serverVersion": serverVersion, 295 | }).Info("connected to cluster") 296 | 297 | return client, nil 298 | } 299 | 300 | func parseSelector(str string) labels.Selector { 301 | selector, err := labels.Parse(str) 302 | if err != nil { 303 | log.WithFields(log.Fields{ 304 | "selector": str, 305 | "err": err, 306 | }).Fatal("failed to parse selector") 307 | } 308 | return selector 309 | } 310 | 311 | func createNotifier() notifier.Notifier { 312 | notifiers := notifier.New() 313 | if slackWebhook != "" { 314 | notifiers.Add(notifier.NewSlackNotifier(slackWebhook)) 315 | } 316 | 317 | return notifiers 318 | } 319 | 320 | func serveMetrics() { 321 | http.Handle("/metrics", promhttp.Handler()) 322 | http.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { 323 | fmt.Fprintln(w, "OK") 324 | }) 325 | http.HandleFunc("/", func(w http.ResponseWriter, _ *http.Request) { 326 | fmt.Fprintln(w, adminPage) 327 | }) 328 | if err := http.ListenAndServe(metricsAddress, nil); err != nil { 329 | log.WithField("err", err).Fatal("failed to start HTTP server") 330 | } 331 | } 332 | 333 | func prettifyCaller(f *runtime.Frame) (string, string) { 334 | _, filename := path.Split(f.File) 335 | return "", fmt.Sprintf("%s:%d", filename, f.Line) 336 | } 337 | 338 | var adminPage = ` 339 | 340 | chaoskube 341 | 342 | 343 |

chaoskube

344 |

Metrics

345 |

Health Check

346 |

pprof

347 | 348 | ` 349 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | // PodsDeletedTotal is the total number of deleted pods. 10 | PodsDeletedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ 11 | Namespace: "chaoskube", 12 | Name: "pods_deleted_total", 13 | Help: "The total number of pods deleted", 14 | }, []string{"namespace"}) 15 | // IntervalsTotal is the total number of intervals, i.e. call to Run(). 16 | IntervalsTotal = promauto.NewCounter(prometheus.CounterOpts{ 17 | Namespace: "chaoskube", 18 | Name: "intervals_total", 19 | Help: "The total number of pod termination logic runs", 20 | }) 21 | // ErrorsTotal is the total number of errors encountered while trying to terminate pods. 22 | ErrorsTotal = promauto.NewCounter(prometheus.CounterOpts{ 23 | Namespace: "chaoskube", 24 | Name: "errors_total", 25 | Help: "The total number of errors on terminate victim operation", 26 | }) 27 | // TerminationDurationSeconds is a histogram over the time it took to terminate pods. 28 | TerminationDurationSeconds = promauto.NewHistogram(prometheus.HistogramOpts{ 29 | Namespace: "chaoskube", 30 | Name: "termination_duration_seconds", 31 | Help: "The time it took a single pod termination to finish", 32 | }) 33 | ) 34 | -------------------------------------------------------------------------------- /notifier/noop.go: -------------------------------------------------------------------------------- 1 | package notifier 2 | 3 | import ( 4 | v1 "k8s.io/api/core/v1" 5 | ) 6 | 7 | const NotifierNoop = "noop" 8 | 9 | type Noop struct { 10 | Calls int 11 | } 12 | 13 | func (t *Noop) NotifyPodTermination(pod v1.Pod) error { 14 | t.Calls++ 15 | return nil 16 | } 17 | -------------------------------------------------------------------------------- /notifier/notifier.go: -------------------------------------------------------------------------------- 1 | package notifier 2 | 3 | import ( 4 | multierror "github.com/hashicorp/go-multierror" 5 | v1 "k8s.io/api/core/v1" 6 | ) 7 | 8 | type Notifier interface { 9 | NotifyPodTermination(pod v1.Pod) error 10 | } 11 | 12 | type Notifiers struct { 13 | notifiers []Notifier 14 | } 15 | 16 | func New() *Notifiers { 17 | return &Notifiers{notifiers: make([]Notifier, 0)} 18 | } 19 | 20 | func (m *Notifiers) NotifyPodTermination(pod v1.Pod) error { 21 | var result error 22 | for _, n := range m.notifiers { 23 | if err := n.NotifyPodTermination(pod); err != nil { 24 | result = multierror.Append(result, err) 25 | } 26 | } 27 | return result 28 | } 29 | 30 | func (m *Notifiers) Add(notifier Notifier) { 31 | m.notifiers = append(m.notifiers, notifier) 32 | } 33 | -------------------------------------------------------------------------------- /notifier/notifier_test.go: -------------------------------------------------------------------------------- 1 | package notifier 2 | 3 | import ( 4 | "fmt" 5 | "github.com/hashicorp/go-multierror" 6 | "testing" 7 | 8 | v1 "k8s.io/api/core/v1" 9 | 10 | "github.com/linki/chaoskube/internal/testutil" 11 | 12 | "github.com/stretchr/testify/suite" 13 | ) 14 | 15 | type NotifierSuite struct { 16 | testutil.TestSuite 17 | } 18 | 19 | type FailingNotifier struct{} 20 | 21 | func (f FailingNotifier) NotifyPodTermination(pod v1.Pod) error { 22 | return fmt.Errorf("notify error") 23 | } 24 | 25 | func (suite *NotifierSuite) TestMultiNotifierWithoutNotifiers() { 26 | manager := New() 27 | err := manager.NotifyPodTermination(v1.Pod{}) 28 | suite.NoError(err) 29 | } 30 | 31 | func (suite *NotifierSuite) TestMultiNotifierWithNotifier() { 32 | manager := New() 33 | n := Noop{} 34 | manager.Add(&n) 35 | err := manager.NotifyPodTermination(v1.Pod{}) 36 | suite.Require().NoError(err) 37 | 38 | suite.Equal(1, n.Calls) 39 | } 40 | 41 | func (suite *NotifierSuite) TestMultiNotifierWithMultipleNotifier() { 42 | manager := New() 43 | n1 := Noop{} 44 | n2 := Noop{} 45 | manager.Add(&n1) 46 | manager.Add(&n2) 47 | 48 | err := manager.NotifyPodTermination(v1.Pod{}) 49 | suite.Require().NoError(err) 50 | 51 | suite.Equal(1, n1.Calls) 52 | suite.Equal(1, n2.Calls) 53 | } 54 | 55 | func (suite *NotifierSuite) TestMultiNotifierWithNotifierError() { 56 | manager := New() 57 | f := FailingNotifier{} 58 | manager.Add(&f) 59 | err := manager.NotifyPodTermination(v1.Pod{}) 60 | suite.Require().Error(err) 61 | } 62 | 63 | func (suite *NotifierSuite) TestMultiNotifierWithNotifierMultipleError() { 64 | manager := New() 65 | f0 := FailingNotifier{} 66 | f1 := FailingNotifier{} 67 | manager.Add(&f0) 68 | manager.Add(&f1) 69 | err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error) 70 | suite.Require().Error(err) 71 | suite.Require().Len(err.Errors, 2) 72 | } 73 | 74 | func (suite *NotifierSuite) TestMultiNotifierWithOneFailingNotifier() { 75 | manager := New() 76 | f := FailingNotifier{} 77 | n := Noop{} 78 | manager.Add(&n) 79 | manager.Add(&f) 80 | err := manager.NotifyPodTermination(v1.Pod{}).(*multierror.Error) 81 | suite.Require().Error(err) 82 | suite.Require().Len(err.Errors, 1) 83 | } 84 | 85 | func TestNotifierSuite(t *testing.T) { 86 | suite.Run(t, new(NotifierSuite)) 87 | } 88 | -------------------------------------------------------------------------------- /notifier/slack.go: -------------------------------------------------------------------------------- 1 | package notifier 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | v1 "k8s.io/api/core/v1" 11 | ) 12 | 13 | const NotifierSlack = "slack" 14 | 15 | var NotificationColor = "#F35A00" 16 | var DefaultTimeout = 10 * time.Second 17 | 18 | type Slack struct { 19 | Webhook string 20 | Client *http.Client 21 | } 22 | 23 | type slackMessage struct { 24 | Message string `json:"text"` 25 | Attachments []attachment `json:"attachments"` 26 | } 27 | 28 | type slackField struct { 29 | Title string `yaml:"title,omitempty" json:"title,omitempty"` 30 | Value string `yaml:"value,omitempty" json:"value,omitempty"` 31 | Short *bool `yaml:"short,omitempty" json:"short,omitempty"` 32 | } 33 | 34 | type attachment struct { 35 | Title string `json:"title,omitempty"` 36 | TitleLink string `json:"title_link,omitempty"` 37 | Pretext string `json:"pretext,omitempty"` 38 | Text string `json:"text"` 39 | Fallback string `json:"fallback"` 40 | CallbackID string `json:"callback_id"` 41 | Fields []slackField `json:"fields,omitempty"` 42 | ImageURL string `json:"image_url,omitempty"` 43 | ThumbURL string `json:"thumb_url,omitempty"` 44 | Footer string `json:"footer"` 45 | Color string `json:"color,omitempty"` 46 | MrkdwnIn []string `json:"mrkdwn_in,omitempty"` 47 | } 48 | 49 | func NewSlackNotifier(webhook string) *Slack { 50 | return &Slack{ 51 | Webhook: webhook, 52 | Client: &http.Client{Timeout: DefaultTimeout}, 53 | } 54 | } 55 | 56 | func (s Slack) NotifyPodTermination(pod v1.Pod) error { 57 | title := "Chaos event - Pod termination" 58 | text := fmt.Sprintf("pod %s has been selected by chaos-kube for termination", pod.Name) 59 | 60 | short := len(pod.Namespace) < 20 && len(pod.Name) < 20 61 | fields := []slackField{ 62 | { 63 | Title: "namespace", 64 | Value: pod.Namespace, 65 | Short: &short, 66 | }, 67 | { 68 | Title: "pod", 69 | Value: pod.Name, 70 | Short: &short, 71 | }, 72 | } 73 | 74 | message := createSlackRequest(title, text, fields) 75 | return s.sendSlackMessage(message) 76 | } 77 | 78 | func createSlackRequest(title string, text string, fields []slackField) slackMessage { 79 | return slackMessage{ 80 | Attachments: []attachment{{ 81 | Title: title, 82 | Text: text, 83 | Footer: "chaos-kube", 84 | Color: NotificationColor, 85 | Fields: fields, 86 | }}, 87 | } 88 | } 89 | 90 | func (s Slack) sendSlackMessage(message slackMessage) error { 91 | messageBody, err := json.Marshal(message) 92 | if err != nil { 93 | return err 94 | } 95 | req, err := http.NewRequest(http.MethodPost, s.Webhook, bytes.NewBuffer(messageBody)) 96 | if err != nil { 97 | return err 98 | } 99 | req.Header.Add("Content-Type", "application/json") 100 | res, err := s.Client.Do(req) 101 | if err != nil { 102 | return err 103 | } 104 | if res.StatusCode != http.StatusOK { 105 | return fmt.Errorf("unexpected status code %d from slack webhook %s", res.StatusCode, s.Webhook) 106 | } 107 | 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /notifier/slack_test.go: -------------------------------------------------------------------------------- 1 | package notifier 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | v1 "k8s.io/api/core/v1" 9 | 10 | "github.com/linki/chaoskube/internal/testutil" 11 | "github.com/linki/chaoskube/util" 12 | 13 | "github.com/stretchr/testify/suite" 14 | ) 15 | 16 | type SlackSuite struct { 17 | testutil.TestSuite 18 | } 19 | 20 | func (suite *SlackSuite) TestSlackNotificationForTerminationStatusOk() { 21 | webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" 22 | 23 | testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { 24 | suite.Require().Equal(webhookPath, req.URL.Path) 25 | res.WriteHeader(200) 26 | _, err := res.Write([]byte("ok")) 27 | suite.Require().NoError(err) 28 | })) 29 | defer testServer.Close() 30 | 31 | testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) 32 | 33 | slack := NewSlackNotifier(testServer.URL + webhookPath) 34 | err := slack.NotifyPodTermination(testPod) 35 | 36 | suite.NoError(err) 37 | } 38 | 39 | func (suite *SlackSuite) TestSlackNotificationForTerminationStatus500() { 40 | webhookPath := "/services/T07M5HUDA/BQ1U5VDGA/yhpIczRK0cZ3jDLK1U8qD634" 41 | 42 | testServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { 43 | suite.Require().Equal(webhookPath, req.URL.Path) 44 | res.WriteHeader(500) 45 | _, err := res.Write([]byte("ok")) 46 | suite.Require().NoError(err) 47 | })) 48 | defer testServer.Close() 49 | 50 | testPod := util.NewPod("chaos", "chaos-57df4db6b-h9ktj", v1.PodRunning) 51 | 52 | slack := NewSlackNotifier(testServer.URL + webhookPath) 53 | err := slack.NotifyPodTermination(testPod) 54 | 55 | suite.Error(err) 56 | } 57 | 58 | func TestSlackSuite(t *testing.T) { 59 | suite.Run(t, new(SlackSuite)) 60 | } 61 | -------------------------------------------------------------------------------- /terminator/delete_pod.go: -------------------------------------------------------------------------------- 1 | package terminator 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | log "github.com/sirupsen/logrus" 8 | 9 | v1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/kubernetes" 12 | ) 13 | 14 | // DeletePodTerminator simply asks k8s to delete the victim pod. 15 | type DeletePodTerminator struct { 16 | client kubernetes.Interface 17 | logger log.FieldLogger 18 | gracePeriod time.Duration 19 | } 20 | 21 | // NewDeletePodTerminator creates and returns a DeletePodTerminator object. 22 | func NewDeletePodTerminator(client kubernetes.Interface, logger log.FieldLogger, gracePeriod time.Duration) *DeletePodTerminator { 23 | return &DeletePodTerminator{ 24 | client: client, 25 | logger: logger.WithField("terminator", "DeletePod"), 26 | gracePeriod: gracePeriod, 27 | } 28 | } 29 | 30 | // Terminate sends a request to Kubernetes to delete the pod. 31 | func (t *DeletePodTerminator) Terminate(ctx context.Context, victim v1.Pod) error { 32 | t.logger.WithFields(log.Fields{ 33 | "namespace": victim.Namespace, 34 | "name": victim.Name, 35 | }).Debug("calling deletePod endpoint") 36 | 37 | return t.client.CoreV1().Pods(victim.Namespace).Delete(ctx, victim.Name, deleteOptions(t.gracePeriod)) 38 | } 39 | 40 | func deleteOptions(gracePeriod time.Duration) metav1.DeleteOptions { 41 | if gracePeriod < 0 { 42 | return metav1.DeleteOptions{} 43 | } 44 | 45 | inSeconds := int64(gracePeriod.Seconds()) 46 | return metav1.DeleteOptions{GracePeriodSeconds: &inSeconds} 47 | } 48 | -------------------------------------------------------------------------------- /terminator/delete_pod_test.go: -------------------------------------------------------------------------------- 1 | package terminator 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | log "github.com/sirupsen/logrus" 9 | "github.com/sirupsen/logrus/hooks/test" 10 | 11 | v1 "k8s.io/api/core/v1" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | "k8s.io/client-go/kubernetes/fake" 14 | 15 | "github.com/linki/chaoskube/internal/testutil" 16 | "github.com/linki/chaoskube/util" 17 | 18 | "github.com/stretchr/testify/suite" 19 | ) 20 | 21 | type DeletePodTerminatorSuite struct { 22 | testutil.TestSuite 23 | } 24 | 25 | var ( 26 | logger, logOutput = test.NewNullLogger() 27 | ) 28 | 29 | func (suite *DeletePodTerminatorSuite) SetupTest() { 30 | logger.SetLevel(log.DebugLevel) 31 | logOutput.Reset() 32 | } 33 | 34 | func (suite *DeletePodTerminatorSuite) TestInterface() { 35 | suite.Implements((*Terminator)(nil), new(DeletePodTerminator)) 36 | } 37 | 38 | func (suite *DeletePodTerminatorSuite) TestTerminate() { 39 | logOutput.Reset() 40 | client := fake.NewSimpleClientset() 41 | terminator := NewDeletePodTerminator(client, logger, 10*time.Second) 42 | 43 | pods := []v1.Pod{ 44 | util.NewPod("default", "foo", v1.PodRunning), 45 | util.NewPod("testing", "bar", v1.PodRunning), 46 | } 47 | 48 | for _, pod := range pods { 49 | _, err := client.CoreV1().Pods(pod.Namespace).Create(context.Background(), &pod, metav1.CreateOptions{}) 50 | suite.Require().NoError(err) 51 | } 52 | 53 | victim := util.NewPod("default", "foo", v1.PodRunning) 54 | 55 | err := terminator.Terminate(context.Background(), victim) 56 | suite.Require().NoError(err) 57 | 58 | suite.AssertLog(logOutput, log.DebugLevel, "calling deletePod endpoint", log.Fields{"namespace": "default", "name": "foo"}) 59 | 60 | remainingPods, err := client.CoreV1().Pods(v1.NamespaceAll).List(context.Background(), metav1.ListOptions{}) 61 | suite.Require().NoError(err) 62 | 63 | suite.AssertPods(remainingPods.Items, []map[string]string{ 64 | {"namespace": "testing", "name": "bar"}, 65 | }) 66 | } 67 | 68 | func (suite *DeletePodTerminatorSuite) TestDeleteOptions() { 69 | for _, tt := range []struct { 70 | gracePeriod time.Duration 71 | expected metav1.DeleteOptions 72 | }{ 73 | { 74 | -1 * time.Second, 75 | metav1.DeleteOptions{}, 76 | }, 77 | { 78 | 0 * time.Second, 79 | metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(0)}, 80 | }, 81 | { 82 | 300 * time.Second, 83 | metav1.DeleteOptions{GracePeriodSeconds: int64Ptr(300)}, 84 | }, 85 | } { 86 | suite.Equal(tt.expected, deleteOptions(tt.gracePeriod)) 87 | } 88 | } 89 | 90 | func TestDeletePodTerminatorSuite(t *testing.T) { 91 | suite.Run(t, new(DeletePodTerminatorSuite)) 92 | } 93 | 94 | func int64Ptr(value int64) *int64 { 95 | return &value 96 | } 97 | -------------------------------------------------------------------------------- /terminator/terminator.go: -------------------------------------------------------------------------------- 1 | package terminator 2 | 3 | import ( 4 | "context" 5 | 6 | v1 "k8s.io/api/core/v1" 7 | ) 8 | 9 | // Terminator is the interface for implementations of pod terminators. 10 | type Terminator interface { 11 | // Terminate terminates the given pod. 12 | Terminate(ctx context.Context, victim v1.Pod) error 13 | } 14 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strings" 7 | "time" 8 | 9 | v1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/apimachinery/pkg/types" 12 | ) 13 | 14 | const ( 15 | // a short time format; like time.Kitchen but with 24-hour notation. 16 | Kitchen24 = "15:04" 17 | // a time format that just cares about the day and month. 18 | YearDay = "Jan_2" 19 | ) 20 | 21 | // TimePeriod represents a time period with a single beginning and end. 22 | type TimePeriod struct { 23 | From time.Time 24 | To time.Time 25 | } 26 | 27 | // NewTimePeriod returns a normalized TimePeriod given a start and end time. 28 | func NewTimePeriod(from, to time.Time) TimePeriod { 29 | return TimePeriod{From: TimeOfDay(from), To: TimeOfDay(to)} 30 | } 31 | 32 | // Includes returns true iff the given pointInTime's time of day is included in time period tp. 33 | func (tp TimePeriod) Includes(pointInTime time.Time) bool { 34 | isAfter := TimeOfDay(pointInTime).After(tp.From) 35 | isBefore := TimeOfDay(pointInTime).Before(tp.To) 36 | 37 | if tp.From.Before(tp.To) { 38 | return isAfter && isBefore 39 | } 40 | if tp.From.After(tp.To) { 41 | return isAfter || isBefore 42 | } 43 | return TimeOfDay(pointInTime).Equal(tp.From) 44 | } 45 | 46 | // String returns tp as a pretty string. 47 | func (tp TimePeriod) String() string { 48 | return fmt.Sprintf("%s-%s", tp.From.Format(Kitchen24), tp.To.Format(Kitchen24)) 49 | } 50 | 51 | // ParseWeekdays takes a comma-separated list of abbreviated weekdays (e.g. sat,sun) and turns them 52 | // into a slice of time.Weekday. It ignores any whitespace and any invalid weekdays. 53 | func ParseWeekdays(weekdays string) []time.Weekday { 54 | var days = map[string]time.Weekday{ 55 | "sun": time.Sunday, 56 | "mon": time.Monday, 57 | "tue": time.Tuesday, 58 | "wed": time.Wednesday, 59 | "thu": time.Thursday, 60 | "fri": time.Friday, 61 | "sat": time.Saturday, 62 | } 63 | 64 | parsedWeekdays := []time.Weekday{} 65 | for _, wd := range strings.Split(weekdays, ",") { 66 | if day, ok := days[strings.TrimSpace(strings.ToLower(wd))]; ok { 67 | parsedWeekdays = append(parsedWeekdays, day) 68 | } 69 | } 70 | return parsedWeekdays 71 | } 72 | 73 | // ParseTimePeriods takes a comma-separated list of time periods in Kitchen24 format and turns them 74 | // into a slice of TimePeriods. It ignores any whitespace. 75 | func ParseTimePeriods(timePeriods string) ([]TimePeriod, error) { 76 | parsedTimePeriods := []TimePeriod{} 77 | 78 | for _, tp := range strings.Split(timePeriods, ",") { 79 | if strings.TrimSpace(tp) == "" { 80 | continue 81 | } 82 | 83 | parts := strings.Split(tp, "-") 84 | if len(parts) != 2 { 85 | return nil, fmt.Errorf("Invalid time range '%v': must contain exactly one '-'", tp) 86 | } 87 | 88 | begin, err := time.Parse(Kitchen24, strings.TrimSpace(parts[0])) 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | end, err := time.Parse(Kitchen24, strings.TrimSpace(parts[1])) 94 | if err != nil { 95 | return nil, err 96 | } 97 | 98 | parsedTimePeriods = append(parsedTimePeriods, NewTimePeriod(begin, end)) 99 | } 100 | 101 | return parsedTimePeriods, nil 102 | } 103 | 104 | func ParseDays(days string) ([]time.Time, error) { 105 | parsedDays := []time.Time{} 106 | 107 | for _, day := range strings.Split(days, ",") { 108 | if strings.TrimSpace(day) == "" { 109 | continue 110 | } 111 | 112 | parsedDay, err := time.Parse(YearDay, strings.TrimSpace(day)) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | parsedDays = append(parsedDays, parsedDay) 118 | } 119 | 120 | return parsedDays, nil 121 | } 122 | 123 | // TimeOfDay normalizes the given point in time by returning a time object that represents the same 124 | // time of day of the given time but on the very first day (day 0). 125 | func TimeOfDay(pointInTime time.Time) time.Time { 126 | return time.Date(0, 0, 0, pointInTime.Hour(), pointInTime.Minute(), pointInTime.Second(), pointInTime.Nanosecond(), time.UTC) 127 | } 128 | 129 | // FormatDays takes a slice of times and returns a slice of strings in YearDate format (e.g. [Apr 1, Sep 24]) 130 | func FormatDays(days []time.Time) []string { 131 | formattedDays := make([]string, 0, len(days)) 132 | for _, d := range days { 133 | formattedDays = append(formattedDays, d.Format(YearDay)) 134 | } 135 | return formattedDays 136 | } 137 | 138 | // NewPod returns a new pod instance for testing purposes. 139 | func NewPod(namespace, name string, phase v1.PodPhase) v1.Pod { 140 | return NewPodWithOwner(namespace, name, phase, "") 141 | } 142 | 143 | // NewPodWithOwner returns a new pod instance for testing purposes with a given owner UID 144 | func NewPodWithOwner(namespace, name string, phase v1.PodPhase, owner types.UID) v1.Pod { 145 | pod := v1.Pod{ 146 | TypeMeta: metav1.TypeMeta{ 147 | APIVersion: "v1", 148 | Kind: "Pod", 149 | }, 150 | ObjectMeta: metav1.ObjectMeta{ 151 | Namespace: namespace, 152 | Name: name, 153 | Labels: map[string]string{ 154 | "app": name, 155 | }, 156 | Annotations: map[string]string{ 157 | "chaos": name, 158 | }, 159 | SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, name), 160 | }, 161 | Status: v1.PodStatus{ 162 | Phase: phase, 163 | }, 164 | } 165 | 166 | if owner != "" { 167 | pod.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ 168 | {UID: owner, Kind: "testkind"}, 169 | } 170 | } 171 | 172 | return pod 173 | } 174 | 175 | // NewNamespace returns a new namespace instance for testing purposes. 176 | func NewNamespace(name string) v1.Namespace { 177 | return v1.Namespace{ 178 | ObjectMeta: metav1.ObjectMeta{ 179 | Name: name, 180 | Labels: map[string]string{ 181 | "env": name, 182 | }, 183 | }, 184 | } 185 | } 186 | 187 | // RandomPodSubSlice creates a shuffled subslice of the give pods slice 188 | func RandomPodSubSlice(pods []v1.Pod, count int) []v1.Pod { 189 | maxCount := len(pods) 190 | if count > maxCount { 191 | count = maxCount 192 | } 193 | 194 | rand.Shuffle(len(pods), func(i, j int) { pods[i], pods[j] = pods[j], pods[i] }) 195 | res := pods[0:count] 196 | return res 197 | } 198 | -------------------------------------------------------------------------------- /util/util_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/suite" 8 | v1 "k8s.io/api/core/v1" 9 | ) 10 | 11 | type Suite struct { 12 | suite.Suite 13 | } 14 | 15 | func (suite *Suite) TestNewTimePeriod() { 16 | timezone, err := time.LoadLocation("Australia/Brisbane") 17 | suite.Require().NoError(err) 18 | 19 | for _, tt := range []struct { 20 | from time.Time 21 | to time.Time 22 | expected TimePeriod 23 | }{ 24 | // when it's already normalized 25 | { 26 | time.Date(0, 0, 0, 15, 04, 05, 06, time.UTC), 27 | time.Date(0, 0, 0, 16, 04, 05, 06, time.UTC), 28 | TimePeriod{ 29 | From: time.Date(0, 0, 0, 15, 04, 05, 06, time.UTC), 30 | To: time.Date(0, 0, 0, 16, 04, 05, 06, time.UTC), 31 | }, 32 | }, 33 | // it normalizes to very first day 34 | { 35 | time.Date(1869, 9, 24, 15, 04, 05, 06, time.UTC), 36 | time.Date(1869, 9, 24, 16, 04, 05, 06, time.UTC), 37 | TimePeriod{ 38 | From: time.Date(0, 0, 0, 15, 04, 05, 06, time.UTC), 39 | To: time.Date(0, 0, 0, 16, 04, 05, 06, time.UTC), 40 | }, 41 | }, 42 | // it ignores the timezone 43 | { 44 | time.Date(1869, 9, 24, 15, 04, 05, 06, timezone), 45 | time.Date(1869, 9, 24, 16, 04, 05, 06, timezone), 46 | TimePeriod{ 47 | From: time.Date(0, 0, 0, 15, 04, 05, 06, time.UTC), 48 | To: time.Date(0, 0, 0, 16, 04, 05, 06, time.UTC), 49 | }, 50 | }, 51 | } { 52 | suite.Equal(tt.expected, NewTimePeriod(tt.from, tt.to)) 53 | } 54 | } 55 | 56 | func (suite *Suite) TestTimePeriodIncludes() { 57 | atTheMoment := NewTimePeriod( 58 | time.Now().Add(-1*time.Minute), 59 | time.Now().Add(+1*time.Minute), 60 | ) 61 | midnight := NewTimePeriod( 62 | time.Date(1869, 9, 23, 23, 00, 00, 00, time.UTC), 63 | time.Date(1869, 9, 24, 01, 00, 00, 00, time.UTC), 64 | ) 65 | now := time.Now() 66 | 67 | for _, tt := range []struct { 68 | pointInTime time.Time 69 | timeOfDay TimePeriod 70 | expected bool 71 | }{ 72 | // it's included 73 | { 74 | now, 75 | atTheMoment, 76 | true, 77 | }, 78 | // it's one day before 79 | { 80 | now.Add(-24 * time.Hour), 81 | atTheMoment, 82 | true, 83 | }, 84 | // it's one day after 85 | { 86 | now.Add(+24 * time.Hour), 87 | atTheMoment, 88 | true, 89 | }, 90 | // it's just before 91 | { 92 | now.Add(-2 * time.Minute), 93 | atTheMoment, 94 | false, 95 | }, 96 | // it's just after 97 | { 98 | now.Add(+2 * time.Minute), 99 | atTheMoment, 100 | false, 101 | }, 102 | // it's slightly inside before day switch 103 | { 104 | time.Date(1869, 9, 23, 23, 30, 00, 00, time.UTC), 105 | midnight, 106 | true, 107 | }, 108 | // it's slightly inside after day switch 109 | { 110 | time.Date(1869, 9, 24, 00, 30, 00, 00, time.UTC), 111 | midnight, 112 | true, 113 | }, 114 | // it's just before 115 | { 116 | time.Date(1869, 9, 23, 22, 30, 00, 00, time.UTC), 117 | midnight, 118 | false, 119 | }, 120 | // it's just after 121 | { 122 | time.Date(1869, 9, 24, 01, 30, 00, 00, time.UTC), 123 | midnight, 124 | false, 125 | }, 126 | // it's exactly matching a point in time 127 | { 128 | now, 129 | TimePeriod{From: TimeOfDay(now), To: TimeOfDay(now)}, 130 | true, 131 | }, 132 | // it's right after exactly matching a point in time 133 | { 134 | now.Add(+1 * time.Second), 135 | TimePeriod{From: TimeOfDay(now), To: TimeOfDay(now)}, 136 | false, 137 | }, 138 | // it's right before exactly matching a point in time 139 | { 140 | now.Add(-1 * time.Second), 141 | TimePeriod{From: TimeOfDay(now), To: TimeOfDay(now)}, 142 | false, 143 | }, 144 | } { 145 | suite.Equal(tt.expected, tt.timeOfDay.Includes(tt.pointInTime)) 146 | } 147 | } 148 | 149 | func (suite *Suite) TestTimePeriodString() { 150 | for _, tt := range []struct { 151 | given TimePeriod 152 | expected string 153 | }{ 154 | // empty time period 155 | { 156 | TimePeriod{}, 157 | "00:00-00:00", 158 | }, 159 | // simple time period 160 | { 161 | TimePeriod{ 162 | From: time.Date(0, 0, 0, 8, 0, 0, 0, time.UTC), 163 | To: time.Date(0, 0, 0, 16, 0, 0, 0, time.UTC), 164 | }, 165 | "08:00-16:00", 166 | }, 167 | // time period across days 168 | { 169 | TimePeriod{ 170 | From: time.Date(0, 0, 0, 16, 0, 0, 0, time.UTC), 171 | To: time.Date(0, 0, 0, 8, 0, 0, 0, time.UTC), 172 | }, 173 | "16:00-08:00", 174 | }, 175 | } { 176 | suite.Equal(tt.expected, tt.given.String()) 177 | } 178 | } 179 | 180 | func (suite *Suite) TestTimeOfDay() { 181 | timezone, err := time.LoadLocation("Australia/Brisbane") 182 | suite.Require().NoError(err) 183 | 184 | for _, tt := range []struct { 185 | pointInTime time.Time 186 | expected time.Time 187 | }{ 188 | // strips of any day information 189 | { 190 | time.Date(1869, 9, 24, 15, 04, 05, 06, time.UTC), 191 | time.Date(0, 0, 0, 15, 04, 05, 06, time.UTC), 192 | }, 193 | // it normalizes to UTC timezone 194 | { 195 | time.Date(1869, 9, 24, 15, 04, 05, 06, timezone), 196 | time.Date(0, 0, 0, 15, 04, 05, 06, time.UTC), 197 | }, 198 | } { 199 | suite.Equal(tt.expected, TimeOfDay(tt.pointInTime)) 200 | } 201 | } 202 | 203 | func (suite *Suite) TestParseWeekdays() { 204 | for _, tt := range []struct { 205 | given string 206 | expected []time.Weekday 207 | }{ 208 | // empty string 209 | { 210 | "", 211 | []time.Weekday{}, 212 | }, 213 | // single weekday 214 | { 215 | "sat", 216 | []time.Weekday{time.Saturday}, 217 | }, 218 | // multiple weekdays 219 | { 220 | "sat,sun", 221 | []time.Weekday{time.Saturday, time.Sunday}, 222 | }, 223 | // case-insensitive 224 | { 225 | "SaT,SUn", 226 | []time.Weekday{time.Saturday, time.Sunday}, 227 | }, 228 | // ignore whitespace 229 | { 230 | " sat , sun ", 231 | []time.Weekday{time.Saturday, time.Sunday}, 232 | }, 233 | // ignore unknown weekdays 234 | { 235 | "sat,unknown,sun", 236 | []time.Weekday{time.Saturday, time.Sunday}, 237 | }, 238 | // deal with all kinds at the same time 239 | { 240 | "Fri, sat ,,,, ,foobar,tue", 241 | []time.Weekday{time.Friday, time.Saturday, time.Tuesday}, 242 | }, 243 | } { 244 | suite.Equal(tt.expected, ParseWeekdays(tt.given)) 245 | } 246 | } 247 | 248 | func (suite *Suite) TestParseTimePeriods() { 249 | for _, tt := range []struct { 250 | given string 251 | expected []TimePeriod 252 | }{ 253 | // empty time period string 254 | { 255 | "", 256 | []TimePeriod{}, 257 | }, 258 | // single range string 259 | { 260 | "08:00-16:00", 261 | []TimePeriod{ 262 | { 263 | From: time.Date(0, 0, 0, 8, 0, 0, 0, time.UTC), 264 | To: time.Date(0, 0, 0, 16, 0, 0, 0, time.UTC), 265 | }, 266 | }, 267 | }, 268 | // multiple ranges string 269 | { 270 | "08:00-16:00,20:00-22:00", 271 | []TimePeriod{ 272 | { 273 | From: time.Date(0, 0, 0, 8, 0, 0, 0, time.UTC), 274 | To: time.Date(0, 0, 0, 16, 0, 0, 0, time.UTC), 275 | }, 276 | { 277 | From: time.Date(0, 0, 0, 20, 0, 0, 0, time.UTC), 278 | To: time.Date(0, 0, 0, 22, 0, 0, 0, time.UTC), 279 | }, 280 | }, 281 | }, 282 | // string containing whitespace 283 | { 284 | " 08:00 - 16:00 ,, , 20:00 - 22:00 ", 285 | []TimePeriod{ 286 | { 287 | From: time.Date(0, 0, 0, 8, 0, 0, 0, time.UTC), 288 | To: time.Date(0, 0, 0, 16, 0, 0, 0, time.UTC), 289 | }, 290 | { 291 | From: time.Date(0, 0, 0, 20, 0, 0, 0, time.UTC), 292 | To: time.Date(0, 0, 0, 22, 0, 0, 0, time.UTC), 293 | }, 294 | }, 295 | }, 296 | } { 297 | timePeriods, err := ParseTimePeriods(tt.given) 298 | suite.Require().NoError(err) 299 | 300 | suite.Equal(tt.expected, timePeriods) 301 | } 302 | } 303 | 304 | func (suite *Suite) TestParseDates() { 305 | for _, tt := range []struct { 306 | given string 307 | expected []time.Time 308 | }{ 309 | // empty string 310 | { 311 | "", 312 | []time.Time{}, 313 | }, 314 | // single date 315 | { 316 | "Apr 1", 317 | []time.Time{ 318 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 319 | }, 320 | }, 321 | // single date leaving out the space 322 | { 323 | "Apr1", 324 | []time.Time{ 325 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 326 | }, 327 | }, 328 | // multiple dates 329 | { 330 | "Apr 1,Dec 24", 331 | []time.Time{ 332 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 333 | time.Date(0, 12, 24, 0, 0, 0, 0, time.UTC), 334 | }, 335 | }, 336 | // case-insensitive 337 | { 338 | "apr 1,dEc 24", 339 | []time.Time{ 340 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 341 | time.Date(0, 12, 24, 0, 0, 0, 0, time.UTC), 342 | }, 343 | }, 344 | // ignore whitespace 345 | { 346 | " apr 1 , dec 24 ", 347 | []time.Time{ 348 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 349 | time.Date(0, 12, 24, 0, 0, 0, 0, time.UTC), 350 | }, 351 | }, 352 | // deal with all kinds at the same time 353 | { 354 | ",Apr 1, dEc 24 ,,,, ,jun08,,", 355 | []time.Time{ 356 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 357 | time.Date(0, 12, 24, 0, 0, 0, 0, time.UTC), 358 | time.Date(0, 6, 8, 0, 0, 0, 0, time.UTC), 359 | }, 360 | }, 361 | } { 362 | days, err := ParseDays(tt.given) 363 | suite.Require().NoError(err) 364 | 365 | suite.Equal(tt.expected, days) 366 | } 367 | } 368 | 369 | func (suite *Suite) TestFormatDays() { 370 | for _, tt := range []struct { 371 | given []time.Time 372 | expected []string 373 | }{ 374 | { 375 | []time.Time{ 376 | time.Date(1869, 9, 24, 15, 04, 05, 06, time.UTC), 377 | }, 378 | []string{"Sep24"}, 379 | }, { 380 | []time.Time{ 381 | time.Date(1869, 9, 24, 15, 04, 05, 06, time.UTC), 382 | time.Date(0, 4, 1, 0, 0, 0, 0, time.UTC), 383 | }, 384 | []string{"Sep24", "Apr 1"}, 385 | }, 386 | } { 387 | suite.Equal(tt.expected, FormatDays(tt.given)) 388 | } 389 | } 390 | 391 | func (suite *Suite) TestNewPod() { 392 | pod := NewPod("namespace", "name", "phase") 393 | 394 | suite.Equal("v1", pod.APIVersion) 395 | suite.Equal("Pod", pod.Kind) 396 | suite.Equal("namespace", pod.Namespace) 397 | suite.Equal("name", pod.Name) 398 | suite.Equal("name", pod.Labels["app"]) 399 | suite.Equal("name", pod.Annotations["chaos"]) 400 | suite.Equal("/api/v1/namespaces/namespace/pods/name", pod.SelfLink) 401 | suite.EqualValues("phase", pod.Status.Phase) 402 | } 403 | 404 | func (suite *Suite) TestNewNamespace() { 405 | namespace := NewNamespace("name") 406 | 407 | suite.Equal("name", namespace.Name) 408 | suite.Equal("name", namespace.Labels["env"]) 409 | } 410 | 411 | func (suite *Suite) TestRandomPodSublice() { 412 | pods := []v1.Pod{ 413 | NewPod("default", "foo", v1.PodRunning), 414 | NewPod("testing", "bar", v1.PodRunning), 415 | NewPod("test", "baz", v1.PodRunning), 416 | } 417 | 418 | for _, tt := range []struct { 419 | name string 420 | in []v1.Pod 421 | count int 422 | expected int 423 | }{ 424 | {"max kill = len(pods)", pods, 3, 3}, 425 | {"empyt pod list should return empty subslice", []v1.Pod{}, 3, 0}, 426 | {"maxKill > len(pods)", pods[0:1], 3, 1}, 427 | {"maxKill = 0 ", pods, 0, 0}, 428 | } { 429 | results := RandomPodSubSlice(tt.in, tt.count) 430 | suite.Assert().Equal(len(results), tt.expected, tt.name) 431 | } 432 | } 433 | 434 | func TestSuite(t *testing.T) { 435 | suite.Run(t, new(Suite)) 436 | } 437 | --------------------------------------------------------------------------------