├── .dockerignore ├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── asciinema_demo.json ├── cmd ├── monitor.go ├── monitor_elasticsearch.go ├── monitor_prometheus.go ├── monitor_sentry.go └── monitor_sentry_test.go ├── examples ├── Makefile ├── README.md ├── app │ ├── Dockerfile │ ├── charts │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── templates │ │ │ ├── NOTES.txt │ │ │ ├── _helpers.tpl │ │ │ ├── deployment.yaml │ │ │ ├── ingress.yaml │ │ │ └── service.yaml │ │ └── values.yaml │ ├── go.mod │ ├── go.sum │ └── main.go ├── elasticsearch-query.json ├── elasticsearch.md ├── prometheus.md └── sentry.md ├── go.mod ├── go.sum ├── helm-monitor-diagram.jpg ├── helm-monitor-failure.png ├── install.sh └── plugin.yaml /.dockerignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .git 3 | .gitignore 4 | .travis.yml 5 | CHANGELOG.md 6 | LICENSE 7 | Makefile 8 | README.md 9 | asciinema_demo.json 10 | examples 11 | helm-monitor-diagram.jpg 12 | helm-monitor-failure.png 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | vendor 3 | examples/app/vendor 4 | _dist 5 | helm-monitor 6 | helm-monitor.exe 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - master 5 | 6 | script: 7 | - make test-all 8 | - make dist 9 | 10 | deploy: 11 | - provider: releases 12 | api_key: ${GITHUB_TOKEN} 13 | file: _dist/helm-monitor* 14 | skip_cleanup: true 15 | file_glob: true 16 | on: 17 | tags: true 18 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | ## v0.4.0 5 | 6 | [PR#9](https://github.com/ContainerSolutions/helm-monitor/pull/9): 7 | 8 | - upgrade dependencies, make it compatible with Helm 2.13. 9 | 10 | ## v0.3.0 11 | 12 | [PR#6](https://github.com/ContainerSolutions/helm-monitor/pull/6): 13 | 14 | - add Sentry support 15 | - adjust documentation and command line helpers 16 | 17 | ## v0.2.0 18 | 19 | [PR#5](https://github.com/ContainerSolutions/helm-monitor/pull/5): 20 | 21 | - switch from dep to go modules 22 | - upgrade dependencies 23 | - simplify a bit the example and its instructions 24 | - add containersol/helm-monitor Docker container 25 | 26 | ## v0.1.2 27 | 28 | - [PR#4](https://github.com/ContainerSolutions/helm-monitor/pull/4) add 29 | `--expected-result-count` flag by @markomalis 30 | 31 | ## v0.1.1 32 | 33 | - install.sh: add missing `-L` flag to curl command and do not rely on bash 34 | 35 | ## v0.1.0 36 | 37 | - Initial release 38 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 AS build 2 | ARG LDFLAGS 3 | COPY . /go 4 | RUN go build -o helm-monitor -ldflags "$LDFLAGS" ./cmd/... 5 | 6 | FROM alpine AS helm 7 | ENV HELM_VERSION=v2.13.0 8 | ENV HELM_TMP_FILE=helm-${HELM_VERSION}-linux-amd64.tar.gz 9 | RUN wget https://storage.googleapis.com/kubernetes-helm/${HELM_TMP_FILE} && \ 10 | wget https://storage.googleapis.com/kubernetes-helm/${HELM_TMP_FILE}.sha256 11 | RUN apk --no-cache add openssl 12 | RUN if [ "$(openssl sha1 -sha256 ${HELM_TMP_FILE} | awk '{print $2}')" != "$(cat helm-${HELM_VERSION}-linux-amd64.tar.gz.sha256)" ]; \ 13 | then \ 14 | echo "SHA sum of ${HELM_TMP_FILE} does not match. Aborting."; \ 15 | exit 1; \ 16 | fi 17 | RUN tar -xvf helm-${HELM_VERSION}-linux-amd64.tar.gz 18 | 19 | FROM alpine:3.8 20 | COPY --from=helm /linux-amd64/helm /usr/local/bin/helm 21 | RUN helm init --skip-refresh --client-only && \ 22 | mkdir -p /root/.helm/plugins/helm-monitor 23 | COPY plugin.yaml /root/.helm/plugins/helm-monitor/plugin.yaml 24 | COPY --from=build /go/helm-monitor /root/.helm/plugins/helm-monitor/helm-monitor 25 | ENTRYPOINT ["helm"] 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 estafette 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | HELM_HOME ?= $(shell helm home) 2 | HELM_PLUGIN_DIR ?= $(HELM_HOME)/plugins/helm-monitor 3 | VERSION := $(shell sed -n -e 's/version:[ "]*\([^"]*\).*/\1/p' plugin.yaml) 4 | DIST := $(CURDIR)/_dist 5 | LDFLAGS := "-X main.version=${VERSION}" 6 | BINARY := "helm-monitor" 7 | DOCKER_IMAGE ?= containersol/helm-monitor 8 | DOCKER_TAG ?= latest 9 | 10 | # go mod ftw 11 | unexport GOPATH 12 | GO111MODULE = on 13 | 14 | .PHONY: install 15 | install: build 16 | cp $(BINARY) $(HELM_PLUGIN_DIR) 17 | cp plugin.yaml $(HELM_PLUGIN_DIR) 18 | 19 | .PHONY: build 20 | build: 21 | go build -o $(BINARY) -ldflags $(LDFLAGS) ./cmd/... 22 | 23 | .PHONY: build.docker 24 | build.docker: 25 | docker build \ 26 | --build-arg LDFLAGS=$(LDFLAGS) \ 27 | --cache-from ${DOCKER_IMAGE} \ 28 | -t ${DOCKER_IMAGE}:$(DOCKER_TAG) . 29 | 30 | .PHONY: dist 31 | dist: 32 | mkdir -p $(DIST) 33 | GOOS=linux GOARCH=amd64 go build -o $(BINARY) -ldflags $(LDFLAGS) ./cmd/... 34 | tar -zcvf $(DIST)/helm-monitor_linux_$(VERSION).tar.gz $(BINARY) README.md LICENSE plugin.yaml 35 | GOOS=darwin GOARCH=amd64 go build -o $(BINARY) -ldflags $(LDFLAGS) ./cmd/... 36 | tar -zcvf $(DIST)/helm-monitor_darwin_$(VERSION).tar.gz $(BINARY) README.md LICENSE plugin.yaml 37 | GOOS=windows GOARCH=amd64 go build -o $(BINARY).exe -ldflags $(LDFLAGS) ./cmd/... 38 | tar -zcvf $(DIST)/helm-monitor_windows_$(VERSION).tar.gz $(BINARY).exe README.md LICENSE plugin.yaml 39 | 40 | .PHONY: test-all 41 | test-all: vet lint test 42 | 43 | .PHONY: test 44 | test: 45 | go test -v -parallel=4 ./cmd/... 46 | 47 | .PHONY: lint 48 | lint: 49 | @go get -u golang.org/x/lint/golint 50 | go list ./cmd/... | xargs -n1 $${HOME}/go/bin/golint 51 | 52 | .PHONY: vet 53 | vet: 54 | go vet ./cmd/... 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Helm Monitor plugin 2 | =================== 3 | 4 | > Monitor a release, rollback to a previous version depending on the result of 5 | a PromQL (Prometheus), events (Sentry), Lucene or DSL query (Elasticsearch). 6 | 7 | ![Helm monitor failure](helm-monitor-failure.png) 8 | 9 | ## Demo 10 | 11 | [![asciicast](https://asciinema.org/a/231906.svg)](https://asciinema.org/a/231906) 12 | 13 | ## Install 14 | 15 | ```bash 16 | $ helm plugin install https://github.com/ContainerSolutions/helm-monitor 17 | ``` 18 | 19 | ## Usage 20 | 21 | ![Helm monitor diagram](helm-monitor-diagram.jpg) 22 | 23 | A rollback happen only if the number of result from the query is greater than 0. 24 | 25 | You can find a step-by-step example in the `./examples` directory. 26 | 27 | ### Prometheus 28 | 29 | Monitor the **peeking-bunny** release against a Prometheus server, a rollback 30 | is initiated if the 5xx error rate is over 0 as measured over the last 5 31 | minutes. 32 | 33 | ```bash 34 | $ helm monitor prometheus peeking-bunny 'rate(http_requests_total{code=~"^5.*$"}[5m]) > 0' 35 | ``` 36 | 37 | You can connect to a given Prometheus instance, by default it will connect to 38 | *http://localhost:9090*. 39 | 40 | ```bash 41 | $ helm monitor prometheus --prometheus=http://prometheus:9090 \ 42 | peeking-bunny \ 43 | 'rate(http_requests_total{code=~"^5.*$"}[5m]) > 0' 44 | ``` 45 | 46 | ### Elasticsearch 47 | 48 | Monitor the **peeking-bunny** release against an Elasticsearch server, a 49 | rollback is initiated if the 5xx error rate is over 0 for the last minute. 50 | 51 | Using a Lucene query: 52 | 53 | ```bash 54 | $ helm monitor elasticsearch peeking-bunny 'status:500 AND kubernetes.labels.app:app AND version:2.0.0' 55 | ``` 56 | 57 | Using a query DSL file: 58 | 59 | ```bash 60 | $ helm monitor elasticsearch peeking-bunny ./query.json 61 | ``` 62 | 63 | You can connect to a given Elasticsearch instance, by default it will connect to 64 | *http://localhost:9200*. 65 | 66 | ```bash 67 | $ helm monitor elasticsearch --elasticsearch=http://elasticsearch:9200 \ 68 | peeking-bunny \ 69 | 'status:500 AND kubernetes.labels.app:app AND version:2.0.0' 70 | ``` 71 | 72 | ### Sentry 73 | 74 | Monitor the **peeking-bunny** release against a Sentry server, a rollback is 75 | initiated if the number of events is over 0 for the release 2.0.0: 76 | 77 | ```bash 78 | $ helm monitor sentry my-app \ 79 | --api-key \ 80 | --organization sentry \ 81 | --project my-project \ 82 | --sentry http://sentry:9000 \ 83 | --tag release=2.0.0 \ 84 | --regexp 85 | 'Error with database connection.*' 86 | ``` 87 | 88 | 89 | ## Docker 90 | 91 | You can also use the Helm monitor backed Docker image to monitor: 92 | 93 | ```bash 94 | $ docker run -ti -v $HOME/.kube:/root/.kube containersol/helm-monitor \ 95 | monitor prometheus --prometheus=http://prometheus:9090 my-release \ 96 | 'rate(http_requests_total{code=~"^5.*$"}[5m]) > 0' 97 | ``` 98 | 99 | ## Development 100 | 101 | Require Go >= 1.11. 102 | 103 | ```bash 104 | # Clone the repo, then add a symlink to the Helm plugin directory: 105 | $ ln -s $GOPATH/src/github.com/ContainerSolutions/helm-monitor ~/.helm/plugins/helm-monitor 106 | 107 | # Build: 108 | $ GOPATH="" GO111MODULE=on go build -o helm-monitor ./cmd/... 109 | 110 | # Run: 111 | $ helm monitor elasticsearch my-release ./examples/elasticsearch-query.json 112 | ``` 113 | 114 | ## Alternatives 115 | 116 | - [Kuberbs](https://github.com/doitintl/kuberbs) - Kubernetes Automatic Rollback 117 | System 118 | -------------------------------------------------------------------------------- /asciinema_demo.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 1, 3 | "width": 145, 4 | "height": 45, 5 | "duration": 104.928887, 6 | "command": "tmux attach -t helm-monitor", 7 | "title": null, 8 | "env": { 9 | "TERM": "xterm-256color", 10 | "SHELL": "/bin/zsh" 11 | }, 12 | "stdout": [ 13 | [ 14 | 0.015744, 15 | "\u001b[?1049h\u001b[?1h\u001b=\u001b[H\u001b[2J\u001b[?12l\u001b[?25h\u001b[?1000l\u001b[?1002l\u001b[?1006l\u001b[?1005l\u001b[?1004h\u001b[c\u001b(B\u001b[m\u001b[?12;25h\u001b[?12l\u001b[?25h\u001b[?1003l\u001b[?1006l\u001b[?2004l\u001b[1;1H\u001b[1;45r\u001b]112\u0007\u001b[3;3H\u001b[?1006h\u001b[?1002h\u001b[?2004h" 16 | ], 17 | [ 18 | 0.004534, 19 | "\u001b[?25l\u001b[H\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m <'/Users/etienne/.tmux/plugins/tmux-cont\u001b(B\u001b[m\u001b[3;3H\u001b[?12l\u001b[?25h\u001b(B\u001b[m\u001b[?12;25h\u001b[?12l\u001b[?25h\u001b[?1003l\u001b[?1006l\u001b[?2004l\u001b[1;1H\u001b[1;45r" 20 | ], 21 | [ 22 | 0.001941, 23 | "\u001b[?25l\u001b[1;1H\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m <'/Users/etienne/.tmux/plugins/tmux-cont\u001b(B\u001b[m\u001b[3;3H\u001b[?12l\u001b[?25h\u001b[?1006h\u001b[?1002h\u001b[?2004h" 24 | ], 25 | [ 26 | 0.069405, 27 | "\u001b[?25l\u001b[45;1H\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[3;3H\u001b[?12l\u001b[?25h" 28 | ], 29 | [ 30 | 2.0, 31 | "k" 32 | ], 33 | [ 34 | 0.071189, 35 | "\bku" 36 | ], 37 | [ 38 | 0.168328, 39 | "b" 40 | ], 41 | [ 42 | 0.055743, 43 | "e" 44 | ], 45 | [ 46 | 0.055903, 47 | "c" 48 | ], 49 | [ 50 | 0.168563, 51 | "t" 52 | ], 53 | [ 54 | 0.063623, 55 | "l" 56 | ], 57 | [ 58 | 0.151628, 59 | " " 60 | ], 61 | [ 62 | 0.504077, 63 | "g" 64 | ], 65 | [ 66 | 0.088426, 67 | "e" 68 | ], 69 | [ 70 | 0.019108, 71 | "t" 72 | ], 73 | [ 74 | 0.076628, 75 | " " 76 | ], 77 | [ 78 | 0.703625, 79 | "p" 80 | ], 81 | [ 82 | 0.048196, 83 | "o" 84 | ], 85 | [ 86 | 0.520438, 87 | "\r\n\u001b[?2004l" 88 | ], 89 | [ 90 | 0.019308, 91 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:kubectl*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[4;1H\u001b[?12l\u001b[?25h" 92 | ], 93 | [ 94 | 0.228819, 95 | "NAME READY STATUS RESTARTS AGE\r\nmy-release-app-c75998dc7-ggmg7 1/1 Running 0 8m\r\nmy-release-app-c75998dc7-mpv9j 1/1 Running 0 8m\r\nmy-release-app-c75998dc7-vfd6z 1/1 Running 0 8m\r\nprometheus-operator-prometheus-operator-5bbfcb6558-v8vjs 1/1 Running 1 9d\r\nprometheus-prometheus-0 2/2 Running 1 5d" 96 | ], 97 | [ 98 | 3.8e-05, 99 | "\r\n" 100 | ], 101 | [ 102 | 0.003129, 103 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \u001b[10;1H \r" 104 | ], 105 | [ 106 | 0.080732, 107 | "\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m \r\n" 108 | ], 109 | [ 110 | 0.007277, 111 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[12;1H\u001b[?12l\u001b[?25h\u001b[1;44r\u001b[13;44r\u001b[32S\u001b[12;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K" 112 | ], 113 | [ 114 | 3.8e-05, 115 | "\u001b[1;45r\u001b[12;3H\u001b[?2004h" 116 | ], 117 | [ 118 | 1.044262, 119 | "h" 120 | ], 121 | [ 122 | 0.071979, 123 | "\bhe" 124 | ], 125 | [ 126 | 0.144567, 127 | "l" 128 | ], 129 | [ 130 | 0.031494, 131 | "m" 132 | ], 133 | [ 134 | 0.087851, 135 | " " 136 | ], 137 | [ 138 | 0.239985, 139 | "l" 140 | ], 141 | [ 142 | 0.047922, 143 | "i" 144 | ], 145 | [ 146 | 0.031913, 147 | "s" 148 | ], 149 | [ 150 | 0.096693, 151 | "t" 152 | ], 153 | [ 154 | 0.184243, 155 | "\r\n\u001b[?2004l" 156 | ], 157 | [ 158 | 0.017616, 159 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:helm*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[13;1H\u001b[?12l\u001b[?25h" 160 | ], 161 | [ 162 | 0.266823, 163 | "NAME \u001b[5CREVISION\u001b[8CUPDATED \u001b[8CSTATUS \u001b[8CCHART \u001b[7CNAMESPACE\r\nmy-release \u001b[5C63 \u001b[8CFri Jan 19 09:19:32 2018\u001b[8CDEPLOYED\u001b[8Capp-0.1.0 \u001b[7Cdefault \r\nprometheus-operator\u001b[5C1 \u001b[8CTue Jan 9 21:12:31 2018\u001b[8CDEPLOYED\u001b[8Cprometheus-operator-0.0.8\u001b[7Cdefault \r\n" 164 | ], 165 | [ 166 | 0.00553, 167 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \u001b[16;1H \r" 168 | ], 169 | [ 170 | 0.083096, 171 | "\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m \r\n" 172 | ], 173 | [ 174 | 0.002036, 175 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[18;1H\u001b[?12l\u001b[?25h\u001b[1;44r\u001b[19;44r\u001b[26S\u001b[18;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K" 176 | ], 177 | [ 178 | 3.2e-05, 179 | "\u001b[1;45r\u001b[18;3H\u001b[?2004h" 180 | ], 181 | [ 182 | 1.304834, 183 | "h" 184 | ], 185 | [ 186 | 0.127647, 187 | "\bhe" 188 | ], 189 | [ 190 | 0.472011, 191 | "l" 192 | ], 193 | [ 194 | 0.031857, 195 | "m" 196 | ], 197 | [ 198 | 0.359907, 199 | " " 200 | ], 201 | [ 202 | 0.214194, 203 | "u" 204 | ], 205 | [ 206 | 0.163713, 207 | "p" 208 | ], 209 | [ 210 | 0.494707, 211 | "g" 212 | ], 213 | [ 214 | 0.039634, 215 | "r" 216 | ], 217 | [ 218 | 0.023971, 219 | "a" 220 | ], 221 | [ 222 | 0.184234, 223 | "d" 224 | ], 225 | [ 226 | 0.048126, 227 | "e" 228 | ], 229 | [ 230 | 0.559905, 231 | " " 232 | ], 233 | [ 234 | 0.20829, 235 | "-" 236 | ], 237 | [ 238 | 0.351212, 239 | "i" 240 | ], 241 | [ 242 | 0.688222, 243 | " " 244 | ], 245 | [ 246 | 0.256351, 247 | "m" 248 | ], 249 | [ 250 | 0.263589, 251 | "y" 252 | ], 253 | [ 254 | 0.232452, 255 | "-" 256 | ], 257 | [ 258 | 0.175524, 259 | "r" 260 | ], 261 | [ 262 | 0.040473, 263 | "e" 264 | ], 265 | [ 266 | 0.143773, 267 | "l" 268 | ], 269 | [ 270 | 0.111568, 271 | "e" 272 | ], 273 | [ 274 | 0.072712, 275 | "a" 276 | ], 277 | [ 278 | 0.07967, 279 | "s" 280 | ], 281 | [ 282 | 0.104546, 283 | "e" 284 | ], 285 | [ 286 | 0.975188, 287 | " " 288 | ], 289 | [ 290 | 0.56072, 291 | "." 292 | ], 293 | [ 294 | 0.047667, 295 | "/" 296 | ], 297 | [ 298 | 0.81588, 299 | "a" 300 | ], 301 | [ 302 | 0.11206, 303 | "p" 304 | ], 305 | [ 306 | 0.120156, 307 | "p" 308 | ], 309 | [ 310 | 0.223598, 311 | "/" 312 | ], 313 | [ 314 | 0.280658, 315 | "c" 316 | ], 317 | [ 318 | 0.127414, 319 | "h" 320 | ], 321 | [ 322 | 0.096359, 323 | "a" 324 | ], 325 | [ 326 | 0.167849, 327 | "r" 328 | ], 329 | [ 330 | 0.151648, 331 | "t" 332 | ], 333 | [ 334 | 0.073704, 335 | "s" 336 | ], 337 | [ 338 | 0.879279, 339 | " " 340 | ], 341 | [ 342 | 0.47963, 343 | "-" 344 | ], 345 | [ 346 | 0.119504, 347 | "-" 348 | ], 349 | [ 350 | 0.112341, 351 | "s" 352 | ], 353 | [ 354 | 0.192465, 355 | "e" 356 | ], 357 | [ 358 | 0.055138, 359 | "t" 360 | ], 361 | [ 362 | 0.07252, 363 | " " 364 | ], 365 | [ 366 | 0.904403, 367 | "i" 368 | ], 369 | [ 370 | 0.047271, 371 | "m" 372 | ], 373 | [ 374 | 0.08056, 375 | "a" 376 | ], 377 | [ 378 | 0.079324, 379 | "g" 380 | ], 381 | [ 382 | 0.056144, 383 | "e" 384 | ], 385 | [ 386 | 0.215895, 387 | "." 388 | ], 389 | [ 390 | 0.224455, 391 | "t" 392 | ], 393 | [ 394 | 0.199909, 395 | "a" 396 | ], 397 | [ 398 | 0.207736, 399 | "g" 400 | ], 401 | [ 402 | 1.312982, 403 | "=" 404 | ], 405 | [ 406 | 0.198841, 407 | "2" 408 | ], 409 | [ 410 | 0.248619, 411 | "." 412 | ], 413 | [ 414 | 0.279996, 415 | "0" 416 | ], 417 | [ 418 | 0.224047, 419 | "." 420 | ], 421 | [ 422 | 0.200153, 423 | "0" 424 | ], 425 | [ 426 | 0.855502, 427 | "\r\n\u001b[?2004l" 428 | ], 429 | [ 430 | 0.011996, 431 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:helm*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[19;1H\u001b[?12l\u001b[?25h" 432 | ], 433 | [ 434 | 0.559789, 435 | "Release \"my-release\" has been upgraded. Happy Helming!\r\n" 436 | ], 437 | [ 438 | 0.31205, 439 | "LAST DEPLOYED: Fri Jan 19 09:27:45 2018\r\nNAMESPACE: default\r\nSTATUS: DEPLOYED\u001b[24;1HRESOURCES:\r\n==> v1beta1/Deployment\r\nNAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\r\nmy-release-app 3 4 2 2 3d\u001b[29;1H==> v1/Pod(related)\r\nNAME READY STATUS RESTARTS AGE\r\nmy-release-app-6b9ff444dd-987qq 0/1 ContainerCreating 0 0s\r\nmy-release-app-6b9ff444dd-qnhp7 0/1 ContainerCreating 0 0s\r\nmy-release-app-c75998dc7-ggmg7 1/1 Running 0 8m\r\nmy-release-app-c75998dc7-mpv9j " 440 | ], 441 | [ 442 | 0.000406, 443 | "1/1 Running 0 8m\r\nmy-release-app-c75998dc7-vfd6z 1/1 Terminating 0 8m\u001b[37;1H==> v1/Service\r\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\r\n\u001b[1;44r\u001b[5S\u001b[34;1Hmy-release-app LoadBalancer 10.96.10.22 80:30995/TCP 3d\u001b[37;1HNOTES:\r\n1. Get the application URL by running these commands:\r\n NOTE: It may take a few minutes for the LoadBalancer IP to be available.\r\n You can watch the status of by running 'kubectl get svc -w my-release-app'\r\n export SERVICE_IP=$(kubectl get svc --namespace default my-release-app -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\r\n echo http://$SERVICE_IP:80\u001b[1;45r\u001b[44;1H" 444 | ], 445 | [ 446 | 0.006065, 447 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \u001b[44;1H \r" 448 | ], 449 | [ 450 | 0.099756, 451 | "\u001b[1;44r\u001b[2S\u001b[43;1H\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m \u001b[1;45r\u001b[44;1H" 452 | ], 453 | [ 454 | 0.009027, 455 | "\u001b[?25l\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m \"etiennebook\" 09:27 19-Jan-18\u001b(B\u001b[m\u001b[44;1H\u001b[?12l\u001b[?25h\u001b[1;44r\u001b[44;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K" 456 | ], 457 | [ 458 | 9e-05, 459 | "\u001b[1;45r\u001b[44;3H\u001b[?2004h" 460 | ], 461 | [ 462 | 2.0, 463 | "h" 464 | ], 465 | [ 466 | 0.12893, 467 | "\bhe" 468 | ], 469 | [ 470 | 0.351051, 471 | "l" 472 | ], 473 | [ 474 | 0.409096, 475 | "m" 476 | ], 477 | [ 478 | 0.911569, 479 | " " 480 | ], 481 | [ 482 | 0.392391, 483 | "m" 484 | ], 485 | [ 486 | 0.047662, 487 | "o" 488 | ], 489 | [ 490 | 0.20778, 491 | "n" 492 | ], 493 | [ 494 | 0.032247, 495 | "i" 496 | ], 497 | [ 498 | 0.161339, 499 | "t" 500 | ], 501 | [ 502 | 0.095482, 503 | "o" 504 | ], 505 | [ 506 | 0.110821, 507 | "r" 508 | ], 509 | [ 510 | 1.104466, 511 | " " 512 | ], 513 | [ 514 | 0.432258, 515 | "p" 516 | ], 517 | [ 518 | 0.103498, 519 | "r" 520 | ], 521 | [ 522 | 0.08817, 523 | "o" 524 | ], 525 | [ 526 | 0.200004, 527 | "m" 528 | ], 529 | [ 530 | 0.095837, 531 | "e" 532 | ], 533 | [ 534 | 0.111693, 535 | "t" 536 | ], 537 | [ 538 | 0.16822, 539 | "h" 540 | ], 541 | [ 542 | 0.072261, 543 | "e" 544 | ], 545 | [ 546 | 0.103701, 547 | "u" 548 | ], 549 | [ 550 | 0.119979, 551 | "s" 552 | ], 553 | [ 554 | 0.152455, 555 | " " 556 | ], 557 | [ 558 | 0.633027, 559 | "-" 560 | ], 561 | [ 562 | 0.102568, 563 | "-" 564 | ], 565 | [ 566 | 0.288158, 567 | "i" 568 | ], 569 | [ 570 | 0.039597, 571 | "n" 572 | ], 573 | [ 574 | 0.079893, 575 | "t" 576 | ], 577 | [ 578 | 0.056893, 579 | "e" 580 | ], 581 | [ 582 | 0.071287, 583 | "r" 584 | ], 585 | [ 586 | 0.232403, 587 | "v" 588 | ], 589 | [ 590 | 0.039974, 591 | "a" 592 | ], 593 | [ 594 | 0.12858, 595 | "l" 596 | ], 597 | [ 598 | 1.112379, 599 | "=" 600 | ], 601 | [ 602 | 0.135537, 603 | "2" 604 | ], 605 | [ 606 | 1.063811, 607 | " " 608 | ], 609 | [ 610 | 0.415772, 611 | "-" 612 | ], 613 | [ 614 | 0.017374, 615 | "\u001b[?25l\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[44;41H\u001b[?12l\u001b[?25h" 616 | ], 617 | [ 618 | 0.118693, 619 | "-" 620 | ], 621 | [ 622 | 0.199331, 623 | "p" 624 | ], 625 | [ 626 | 0.097116, 627 | "r" 628 | ], 629 | [ 630 | 0.062987, 631 | "o" 632 | ], 633 | [ 634 | 0.200582, 635 | "m" 636 | ], 637 | [ 638 | 0.080288, 639 | "e" 640 | ], 641 | [ 642 | 0.111836, 643 | "t" 644 | ], 645 | [ 646 | 0.1041, 647 | "h" 648 | ], 649 | [ 650 | 0.073529, 651 | "e" 652 | ], 653 | [ 654 | 0.094191, 655 | "u" 656 | ], 657 | [ 658 | 0.113009, 659 | "s" 660 | ], 661 | [ 662 | 0.64003, 663 | "=" 664 | ], 665 | [ 666 | 0.31149, 667 | "h" 668 | ], 669 | [ 670 | 0.136099, 671 | "t" 672 | ], 673 | [ 674 | 0.134419, 675 | "t" 676 | ], 677 | [ 678 | 0.104078, 679 | "p" 680 | ], 681 | [ 682 | 0.288615, 683 | ":" 684 | ], 685 | [ 686 | 0.191777, 687 | "/" 688 | ], 689 | [ 690 | 0.136258, 691 | "/" 692 | ], 693 | [ 694 | 0.223462, 695 | "l" 696 | ], 697 | [ 698 | 0.152229, 699 | "o" 700 | ], 701 | [ 702 | 0.071732, 703 | "c" 704 | ], 705 | [ 706 | 0.024041, 707 | "a" 708 | ], 709 | [ 710 | 0.096341, 711 | "l" 712 | ], 713 | [ 714 | 0.343786, 715 | "h" 716 | ], 717 | [ 718 | 0.03193, 719 | "o" 720 | ], 721 | [ 722 | 0.072058, 723 | "s" 724 | ], 725 | [ 726 | 0.064284, 727 | "t" 728 | ], 729 | [ 730 | 0.616102, 731 | ":" 732 | ], 733 | [ 734 | 0.600838, 735 | "9" 736 | ], 737 | [ 738 | 0.047477, 739 | "0" 740 | ], 741 | [ 742 | 0.296576, 743 | "9" 744 | ], 745 | [ 746 | 0.031492, 747 | "0" 748 | ], 749 | [ 750 | 1.232341, 751 | " " 752 | ], 753 | [ 754 | 0.471941, 755 | "m" 756 | ], 757 | [ 758 | 0.24769, 759 | "y" 760 | ], 761 | [ 762 | 0.256013, 763 | "-" 764 | ], 765 | [ 766 | 0.176522, 767 | "r" 768 | ], 769 | [ 770 | 0.058124, 771 | "e" 772 | ], 773 | [ 774 | 0.125146, 775 | "l" 776 | ], 777 | [ 778 | 0.14435, 779 | "e" 780 | ], 781 | [ 782 | 0.047628, 783 | "a" 784 | ], 785 | [ 786 | 0.072316, 787 | "s" 788 | ], 789 | [ 790 | 0.096205, 791 | "e" 792 | ], 793 | [ 794 | 0.99172, 795 | " " 796 | ], 797 | [ 798 | 0.408783, 799 | "'" 800 | ], 801 | [ 802 | 0.175623, 803 | "r" 804 | ], 805 | [ 806 | 0.041241, 807 | "a" 808 | ], 809 | [ 810 | 0.50222, 811 | "t" 812 | ], 813 | [ 814 | 0.040536, 815 | "e" 816 | ], 817 | [ 818 | 1.056839, 819 | "(" 820 | ], 821 | [ 822 | 0.903014, 823 | "h" 824 | ], 825 | [ 826 | 0.143744, 827 | "t" 828 | ], 829 | [ 830 | 0.240484, 831 | "t" 832 | ], 833 | [ 834 | 0.271759, 835 | "p" 836 | ], 837 | [ 838 | 0.359418, 839 | "_" 840 | ], 841 | [ 842 | 0.192911, 843 | "r" 844 | ], 845 | [ 846 | 0.055866, 847 | "e" 848 | ], 849 | [ 850 | 0.208006, 851 | "q" 852 | ], 853 | [ 854 | 0.096354, 855 | "u" 856 | ], 857 | [ 858 | 0.103322, 859 | "e" 860 | ], 861 | [ 862 | 0.16831, 863 | "s" 864 | ], 865 | [ 866 | 0.063796, 867 | "t" 868 | ], 869 | [ 870 | 0.177971, 871 | "s" 872 | ], 873 | [ 874 | 0.574348, 875 | "_" 876 | ], 877 | [ 878 | 0.224209, 879 | "t" 880 | ], 881 | [ 882 | 0.103747, 883 | "o" 884 | ], 885 | [ 886 | 0.135938, 887 | "t" 888 | ], 889 | [ 890 | 0.191542, 891 | "a" 892 | ], 893 | [ 894 | 0.168591, 895 | "l" 896 | ], 897 | [ 898 | 0.600033, 899 | "{" 900 | ], 901 | [ 902 | 0.192097, 903 | "c" 904 | ], 905 | [ 906 | 0.183575, 907 | "o" 908 | ], 909 | [ 910 | 0.159702, 911 | "d" 912 | ], 913 | [ 914 | 0.048565, 915 | "e" 916 | ], 917 | [ 918 | 0.743974, 919 | "=" 920 | ], 921 | [ 922 | 0.488089, 923 | "~" 924 | ], 925 | [ 926 | 1.583887, 927 | "\"" 928 | ], 929 | [ 930 | 0.760034, 931 | "^" 932 | ], 933 | [ 934 | 0.975835, 935 | "5" 936 | ], 937 | [ 938 | 0.304269, 939 | "." 940 | ], 941 | [ 942 | 1.487507, 943 | "*" 944 | ], 945 | [ 946 | 2.0, 947 | "$" 948 | ], 949 | [ 950 | 0.657102, 951 | "\"" 952 | ], 953 | [ 954 | 0.918845, 955 | "}" 956 | ], 957 | [ 958 | 0.944219, 959 | "[" 960 | ], 961 | [ 962 | 0.167986, 963 | "1" 964 | ], 965 | [ 966 | 0.68799, 967 | "m" 968 | ], 969 | [ 970 | 1.440151, 971 | "]" 972 | ], 973 | [ 974 | 0.344044, 975 | ")" 976 | ], 977 | [ 978 | 0.376116, 979 | " " 980 | ], 981 | [ 982 | 0.960236, 983 | ">" 984 | ], 985 | [ 986 | 0.391439, 987 | " " 988 | ], 989 | [ 990 | 0.176239, 991 | "0" 992 | ], 993 | [ 994 | 1.040204, 995 | "'" 996 | ], 997 | [ 998 | 0.591462, 999 | "\u001b[1;44r\u001b[44;145H\n\u001b[1;45r\u001b[44;1H\u001b[?2004l" 1000 | ], 1001 | [ 1002 | 0.022526, 1003 | "\u001b[?25l\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:helm*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[44;1H\u001b[?12l\u001b[?25h" 1004 | ], 1005 | [ 1006 | 0.294117, 1007 | "\u001b[1;44r\u001b[44;145H\n\u001b[43;1HMonitoring my-release...\u001b[1;45r\u001b[44;1H" 1008 | ], 1009 | [ 1010 | 1.879496, 1011 | "\u001b[?25l\u001b[38;5;235m\u001b[21A─────────────────────────────────────────────────────────────────────────\u001b[38;5;240m────────────────────────────────────────────────────────────────────────\u001b[1;1H\u001b(B\u001b[mmy-release-app-6b9ff444dd-qnhp7 0/1 ContainerCreating 0 0s\u001b[K\r\nmy-release-app-c75998dc7-ggmg7 1/1 Running 0 8m\u001b[K\r\nmy-release-app-c75998dc7-mpv9j 1/1 Running 0 8m\u001b[K\r\nmy-release-app-c75998dc7-vfd6z 1/1 Terminating 0 8m\u001b[K\r\n\u001b[K\r\n==> v1/Service\u001b[K\r\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\u001b[K\r\nmy-release-app LoadBalancer 10.96.10.22 80:30995/TCP 3d\u001b[K\r\n\u001b[K\r\n\u001b[K\r\nNOTES:\u001b[K\r\n1. Get the application URL by running th" 1012 | ], 1013 | [ 1014 | 0.000208, 1015 | "ese commands:\u001b[K\r\n NOTE: It may take a few minutes for the LoadBalancer IP to be available.\u001b[K\r\n You can watch the status of by running 'kubectl get svc -w my-release-app'\u001b[K\r\n export SERVICE_IP=$(kubectl get svc --namespace default my-release-app -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\u001b[K\r\n echo http://$SERVICE_IP:80\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m helm monitor prometheus --interval=2 --prometheus=http://localhost:9090 my-release 'rate(http_requests_total{code=~\"^5.*$\"}[1m]) > 0'\u001b[K\r\nMonitoring my-release...\u001b[K\r\n\u001b[K\u001b[2B\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:helm*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[24;1H\u001b[?12l\u001b[?25h" 1016 | ], 1017 | [ 1018 | 1.151155, 1019 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \u001b[24;1H \r" 1020 | ], 1021 | [ 1022 | 0.062525, 1023 | "\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m \r\n" 1024 | ], 1025 | [ 1026 | 0.01378, 1027 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:~/workspace/src*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[26;1H\u001b[?12l\u001b[?25h" 1028 | ], 1029 | [ 1030 | 0.000607, 1031 | "\u001b[24;44r\u001b[27;44r\u001b[18S\u001b[26;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[26;3H\u001b[?2004h" 1032 | ], 1033 | [ 1034 | 1.465039, 1035 | "c" 1036 | ], 1037 | [ 1038 | 0.094979, 1039 | "\bcu" 1040 | ], 1041 | [ 1042 | 0.112182, 1043 | "r" 1044 | ], 1045 | [ 1046 | 0.135349, 1047 | "l" 1048 | ], 1049 | [ 1050 | 0.632684, 1051 | " " 1052 | ], 1053 | [ 1054 | 0.430822, 1055 | "1" 1056 | ], 1057 | [ 1058 | 0.112521, 1059 | "9" 1060 | ], 1061 | [ 1062 | 0.47238, 1063 | "2" 1064 | ], 1065 | [ 1066 | 0.784309, 1067 | "." 1068 | ], 1069 | [ 1070 | 0.231898, 1071 | "1" 1072 | ], 1073 | [ 1074 | 0.295593, 1075 | "6" 1076 | ], 1077 | [ 1078 | 0.048157, 1079 | "8" 1080 | ], 1081 | [ 1082 | 1.51246, 1083 | "." 1084 | ], 1085 | [ 1086 | 0.457358, 1087 | "9" 1088 | ], 1089 | [ 1090 | 0.206543, 1091 | "9" 1092 | ], 1093 | [ 1094 | 0.512343, 1095 | "." 1096 | ], 1097 | [ 1098 | 0.263548, 1099 | "1" 1100 | ], 1101 | [ 1102 | 0.128147, 1103 | "0" 1104 | ], 1105 | [ 1106 | 0.119663, 1107 | "0" 1108 | ], 1109 | [ 1110 | 1.904517, 1111 | ":" 1112 | ], 1113 | [ 1114 | 0.575748, 1115 | "3" 1116 | ], 1117 | [ 1118 | 0.687489, 1119 | "0" 1120 | ], 1121 | [ 1122 | 0.761427, 1123 | "9" 1124 | ], 1125 | [ 1126 | 0.134837, 1127 | "9" 1128 | ], 1129 | [ 1130 | 0.423936, 1131 | "5" 1132 | ], 1133 | [ 1134 | 0.616587, 1135 | "/" 1136 | ], 1137 | [ 1138 | 0.343983, 1139 | "i" 1140 | ], 1141 | [ 1142 | 0.026288, 1143 | "n" 1144 | ], 1145 | [ 1146 | 0.077371, 1147 | "t" 1148 | ], 1149 | [ 1150 | 0.104049, 1151 | "e" 1152 | ], 1153 | [ 1154 | 0.056367, 1155 | "r" 1156 | ], 1157 | [ 1158 | 0.127691, 1159 | "n" 1160 | ], 1161 | [ 1162 | 0.095813, 1163 | "a" 1164 | ], 1165 | [ 1166 | 0.184452, 1167 | "l" 1168 | ], 1169 | [ 1170 | 0.263946, 1171 | "-" 1172 | ], 1173 | [ 1174 | 0.248124, 1175 | "e" 1176 | ], 1177 | [ 1178 | 0.041673, 1179 | "r" 1180 | ], 1181 | [ 1182 | 0.149999, 1183 | "r" 1184 | ], 1185 | [ 1186 | 0.112046, 1187 | "o" 1188 | ], 1189 | [ 1190 | 0.056337, 1191 | "r" 1192 | ], 1193 | [ 1194 | 0.695676, 1195 | "\r\n\u001b[?2004l" 1196 | ], 1197 | [ 1198 | 0.012696, 1199 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:curl*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[27;1H\u001b[?12l\u001b[?25h" 1200 | ], 1201 | [ 1202 | 0.02326, 1203 | "Internal Server Error" 1204 | ], 1205 | [ 1206 | 0.003346, 1207 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \r \r" 1208 | ], 1209 | [ 1210 | 0.052255, 1211 | "\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m \r\n" 1212 | ], 1213 | [ 1214 | 0.002444, 1215 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:~/workspace/src*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[30;1H\u001b[?12l\u001b[?25h" 1216 | ], 1217 | [ 1218 | 0.000146, 1219 | "\u001b[24;44r\u001b[31;44r\u001b[14S\u001b[30;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[30;3H\u001b[?2004h" 1220 | ], 1221 | [ 1222 | 0.888777, 1223 | "curl 192.168.99.100:30995/internal-error" 1224 | ], 1225 | [ 1226 | 0.072178, 1227 | "\r\n\u001b[?2004l" 1228 | ], 1229 | [ 1230 | 0.01059, 1231 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:curl*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[31;1H\u001b[?12l\u001b[?25h" 1232 | ], 1233 | [ 1234 | 0.012515, 1235 | "Internal Server Error" 1236 | ], 1237 | [ 1238 | 0.003596, 1239 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \r \r" 1240 | ], 1241 | [ 1242 | 0.062747, 1243 | "\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m \u001b[?25l\u001b[45;1H\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:~/workspace/src*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[34;1H\u001b[?12l\u001b[?25h" 1244 | ], 1245 | [ 1246 | 6.8e-05, 1247 | "\u001b[24;44r\u001b[35;44r\u001b[10S\u001b[34;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[34;3H\u001b[?2004h" 1248 | ], 1249 | [ 1250 | 0.247087, 1251 | "curl 192.168.99.100:30995/internal-error" 1252 | ], 1253 | [ 1254 | 0.039979, 1255 | "\r\n\u001b[?2004l" 1256 | ], 1257 | [ 1258 | 0.007936, 1259 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:curl*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[35;1H\u001b[?12l\u001b[?25h" 1260 | ], 1261 | [ 1262 | 0.016297, 1263 | "Internal Server Error" 1264 | ], 1265 | [ 1266 | 0.002137, 1267 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \r \r" 1268 | ], 1269 | [ 1270 | 0.034651, 1271 | "\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m \r\n" 1272 | ], 1273 | [ 1274 | 0.001762, 1275 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:~/workspace/src*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[38;1H\u001b[?12l\u001b[?25h\u001b[24;44r\u001b[39;44r\u001b[6S\u001b[38;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[38;3H\u001b[?2004h" 1276 | ], 1277 | [ 1278 | 0.256827, 1279 | "curl 192.168.99.100:30995/internal-error" 1280 | ], 1281 | [ 1282 | 0.05689, 1283 | "\r\n\u001b[?2004l" 1284 | ], 1285 | [ 1286 | 0.005497, 1287 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:curl*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[39;1H\u001b[?12l\u001b[?25h" 1288 | ], 1289 | [ 1290 | 0.015206, 1291 | "Internal Server Error" 1292 | ], 1293 | [ 1294 | 0.004768, 1295 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \r \r" 1296 | ], 1297 | [ 1298 | 0.033308, 1299 | "\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m \r\n" 1300 | ], 1301 | [ 1302 | 0.002039, 1303 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:~/workspace/src*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[42;1H\u001b[?12l\u001b[?25h" 1304 | ], 1305 | [ 1306 | 7.9e-05, 1307 | "\u001b[24;44r\u001b[43;1H\u001b[K\r\n\u001b[K\u001b[2A\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[42;3H\u001b[?2004h" 1308 | ], 1309 | [ 1310 | 0.546427, 1311 | "curl 192.168.99.100:30995/internal-error" 1312 | ], 1313 | [ 1314 | 0.048249, 1315 | "\r\n\u001b[?2004l" 1316 | ], 1317 | [ 1318 | 0.010965, 1319 | "\u001b[?25l\u001b[45d\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:curl*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[43;1H\u001b[?12l\u001b[?25h" 1320 | ], 1321 | [ 1322 | 0.013907, 1323 | "Internal Server Error" 1324 | ], 1325 | [ 1326 | 0.003799, 1327 | "\u001b[1m\u001b[7m%\u001b(B\u001b[m \r \r" 1328 | ], 1329 | [ 1330 | 0.040192, 1331 | "\u001b[24;44r\u001b[2S\u001b[43;1H\u001b[34m~/workspace/src \u001b[90m \u001b[39m \u001b[1;45r\u001b[44;1H" 1332 | ], 1333 | [ 1334 | 0.003568, 1335 | "\u001b[?25l\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:~/workspace/src*\u001b[38;5;136m \"etiennebook\" 09:28 19-Jan-18\u001b(B\u001b[m\u001b[44;1H\u001b[?12l\u001b[?25h\u001b[24;44r\u001b[44;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[44;3H\u001b[?2004h" 1336 | ], 1337 | [ 1338 | 2.0, 1339 | "\u001b[1;22r\u001b[22;145H\n\u001b[21;1HFailure detected, rolling back...\u001b[1;45r\u001b[44;3H" 1340 | ], 1341 | [ 1342 | 0.286311, 1343 | "\u001b[1;22r\u001b[22;145H\n\u001b[21;1HSuccessfully rolled back to previous revision!\u001b[1;45r\u001b[44;3H" 1344 | ], 1345 | [ 1346 | 0.00903, 1347 | "\u001b[22;1H\u001b[1m\u001b[7m%\u001b(B\u001b[m \u001b[22;1H \u001b[44;3H" 1348 | ], 1349 | [ 1350 | 0.104147, 1351 | "\u001b[1;22r\u001b[2S\u001b[21;1H\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m \u001b[33m25s\u001b[1;45r\u001b[44;3H\u001b(B\u001b[m" 1352 | ], 1353 | [ 1354 | 0.017086, 1355 | "\u001b[?25l\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m \"etiennebook\" 09:29 19-Jan-18\u001b(B\u001b[m\u001b[44;3H\u001b[?12l\u001b[?25h\u001b[1;22r\u001b[22;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[44;3H" 1356 | ], 1357 | [ 1358 | 0.914659, 1359 | "\u001b[?25l\u001b[38;5;240m\u001b[23;1H─────────────────────────────────────────────────────────────────────────\u001b[38;5;235m────────────────────────────────────────────────────────────────────────\u001b(B\u001b[m\u001b[22;3H\u001b[?12l\u001b[?25h" 1360 | ], 1361 | [ 1362 | 1.384264, 1363 | "\u001b[?25l\u001b[H \u001b[30m\u001b[43m[0/41]\u001b[2;1H\u001b[39m\u001b[49m==> v1/Service\u001b[K\r\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\u001b[K\r\nmy-release-app LoadBalancer 10.96.10.22 80:30995/TCP 3d\u001b[K\r\n\u001b[K\r\n\u001b[K\r\nNOTES:\u001b[K\r\n1. Get the application URL by running these commands:\u001b[K\r\n NOTE: It may take a few minutes for the LoadBalancer IP to be available.\u001b[K\r\n You can watch the status of by running 'kubectl get svc -w my-release-app'\u001b[K\r\n export SERVICE_IP=$(kubectl get svc --namespace default my-release-app -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\u001b[K\r\n echo http://$SERVICE_IP:80\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m helm monitor prometheus --interval=2 --prometheus=http://localhost:9090 my-release 'rate(http_requests_total{code=~\"^5.*$\"}[1m]) > 0'\u001b[K\r\nMonitoring my-release...\u001b[" 1364 | ], 1365 | [ 1366 | 0.000107, 1367 | "K\r\nFailure detected, rolling back...\u001b[K\r\nSuccessfully rolled back to previous revision!\u001b[K\r\n\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m \u001b[33m25s\u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m\u001b[K\u001b[C\u001b[?12l\u001b[?25h\u001b[?2004l" 1368 | ], 1369 | [ 1370 | 0.358667, 1371 | "\u001b[21;39H" 1372 | ], 1373 | [ 1374 | 0.140667, 1375 | "\u001b[20;1H" 1376 | ], 1377 | [ 1378 | 0.135719, 1379 | "\u001b[19;47H" 1380 | ], 1381 | [ 1382 | 0.480098, 1383 | "\b" 1384 | ], 1385 | [ 1386 | 0.159269, 1387 | "\u001b[16;1H\u001b[35m❯\u001b[19;46H\u001b[30m\u001b[43m!\u001b[22;1H\u001b[49m\u001b[35m❯\u001b[19;46H\u001b(B\u001b[m" 1388 | ], 1389 | [ 1390 | 0.135573, 1391 | "\u001b[A\u001b[30m\u001b[43m \u001b[19;1HSuccessfully rolled back to previous revision!\u001b[18;34H \u001b[18;34H\u001b(B\u001b[m" 1392 | ], 1393 | [ 1394 | 0.131333, 1395 | "\u001b[17;46H\u001b[30m\u001b[43m \u001b[18;1HFailure detected, rolling back... \u001b[17;25H \u001b[17;25H\u001b(B\u001b[m" 1396 | ], 1397 | [ 1398 | 0.461341, 1399 | "\u001b[16;1H\u001b[35m❯\u001b[44C\u001b[30m\u001b[43metheus=http://localhost:9090 my-release 'rate(http_requests_total{code=~\"^5.*$\"}[1m]) > 0' \u001b[17;1HMonitoring my-release... \u001b[16;46H\u001b(B\u001b[m" 1400 | ], 1401 | [ 1402 | 0.779193, 1403 | "\r\u001b[30m\u001b[43m❯ helm monitor prometheus --interval=2 --prometheus=http://localhost:9090 my-release 'rate(http_requests_total{code=~\"^5.*$\"}[1m]) > 0' \u001b[16;1H\u001b(B\u001b[m" 1404 | ], 1405 | [ 1406 | 2.0, 1407 | "\u001b[?25l\u001b[H\u001b[K\r\n==> v1/Service\u001b[K\r\nNAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\u001b[K\r\nmy-release-app LoadBalancer 10.96.10.22 80:30995/TCP 3d\u001b[K\r\n\u001b[K\r\n\u001b[K\r\nNOTES:\u001b[K\r\n1. Get the application URL by running these commands:\u001b[K\r\n NOTE: It may take a few minutes for the LoadBalancer IP to be available.\u001b[K\r\n You can watch the status of by running 'kubectl get svc -w my-release-app'\u001b[K\r\n export SERVICE_IP=$(kubectl get svc --namespace default my-release-app -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\u001b[K\r\n echo http://$SERVICE_IP:80\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m helm monitor prometheus --interval=2 --prometheus=http://localhost:9090 my-release 'rate(http_requests_total{code=~\"^5.*$\"}[1m]) > 0'\u001b[K\r\nMonitoring my-release...\u001b[K\r\nFailure detected, rolling back...\u001b[K\r\nSuccessfully rolled back to previous revision!\u001b[K\r\n\u001b[K\r\n\u001b[34m\u001b[1mhelm-monitor\u001b(B\u001b[m\u001b[34m/examples \u001b[90mgit/master* \u001b[39m \u001b[33" 1408 | ], 1409 | [ 1410 | 0.000132, 1411 | "m25s\u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m\u001b[K\u001b[C\u001b[?12l\u001b[?25h\u001b[?2004h" 1412 | ], 1413 | [ 1414 | 0.645838, 1415 | "\u001b[1;22r\u001b[22;145H\n\u001b[1;45r\u001b[22;1H\u001b[?2004l" 1416 | ], 1417 | [ 1418 | 0.058149, 1419 | "\u001b[?25l\u001b[H\u001b[K\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m curl 192.168.99.100:30995/internal-error\u001b[K\r\nInternal Server Error\u001b[1m\u001b[7m%\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m curl 192.168.99.100:30995/internal-error\u001b[K\r\nInternal Server Error\u001b[1m\u001b[7m%\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m curl 192.168.99.100:30995/internal-error\u001b[K\r\nInternal Server Error\u001b[1m\u001b[7m%\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m curl 192.168.99.100:30995/internal-error\u001b[K\r\nInternal Server Error\u001b[1m\u001b[7m%\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m curl 192.168.99.100:30995/internal-error\u001b[K\r\nInternal Server Error\u001b[1m\u001b[7m%\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[34m~/workspace/src \u001b[90m \u001b[39m\u001b[K\r\n\u001b[35m❯\u001b[39m\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[38;5;136m\u001b[48;5;235m[helm-moni\u001b[38;5;166m1:..itor/examples*\u001b[38;5;136m " 1420 | ], 1421 | [ 1422 | 0.000251, 1423 | " \"etiennebook\" 09:29 19-Jan-18\u001b(B\u001b[m\u001b[23;3H\u001b[?12l\u001b[?25h\u001b[?2004h" 1424 | ], 1425 | [ 1426 | 4.4e-05, 1427 | "\u001b[1;44r\u001b[24;44r\u001b[21S\u001b[23;1H\u001b[K\u001b[35m❯\u001b[39m \u001b[K\u001b[1;45r\u001b[23;3H" 1428 | ], 1429 | [ 1430 | 0.125764, 1431 | "\r\n\u001b[?2004l" 1432 | ], 1433 | [ 1434 | 0.058426, 1435 | "\u001b[1;45r\u001b(B\u001b[m\u001b[?1l\u001b>\u001b[H\u001b[2J\u001b]112\u0007\u001b[?12l\u001b[?25h\u001b[?1000l\u001b[?1002l\u001b[?1006l\u001b[?1005l\u001b[?1004l\u001b[?1049l" 1436 | ], 1437 | [ 1438 | 0.000775, 1439 | "[exited]\r\n" 1440 | ] 1441 | ] 1442 | } -------------------------------------------------------------------------------- /cmd/monitor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/spf13/cobra" 10 | "google.golang.org/grpc" 11 | "k8s.io/helm/pkg/helm" 12 | helm_env "k8s.io/helm/pkg/helm/environment" 13 | ) 14 | 15 | var ( 16 | settings helm_env.EnvSettings 17 | monitor *monitorCmd 18 | verbose bool 19 | ) 20 | 21 | type monitorCmd struct { 22 | disableHooks bool 23 | dryRun bool 24 | expectedResultCount int64 25 | force bool 26 | interval int64 27 | rollbackTimeout int64 28 | timeout int64 29 | wait bool 30 | } 31 | 32 | const monitorDesc = ` 33 | This command monitor a release by querying Prometheus or Elasticsearch at a 34 | given interval and take care of rolling back to the previous version if the 35 | query return a non-empty result. 36 | ` 37 | 38 | func setupConnection(c *cobra.Command, args []string) error { 39 | settings.TillerHost = os.Getenv("TILLER_HOST") 40 | return nil 41 | } 42 | 43 | func ensureHelmClient(h helm.Interface) helm.Interface { 44 | if h != nil { 45 | return h 46 | } 47 | 48 | return helm.NewClient(helm.Host(settings.TillerHost)) 49 | } 50 | 51 | func prettyError(err error) error { 52 | if err == nil { 53 | return nil 54 | } 55 | return errors.New(grpc.ErrorDesc(err)) 56 | } 57 | 58 | func debug(format string, args ...interface{}) { 59 | if verbose { 60 | format = fmt.Sprintf("[debug] %s\n", format) 61 | fmt.Printf(format, args...) 62 | } 63 | } 64 | 65 | func newMonitorCmd(out io.Writer) *cobra.Command { 66 | monitor = &monitorCmd{} 67 | 68 | cmd := &cobra.Command{ 69 | Use: "monitor prometheus|elasticsearch", 70 | Short: "monitor a release", 71 | Long: monitorDesc, 72 | } 73 | 74 | p := cmd.PersistentFlags() 75 | p.BoolVar(&monitor.disableHooks, "no-hooks", false, "prevent hooks from running during rollback") 76 | p.BoolVar(&monitor.dryRun, "dry-run", false, "simulate a rollback if triggered by query result") 77 | p.Int64Var(&monitor.expectedResultCount, "expected-result-count", 0, "number of results that are expected to be returned by the query (rollback triggered if the number of results exceeds this value)") 78 | p.BoolVar(&monitor.force, "force", false, "force resource update through delete/recreate if needed") 79 | p.BoolVar(&monitor.wait, "wait", false, "if set, will wait until all Pods, PVCs, Services, and minimum number of Pods of a Deployment are in a ready state before marking a rollback as successful. It will wait for as long as --rollback-timeout") 80 | p.BoolVarP(&verbose, "verbose", "v", false, "enable verbose output") 81 | p.Int64Var(&monitor.rollbackTimeout, "rollback-timeout", 300, "time in seconds to wait for any individual Kubernetes operation during the rollback (like Jobs for hooks)") 82 | p.Int64Var(&monitor.timeout, "timeout", 300, "time in seconds to wait before assuming a monitoring action is successfull") 83 | p.Int64VarP(&monitor.interval, "interval", "i", 10, "time in seconds between each query") 84 | 85 | cmd.AddCommand( 86 | newMonitorPrometheusCmd(out), 87 | newMonitorElasticsearchCmd(out), 88 | newMonitorSentryCmd(out), 89 | ) 90 | 91 | return cmd 92 | } 93 | 94 | func main() { 95 | cmd := newMonitorCmd(os.Stdout) 96 | if err := cmd.Execute(); err != nil { 97 | os.Exit(1) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /cmd/monitor_elasticsearch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/spf13/cobra" 15 | "k8s.io/helm/pkg/helm" 16 | ) 17 | 18 | const monitorElasticsearchDesc = ` 19 | This command monitor a release by querying Elasticsearch at a given interval 20 | and take care of rolling back to the previous version if the query return a non- 21 | empty result. 22 | 23 | The query argument can be either the path of a query DSL json file or a Lucene 24 | query string. 25 | 26 | Example with Lucene query: 27 | 28 | $ helm monitor elasticsearch my-release 'status:500 AND kubernetes.labels.app:app AND version:2.0.0' 29 | 30 | Example with query DSL file: 31 | 32 | $ helm monitor elasticsearch my-release ./examples/elasticsearch-query.json 33 | 34 | 35 | Reference: 36 | 37 | https://www.elastic.co/guide/en/elasticsearch/reference/current/search-count.html 38 | 39 | ` 40 | 41 | type monitorElasticsearchCmd struct { 42 | name string 43 | out io.Writer 44 | client helm.Interface 45 | elasticsearchAddr string 46 | query string 47 | } 48 | 49 | type elasticsearchQueryResponse struct { 50 | Count int64 `json:"count"` 51 | } 52 | 53 | func newMonitorElasticsearchCmd(out io.Writer) *cobra.Command { 54 | m := &monitorElasticsearchCmd{ 55 | out: out, 56 | } 57 | 58 | cmd := &cobra.Command{ 59 | Use: "elasticsearch [flags] RELEASE [QUERY DSL PATH|LUCENE QUERY]", 60 | Short: "query an elasticsearch server", 61 | Long: monitorElasticsearchDesc, 62 | PreRunE: setupConnection, 63 | RunE: func(cmd *cobra.Command, args []string) error { 64 | if len(args) != 2 { 65 | return fmt.Errorf("This command neeeds 2 argument: release name, query DSL path or Lucene query") 66 | } 67 | 68 | m.name = args[0] 69 | m.query = args[1] 70 | m.client = ensureHelmClient(m.client) 71 | 72 | return m.run() 73 | }, 74 | } 75 | 76 | f := cmd.Flags() 77 | f.StringVar(&m.elasticsearchAddr, "elasticsearch", "http://localhost:9200", "elasticsearch address") 78 | 79 | return cmd 80 | } 81 | 82 | func (m *monitorElasticsearchCmd) run() error { 83 | _, err := m.client.ReleaseContent(m.name) 84 | if err != nil { 85 | return prettyError(err) 86 | } 87 | 88 | fmt.Fprintf(m.out, "Monitoring %s...\n", m.name) 89 | 90 | client := &http.Client{Timeout: 10 * time.Second} 91 | 92 | queryBody, err := os.Open(m.query) 93 | 94 | var req *http.Request 95 | if err != nil { 96 | req, err = http.NewRequest("GET", m.elasticsearchAddr+"/_count", nil) 97 | if err != nil { 98 | return prettyError(err) 99 | } 100 | 101 | q := req.URL.Query() 102 | q.Add("q", m.query) 103 | req.URL.RawQuery = q.Encode() 104 | } else { 105 | req, err = http.NewRequest("GET", m.elasticsearchAddr+"/_count", queryBody) 106 | if err != nil { 107 | return prettyError(err) 108 | } 109 | req.Header.Set("Content-Type", "application/json") 110 | } 111 | 112 | quit := make(chan os.Signal) 113 | signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) 114 | 115 | ticker := time.NewTicker(time.Second * time.Duration(monitor.interval)) 116 | 117 | go func() { 118 | time.Sleep(time.Second * time.Duration(monitor.timeout)) 119 | fmt.Fprintf(m.out, "No results after %d second(s)\n", monitor.timeout) 120 | close(quit) 121 | }() 122 | 123 | for { 124 | select { 125 | case <-ticker.C: 126 | debug("Processing URL %s", req.URL.String()) 127 | 128 | res, err := client.Do(req) 129 | if err != nil { 130 | return prettyError(err) 131 | } 132 | 133 | defer res.Body.Close() 134 | 135 | body, err := ioutil.ReadAll(res.Body) 136 | 137 | if err != nil { 138 | return prettyError(err) 139 | } 140 | 141 | fmt.Printf("Body: %s\n", string(body)) 142 | 143 | response := &elasticsearchQueryResponse{} 144 | err = json.Unmarshal(body, response) 145 | if err != nil { 146 | return prettyError(err) 147 | } 148 | 149 | debug("Response: %v", response) 150 | debug("Result count: %d", response.Count) 151 | 152 | if response.Count > monitor.expectedResultCount { 153 | ticker.Stop() 154 | 155 | fmt.Fprintf(m.out, "Failure detected, rolling back...\n") 156 | 157 | _, err := m.client.RollbackRelease( 158 | m.name, 159 | helm.RollbackDryRun(monitor.dryRun), 160 | helm.RollbackRecreate(false), 161 | helm.RollbackForce(monitor.force), 162 | helm.RollbackDisableHooks(monitor.disableHooks), 163 | helm.RollbackVersion(0), 164 | helm.RollbackTimeout(monitor.rollbackTimeout), 165 | helm.RollbackWait(monitor.wait)) 166 | 167 | if err != nil { 168 | return prettyError(err) 169 | } 170 | 171 | fmt.Fprintf(m.out, "Successfully rolled back to previous revision!\n") 172 | return nil 173 | } 174 | 175 | case <-quit: 176 | ticker.Stop() 177 | debug("Quitting...") 178 | return nil 179 | } 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /cmd/monitor_prometheus.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/spf13/cobra" 15 | "k8s.io/helm/pkg/helm" 16 | ) 17 | 18 | const monitorPrometheusDesc = ` 19 | This command monitor a release by querying Prometheus at a given interval and 20 | take care of rolling back to the previous version if the query return a non- 21 | empty result. 22 | 23 | Example: 24 | 25 | $ helm monitor prometheus my-release 'rate(http_requests_total{code=~"^5.*$"}[5m]) > 0' 26 | 27 | 28 | Reference: 29 | 30 | https://prometheus.io/docs/prometheus/latest/querying/basics/ 31 | 32 | ` 33 | 34 | type monitorPrometheusCmd struct { 35 | name string 36 | out io.Writer 37 | client helm.Interface 38 | prometheusAddr string 39 | query string 40 | } 41 | 42 | type prometheusQueryResponse struct { 43 | Data struct { 44 | Result []struct{} `json:"result"` 45 | } `json:"data"` 46 | } 47 | 48 | func newMonitorPrometheusCmd(out io.Writer) *cobra.Command { 49 | m := &monitorPrometheusCmd{ 50 | out: out, 51 | } 52 | 53 | cmd := &cobra.Command{ 54 | Use: "prometheus [flags] RELEASE PROMQL", 55 | Short: "query a prometheus server", 56 | Long: monitorPrometheusDesc, 57 | PreRunE: setupConnection, 58 | RunE: func(cmd *cobra.Command, args []string) error { 59 | if len(args) != 2 { 60 | return fmt.Errorf("This command neeeds 2 argument: release name, promql") 61 | } 62 | 63 | m.name = args[0] 64 | m.query = args[1] 65 | m.client = ensureHelmClient(m.client) 66 | 67 | return m.run() 68 | }, 69 | } 70 | 71 | f := cmd.Flags() 72 | f.StringVar(&m.prometheusAddr, "prometheus", "http://localhost:9090", "prometheus address") 73 | 74 | return cmd 75 | } 76 | 77 | func (m *monitorPrometheusCmd) run() error { 78 | _, err := m.client.ReleaseContent(m.name) 79 | if err != nil { 80 | return prettyError(err) 81 | } 82 | 83 | fmt.Fprintf(m.out, "Monitoring %s...\n", m.name) 84 | 85 | client := &http.Client{Timeout: 5 * time.Second} 86 | 87 | req, err := http.NewRequest("GET", m.prometheusAddr+"/api/v1/query", nil) 88 | if err != nil { 89 | return prettyError(err) 90 | } 91 | 92 | q := req.URL.Query() 93 | q.Add("query", m.query) 94 | req.URL.RawQuery = q.Encode() 95 | 96 | quit := make(chan os.Signal) 97 | signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) 98 | 99 | ticker := time.NewTicker(time.Second * time.Duration(monitor.interval)) 100 | 101 | go func() { 102 | time.Sleep(time.Second * time.Duration(monitor.timeout)) 103 | fmt.Fprintf(m.out, "No results after %d second(s)\n", monitor.timeout) 104 | close(quit) 105 | }() 106 | 107 | for { 108 | select { 109 | case <-ticker.C: 110 | debug("Processing URL %s", req.URL.String()) 111 | 112 | res, err := client.Do(req) 113 | if err != nil { 114 | return prettyError(err) 115 | } 116 | 117 | defer res.Body.Close() 118 | 119 | body, err := ioutil.ReadAll(res.Body) 120 | 121 | if err != nil { 122 | return prettyError(err) 123 | } 124 | 125 | response := &prometheusQueryResponse{} 126 | err = json.Unmarshal(body, response) 127 | if err != nil { 128 | return prettyError(err) 129 | } 130 | 131 | debug("Response: %v", response) 132 | debug("Result count: %d", len(response.Data.Result)) 133 | 134 | if len(response.Data.Result) > int(monitor.expectedResultCount) { 135 | ticker.Stop() 136 | 137 | fmt.Fprintf(m.out, "Failure detected, rolling back...\n") 138 | 139 | _, err := m.client.RollbackRelease( 140 | m.name, 141 | helm.RollbackDryRun(monitor.dryRun), 142 | helm.RollbackRecreate(false), 143 | helm.RollbackForce(monitor.force), 144 | helm.RollbackDisableHooks(monitor.disableHooks), 145 | helm.RollbackVersion(0), 146 | helm.RollbackTimeout(monitor.rollbackTimeout), 147 | helm.RollbackWait(monitor.wait)) 148 | 149 | if err != nil { 150 | return prettyError(err) 151 | } 152 | 153 | fmt.Fprintf(m.out, "Successfully rolled back to previous revision!\n") 154 | return nil 155 | } 156 | 157 | case <-quit: 158 | ticker.Stop() 159 | debug("Quitting...") 160 | return nil 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /cmd/monitor_sentry.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "os" 10 | "os/signal" 11 | "regexp" 12 | "strings" 13 | "syscall" 14 | "time" 15 | 16 | "github.com/spf13/cobra" 17 | "k8s.io/helm/pkg/helm" 18 | ) 19 | 20 | const monitorSentryDesc = ` 21 | This command monitor a release by querying Sentry at a given interval and 22 | take care of rolling back to the previous version if the query return a non- 23 | empty result. 24 | 25 | Example: 26 | 27 | $ helm monitor sentry my-release \ 28 | --api-key \ 29 | --organization my-organization \ 30 | --project my-project \ 31 | --sentry https://sentry-endpoint/ \ 32 | --tag environment=production \ 33 | --tag release=2.0.0 \ 34 | --message 'Error message' 35 | 36 | Example with event message matching regular expression: 37 | 38 | $ helm monitor sentry my-release \ 39 | --api-key \ 40 | --organization my-organization \ 41 | --project my-project \ 42 | --sentry https://sentry-endpoint/ \ 43 | --message 'pointer.+' \ 44 | --regexp 45 | 46 | ` 47 | 48 | type monitorSentryCmd struct { 49 | name string 50 | out io.Writer 51 | client helm.Interface 52 | sentryAddr string 53 | sentryAPIKey string 54 | sentryOrganization string 55 | sentryProject string 56 | message string 57 | regexp bool 58 | tags []string 59 | } 60 | 61 | type tag struct { 62 | Key string `json:"key"` 63 | Value string `json:"value"` 64 | } 65 | 66 | type sentryEvent struct { 67 | Message string `json:"message"` 68 | Tags []*tag `json:"tags"` 69 | } 70 | 71 | func newMonitorSentryCmd(out io.Writer) *cobra.Command { 72 | m := &monitorSentryCmd{ 73 | out: out, 74 | } 75 | 76 | cmd := &cobra.Command{ 77 | Use: "sentry [flags] RELEASE", 78 | Short: "query a sentry server", 79 | Long: monitorSentryDesc, 80 | PreRunE: setupConnection, 81 | RunE: func(cmd *cobra.Command, args []string) error { 82 | if len(args) != 1 { 83 | return fmt.Errorf("This command neeeds 1 argument: release name") 84 | } 85 | 86 | m.name = args[0] 87 | m.client = ensureHelmClient(m.client) 88 | 89 | return m.run() 90 | }, 91 | } 92 | 93 | f := cmd.Flags() 94 | f.StringVar(&m.sentryAddr, "sentry", "http://localhost:9000", "sentry address") 95 | f.StringVar(&m.sentryAPIKey, "api-key", "", "sentry api key") 96 | f.StringVar(&m.sentryOrganization, "organization", "", "sentry organization") 97 | f.StringVar(&m.sentryProject, "project", "", "sentry project") 98 | f.StringVar(&m.message, "message", "", "event message to match") 99 | f.BoolVar(&m.regexp, "regexp", false, "enable regular expression") 100 | f.StringSliceVar(&m.tags, "tag", []string{}, "tags, ie: --tag release=2.0.0 --tag environment=production") 101 | 102 | cmd.MarkFlagRequired("api-key") 103 | cmd.MarkFlagRequired("organization") 104 | cmd.MarkFlagRequired("project") 105 | 106 | return cmd 107 | } 108 | 109 | func convertStringToTags(s []string) (tagList []*tag) { 110 | tagList = []*tag{} 111 | for _, t := range s { 112 | a := strings.Split(t, "=") 113 | if len(a) != 2 { 114 | debug("Provided tag is malformed, should match pattern key=value, got %s", t) 115 | continue 116 | } 117 | tagList = append(tagList, &tag{ 118 | Key: a[0], 119 | Value: a[1], 120 | }) 121 | } 122 | 123 | return 124 | } 125 | 126 | func matchEvents(eventList []*sentryEvent, message string, tagList []*tag, useRegexp bool) (output []*sentryEvent, err error) { 127 | if message == "" && len(tagList) == 0 { 128 | return eventList, nil 129 | } 130 | 131 | var r *regexp.Regexp 132 | if useRegexp { 133 | r, err = regexp.Compile(message) 134 | if err != nil { 135 | return nil, err 136 | } 137 | } 138 | 139 | for _, event := range eventList { 140 | match := false 141 | if useRegexp && r.MatchString(event.Message) { 142 | match = true 143 | } else if message != "" && event.Message == message { 144 | match = true 145 | } 146 | 147 | if match && len(tagList) > 0 && !matchTags(tagList, event.Tags) { 148 | continue 149 | } 150 | 151 | if match { 152 | output = append(output, event) 153 | } 154 | } 155 | 156 | return 157 | } 158 | 159 | func matchTags(tagList []*tag, matchTagList []*tag) bool { 160 | matchCount := 0 161 | for _, matchTag := range matchTagList { 162 | for _, t := range tagList { 163 | if matchTag.Key == t.Key && matchTag.Value == t.Value { 164 | matchCount++ 165 | } 166 | } 167 | } 168 | if matchCount != len(tagList) { 169 | return false 170 | } 171 | 172 | return true 173 | } 174 | 175 | func (m *monitorSentryCmd) run() error { 176 | _, err := m.client.ReleaseContent(m.name) 177 | if err != nil { 178 | return prettyError(err) 179 | } 180 | 181 | fmt.Fprintf(m.out, "Monitoring %s...\n", m.name) 182 | 183 | client := &http.Client{Timeout: 5 * time.Second} 184 | 185 | req, err := http.NewRequest("GET", m.sentryAddr+"/api/0/projects/"+m.sentryOrganization+"/"+m.sentryProject+"/events/", nil) 186 | if err != nil { 187 | return prettyError(err) 188 | } 189 | 190 | req.Header.Add("Authorization", "Bearer "+m.sentryAPIKey) 191 | 192 | quit := make(chan os.Signal) 193 | signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) 194 | 195 | ticker := time.NewTicker(time.Second * time.Duration(monitor.interval)) 196 | 197 | go func() { 198 | time.Sleep(time.Second * time.Duration(monitor.timeout)) 199 | fmt.Fprintf(m.out, "No results after %d second(s)\n", monitor.timeout) 200 | close(quit) 201 | }() 202 | 203 | for { 204 | select { 205 | case <-ticker.C: 206 | debug("Processing URL %s", req.URL.String()) 207 | 208 | res, err := client.Do(req) 209 | if err != nil { 210 | return prettyError(err) 211 | } 212 | 213 | defer res.Body.Close() 214 | 215 | body, err := ioutil.ReadAll(res.Body) 216 | 217 | if err != nil { 218 | return prettyError(err) 219 | } 220 | 221 | var response []*sentryEvent 222 | err = json.Unmarshal(body, &response) 223 | if err != nil { 224 | return prettyError(err) 225 | } 226 | 227 | debug("Response: %v", response) 228 | debug("Result count: %d", len(response)) 229 | 230 | time.Sleep(30 * time.Second) 231 | 232 | events, err := matchEvents( 233 | response, 234 | m.message, 235 | convertStringToTags(m.tags), 236 | m.regexp, 237 | ) 238 | 239 | if err != nil { 240 | return prettyError(err) 241 | } 242 | 243 | debug("Matched events: %d", len(events)) 244 | 245 | if len(events) > int(monitor.expectedResultCount) { 246 | ticker.Stop() 247 | 248 | fmt.Fprintf(m.out, "Failure detected, rolling back...\n") 249 | 250 | _, err := m.client.RollbackRelease( 251 | m.name, 252 | helm.RollbackDryRun(monitor.dryRun), 253 | helm.RollbackRecreate(false), 254 | helm.RollbackForce(monitor.force), 255 | helm.RollbackDisableHooks(monitor.disableHooks), 256 | helm.RollbackVersion(0), 257 | helm.RollbackTimeout(monitor.rollbackTimeout), 258 | helm.RollbackWait(monitor.wait)) 259 | 260 | if err != nil { 261 | return prettyError(err) 262 | } 263 | 264 | fmt.Fprintf(m.out, "Successfully rolled back to previous revision!\n") 265 | return nil 266 | } 267 | 268 | case <-quit: 269 | ticker.Stop() 270 | debug("Quitting...") 271 | return nil 272 | } 273 | } 274 | } 275 | -------------------------------------------------------------------------------- /cmd/monitor_sentry_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/davecgh/go-spew/spew" 9 | ) 10 | 11 | func TestConvertStringToTags(t *testing.T) { 12 | for _, test := range []struct { 13 | name string 14 | input []string 15 | expected []*tag 16 | }{ 17 | { 18 | name: "it should convert a list of string into tags", 19 | input: []string{"key1=value1", "key2=value2"}, 20 | expected: []*tag{ 21 | &tag{Key: "key1", Value: "value1"}, 22 | &tag{Key: "key2", Value: "value2"}, 23 | }, 24 | }, 25 | { 26 | name: "it should not convert wrongly formatted tags", 27 | input: []string{"key1"}, 28 | expected: []*tag{}, 29 | }, 30 | } { 31 | t.Run(fmt.Sprintf("%s", test.name), func(t *testing.T) { 32 | output := convertStringToTags(test.input) 33 | if !reflect.DeepEqual(test.expected, output) { 34 | t.Errorf( 35 | "\ngiven %v\nexpected: %v\ngot: %v\n", 36 | spew.Sdump(test.input), 37 | spew.Sdump(test.expected), 38 | spew.Sdump(output), 39 | ) 40 | } 41 | }) 42 | } 43 | } 44 | 45 | type matchTagsInput struct { 46 | tagList []*tag 47 | matchTagList []*tag 48 | } 49 | 50 | func TestMatchTags(t *testing.T) { 51 | for _, test := range []struct { 52 | name string 53 | input matchTagsInput 54 | expected bool 55 | }{ 56 | { 57 | name: "it should return true if the tags matches", 58 | input: matchTagsInput{ 59 | tagList: []*tag{ 60 | &tag{Key: "key2", Value: "value2"}, 61 | }, 62 | matchTagList: []*tag{ 63 | &tag{Key: "key1", Value: "value1"}, 64 | &tag{Key: "key2", Value: "value2"}, 65 | &tag{Key: "key3", Value: "value3"}, 66 | }, 67 | }, 68 | expected: true, 69 | }, 70 | { 71 | name: "it should return false if not tags matches", 72 | input: matchTagsInput{ 73 | tagList: []*tag{ 74 | &tag{Key: "key2", Value: "value2"}, 75 | }, 76 | matchTagList: []*tag{ 77 | &tag{Key: "key1", Value: "value1"}, 78 | &tag{Key: "key3", Value: "value3"}, 79 | }, 80 | }, 81 | expected: false, 82 | }, 83 | } { 84 | t.Run(fmt.Sprintf("%s", test.name), func(t *testing.T) { 85 | output := matchTags(test.input.tagList, test.input.matchTagList) 86 | if !reflect.DeepEqual(test.expected, output) { 87 | t.Errorf( 88 | "\ngiven %v\nexpected: %v\ngot: %v\n", 89 | spew.Sdump(test.input), 90 | spew.Sdump(test.expected), 91 | spew.Sdump(output), 92 | ) 93 | } 94 | }) 95 | } 96 | } 97 | 98 | type matchEventsInput struct { 99 | eventList []*sentryEvent 100 | message string 101 | tagList []*tag 102 | useRegexp bool 103 | } 104 | 105 | func TestMatchEvents(t *testing.T) { 106 | for _, test := range []struct { 107 | name string 108 | input matchEventsInput 109 | expected []*sentryEvent 110 | }{ 111 | { 112 | name: "it should match events by message and tags", 113 | input: matchEventsInput{ 114 | message: "This is an event", 115 | tagList: []*tag{ 116 | &tag{Key: "key2", Value: "value2"}, 117 | &tag{Key: "key3", Value: "value3"}, 118 | }, 119 | eventList: []*sentryEvent{ 120 | &sentryEvent{ 121 | Message: "This is an event", 122 | Tags: []*tag{ 123 | &tag{Key: "key1", Value: "value1"}, 124 | }, 125 | }, 126 | &sentryEvent{ 127 | Message: "This is an event", 128 | Tags: []*tag{ 129 | &tag{Key: "key1", Value: "value1"}, 130 | &tag{Key: "key2", Value: "value2"}, 131 | &tag{Key: "key3", Value: "value3"}, 132 | }, 133 | }, 134 | &sentryEvent{ 135 | Message: "", 136 | Tags: []*tag{ 137 | &tag{Key: "key2", Value: "value2"}, 138 | &tag{Key: "key3", Value: "value3"}, 139 | }, 140 | }, 141 | &sentryEvent{ 142 | Message: "This is an event", 143 | Tags: []*tag{ 144 | &tag{Key: "key2", Value: "value2"}, 145 | &tag{Key: "key3", Value: "value3"}, 146 | }, 147 | }, 148 | }, 149 | }, 150 | expected: []*sentryEvent{ 151 | &sentryEvent{ 152 | Message: "This is an event", 153 | Tags: []*tag{ 154 | &tag{Key: "key1", Value: "value1"}, 155 | &tag{Key: "key2", Value: "value2"}, 156 | &tag{Key: "key3", Value: "value3"}, 157 | }, 158 | }, 159 | &sentryEvent{ 160 | Message: "This is an event", 161 | Tags: []*tag{ 162 | &tag{Key: "key2", Value: "value2"}, 163 | &tag{Key: "key3", Value: "value3"}, 164 | }, 165 | }, 166 | }, 167 | }, 168 | { 169 | name: "it should match events by regular expression", 170 | input: matchEventsInput{ 171 | eventList: []*sentryEvent{ 172 | &sentryEvent{Message: "This is a first event"}, 173 | &sentryEvent{Message: "This is a second event"}, 174 | &sentryEvent{Message: "This is a third event"}, 175 | }, 176 | message: "first|second", 177 | useRegexp: true, 178 | }, 179 | expected: []*sentryEvent{ 180 | &sentryEvent{Message: "This is a first event"}, 181 | &sentryEvent{Message: "This is a second event"}, 182 | }, 183 | }, 184 | { 185 | name: "it should match all events if message and tag list is not provided", 186 | input: matchEventsInput{ 187 | eventList: []*sentryEvent{ 188 | &sentryEvent{Message: "This is a first event"}, 189 | &sentryEvent{Message: "This is a second event"}, 190 | &sentryEvent{Message: "This is a third event"}, 191 | }, 192 | message: "", 193 | useRegexp: false, 194 | }, 195 | expected: []*sentryEvent{ 196 | &sentryEvent{Message: "This is a first event"}, 197 | &sentryEvent{Message: "This is a second event"}, 198 | &sentryEvent{Message: "This is a third event"}, 199 | }, 200 | }, 201 | } { 202 | t.Run(fmt.Sprintf("%s", test.name), func(t *testing.T) { 203 | output, err := matchEvents( 204 | test.input.eventList, 205 | test.input.message, 206 | test.input.tagList, 207 | test.input.useRegexp, 208 | ) 209 | if !reflect.DeepEqual(test.expected, output) || err != nil { 210 | t.Errorf("\ngiven: \n%v\nexpected: \n%v\ngot: \n%v", 211 | spew.Sdump(test.input), 212 | spew.Sdump(test.expected), 213 | spew.Sdump(output), 214 | ) 215 | } 216 | }) 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | PROMETHEUS_VERSION=7.0.2 2 | 3 | .PHONY: build 4 | build: 5 | eval $$(minikube docker-env) && \ 6 | docker build \ 7 | --build-arg "LDFLAGS=-X main.version=1.0.0" \ 8 | -t my-app:1.0.0 \ 9 | app && \ 10 | docker build \ 11 | --build-arg "LDFLAGS=-X main.version=2.0.0" \ 12 | -t my-app:2.0.0 \ 13 | app 14 | 15 | .PHONY: deployv1 16 | deployv1: 17 | helm upgrade -i my-app --set image.tag=1.0.0 ./app/charts 18 | 19 | .PHONY: deployv2 20 | deployv2: 21 | helm upgrade -i my-app --set image.tag=2.0.0 ./app/charts 22 | 23 | .PHONY: installprometheus 24 | installprometheus: 25 | helm upgrade -i \ 26 | --version $(PROMETHEUS_VERSION) \ 27 | --set server.service.type=LoadBalancer \ 28 | --set server.global.scrape_interval=30s \ 29 | --set alertmanager.enabled=false \ 30 | --set kubeStateMetrics.enabled=false \ 31 | --set nodeExporter.enabled=false \ 32 | --set pushgateway.enabled=false \ 33 | prometheus \ 34 | stable/prometheus 35 | 36 | .PHONY: cleanup 37 | helm del --purge prometheus my-app 38 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | Helm monitor example 2 | ==================== 3 | 4 | > This example demonstrate how to use helm-monitor to rollback a Helm release 5 | based on events. 6 | 7 | ## Prepare 8 | 9 | Prepare you environment (Minikube, Tiller and build the required images): 10 | 11 | ```bash 12 | # initialise Tiller with RBAC 13 | $ kubectl create serviceaccount tiller -n kube-system 14 | $ kubectl create clusterrolebinding tiller --clusterrole=cluster-admin --serviceaccount=kube-system:tiller 15 | $ helm init --wait 16 | 17 | # build the application for Minikube 18 | $ make build 19 | 20 | # release version 1 21 | $ helm upgrade -i my-app --set image.tag=1.0.0 ./app/charts 22 | 23 | # access the application 24 | $ minikube service my-app 25 | ``` 26 | 27 | ## Follow the monitoring procedure from the example below 28 | 29 | - [rollback based on Prometheus](prometheus.md) 30 | - [rollback based on Elasticsearch](elasticsearch.md) 31 | - [rollback based on Sentry](sentry.md) 32 | -------------------------------------------------------------------------------- /examples/app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.11 AS build 2 | ENV GOPATH="" 3 | ENV GO111MODULE=on 4 | ARG LDFLAGS 5 | COPY . /go 6 | RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags "$LDFLAGS" -o app . 7 | 8 | FROM scratch 9 | COPY --from=build /go/app /app 10 | EXPOSE 8080 11 | CMD ["/app"] 12 | -------------------------------------------------------------------------------- /examples/app/charts/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | -------------------------------------------------------------------------------- /examples/app/charts/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: "1.0" 3 | description: A Helm chart for Kubernetes 4 | name: my-app 5 | version: 0.1.0 6 | -------------------------------------------------------------------------------- /examples/app/charts/templates/NOTES.txt: -------------------------------------------------------------------------------- 1 | 1. Get the application URL by running these commands: 2 | {{- if .Values.ingress.enabled }} 3 | {{- range .Values.ingress.hosts }} 4 | http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }} 5 | {{- end }} 6 | {{- else if contains "NodePort" .Values.service.type }} 7 | export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "my-app.fullname" . }}) 8 | export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") 9 | echo http://$NODE_IP:$NODE_PORT 10 | {{- else if contains "LoadBalancer" .Values.service.type }} 11 | NOTE: It may take a few minutes for the LoadBalancer IP to be available. 12 | You can watch the status of by running 'kubectl get svc -w {{ include "my-app.fullname" . }}' 13 | export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "my-app.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') 14 | echo http://$SERVICE_IP:{{ .Values.service.port }} 15 | {{- else if contains "ClusterIP" .Values.service.type }} 16 | export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ include "my-app.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") 17 | echo "Visit http://127.0.0.1:8080 to use your application" 18 | kubectl port-forward $POD_NAME 8080:80 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /examples/app/charts/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "my-app.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "my-app.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "my-app.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | -------------------------------------------------------------------------------- /examples/app/charts/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1beta2 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "my-app.fullname" . }} 5 | labels: 6 | app: {{ include "my-app.name" . }} 7 | chart: {{ include "my-app.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | spec: 11 | replicas: {{ .Values.replicaCount }} 12 | selector: 13 | matchLabels: 14 | app: {{ include "my-app.name" . }} 15 | release: {{ .Release.Name }} 16 | template: 17 | metadata: 18 | labels: 19 | app: {{ include "my-app.name" . }} 20 | release: {{ .Release.Name }} 21 | annotations: 22 | prometheus.io/scrape: "true" 23 | prometheus.io/port: "9090" 24 | spec: 25 | containers: 26 | - name: {{ .Chart.Name }} 27 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 28 | imagePullPolicy: {{ .Values.image.pullPolicy }} 29 | env: 30 | {{- range $key, $value := .Values.env }} 31 | - name: {{ $key }} 32 | value: {{ $value | quote }} 33 | {{- end }} 34 | ports: 35 | - name: http 36 | containerPort: 8080 37 | protocol: TCP 38 | - name: probes 39 | containerPort: 8086 40 | protocol: TCP 41 | - name: metrics 42 | containerPort: 9090 43 | protocol: TCP 44 | livenessProbe: 45 | httpGet: 46 | path: /live 47 | port: probes 48 | readinessProbe: 49 | httpGet: 50 | path: /ready 51 | port: probes 52 | resources: 53 | {{ toYaml .Values.resources | indent 12 }} 54 | {{- with .Values.nodeSelector }} 55 | nodeSelector: 56 | {{ toYaml . | indent 8 }} 57 | {{- end }} 58 | {{- with .Values.affinity }} 59 | affinity: 60 | {{ toYaml . | indent 8 }} 61 | {{- end }} 62 | {{- with .Values.tolerations }} 63 | tolerations: 64 | {{ toYaml . | indent 8 }} 65 | {{- end }} 66 | -------------------------------------------------------------------------------- /examples/app/charts/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "my-app.fullname" . -}} 3 | {{- $ingressPath := .Values.ingress.path -}} 4 | apiVersion: extensions/v1beta1 5 | kind: Ingress 6 | metadata: 7 | name: {{ $fullName }} 8 | labels: 9 | app: {{ include "my-app.name" . }} 10 | chart: {{ include "my-app.chart" . }} 11 | release: {{ .Release.Name }} 12 | heritage: {{ .Release.Service }} 13 | {{- with .Values.ingress.annotations }} 14 | annotations: 15 | {{ toYaml . | indent 4 }} 16 | {{- end }} 17 | spec: 18 | {{- if .Values.ingress.tls }} 19 | tls: 20 | {{- range .Values.ingress.tls }} 21 | - hosts: 22 | {{- range .hosts }} 23 | - {{ . | quote }} 24 | {{- end }} 25 | secretName: {{ .secretName }} 26 | {{- end }} 27 | {{- end }} 28 | rules: 29 | {{- range .Values.ingress.hosts }} 30 | - host: {{ . | quote }} 31 | http: 32 | paths: 33 | - path: {{ $ingressPath }} 34 | backend: 35 | serviceName: {{ $fullName }} 36 | servicePort: http 37 | {{- end }} 38 | {{- end }} 39 | -------------------------------------------------------------------------------- /examples/app/charts/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "my-app.fullname" . }} 5 | labels: 6 | app: {{ include "my-app.name" . }} 7 | chart: {{ include "my-app.chart" . }} 8 | release: {{ .Release.Name }} 9 | heritage: {{ .Release.Service }} 10 | spec: 11 | type: {{ .Values.service.type }} 12 | ports: 13 | - port: {{ .Values.service.port }} 14 | targetPort: http 15 | protocol: TCP 16 | name: http 17 | selector: 18 | app: {{ include "my-app.name" . }} 19 | release: {{ .Release.Name }} 20 | -------------------------------------------------------------------------------- /examples/app/charts/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for charts. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | replicaCount: 1 6 | 7 | # Sentry DSN to send event to, ie: https://:@sentry.io/ 8 | env: 9 | # SENTRY_DSN: 10 | # SENTRY_RELEASE: 11 | # SENTRY_ENVIRONMENT: 12 | 13 | image: 14 | repository: my-app 15 | tag: latest 16 | pullPolicy: IfNotPresent 17 | 18 | nameOverride: "" 19 | fullnameOverride: my-app 20 | 21 | service: 22 | type: NodePort 23 | port: 80 24 | 25 | ingress: 26 | enabled: false 27 | annotations: {} 28 | # kubernetes.io/ingress.class: nginx 29 | # kubernetes.io/tls-acme: "true" 30 | path: / 31 | hosts: 32 | - chart-example.local 33 | tls: [] 34 | # - secretName: chart-example-tls 35 | # hosts: 36 | # - chart-example.local 37 | 38 | resources: {} 39 | # We usually recommend not to specify default resources and to leave this as a conscious 40 | # choice for the user. This also increases chances charts run on environments with little 41 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 42 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 43 | # limits: 44 | # cpu: 100m 45 | # memory: 128Mi 46 | # requests: 47 | # cpu: 100m 48 | # memory: 128Mi 49 | 50 | nodeSelector: {} 51 | 52 | tolerations: [] 53 | 54 | affinity: {} 55 | -------------------------------------------------------------------------------- /examples/app/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ContainerSolutions/helm-monitor/examples/app 2 | 3 | require ( 4 | github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a // indirect 5 | github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2 // indirect 6 | github.com/getsentry/raven-go v0.0.0-20180903072508-084a9de9eb03 7 | github.com/golang/protobuf v1.0.0 // indirect 8 | github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f // indirect 9 | github.com/gorilla/mux v1.6.2 10 | github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 11 | github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da 12 | github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect 13 | github.com/pkg/errors v0.8.0 // indirect 14 | github.com/prometheus/client_golang v0.9.0-pre1 15 | github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect 16 | github.com/prometheus/common v0.0.0-20180110214958-89604d197083 // indirect 17 | github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 // indirect 18 | github.com/rs/xid v0.0.0-20170604230408-02dd45c33376 // indirect 19 | github.com/rs/zerolog v1.8.0 20 | github.com/stretchr/testify v1.3.0 // indirect 21 | github.com/zenazn/goji v0.0.0-20160507202103-64eb34159fe5 // indirect 22 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /examples/app/go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c= 2 | github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 3 | github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2 h1:MmeatFT1pTPSVb4nkPmBFN/LRZ97vPjsFKsZrU3KKTs= 4 | github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= 5 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 6 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/getsentry/raven-go v0.0.0-20180903072508-084a9de9eb03 h1:G/9fPivTr5EiyqE9OlW65iMRUxFXMGRHgZFGo50uG8Q= 8 | github.com/getsentry/raven-go v0.0.0-20180903072508-084a9de9eb03/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= 9 | github.com/golang/protobuf v1.0.0 h1:lsek0oXi8iFE9L+EXARyHIjU5rlWIhhTkjDz3vHhWWQ= 10 | github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 11 | github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk= 12 | github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 13 | github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= 14 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 15 | github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= 16 | github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= 17 | github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da h1:5y58+OCjoHCYB8182mpf/dEsq0vwTKPOo4zGfH0xW9A= 18 | github.com/justinas/alice v0.0.0-20171023064455-03f45bd4b7da/go.mod h1:oLH0CmIaxCGXD67VKGR5AacGXZSMznlmeqM8RzPrcY8= 19 | github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc= 20 | github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 21 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 22 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 23 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 24 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 25 | github.com/prometheus/client_golang v0.9.0-pre1 h1:AWTOhsOI9qxeirTuA0A4By/1Es1+y9EcCGY6bBZ2fhM= 26 | github.com/prometheus/client_golang v0.9.0-pre1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 27 | github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4= 28 | github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 29 | github.com/prometheus/common v0.0.0-20180110214958-89604d197083 h1:BVsJT8+ZbyuL3hypz/HmEiM8h2P6hBQGig4el9/MdjA= 30 | github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 31 | github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 h1:hhvfGDVThBnd4kYisSFmYuHYeUhglxcwag7FhVPH9zM= 32 | github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 33 | github.com/rs/xid v0.0.0-20170604230408-02dd45c33376 h1:pisBoZ1sLLFc+g7EZflpvatXVqmQKv8EjPP8/radknQ= 34 | github.com/rs/xid v0.0.0-20170604230408-02dd45c33376/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 35 | github.com/rs/zerolog v1.8.0 h1:Oglcb4i6h42uWacEjomB2MI8gfkwCwTMFaDY3+Vgj5k= 36 | github.com/rs/zerolog v1.8.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 39 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 40 | github.com/zenazn/goji v0.0.0-20160507202103-64eb34159fe5 h1:u8oGm2Ef+uUdJIbBXJvdPqKeo1u8NPGMtWH521eW2xA= 41 | github.com/zenazn/goji v0.0.0-20160507202103-64eb34159fe5/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= 42 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= 43 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 44 | -------------------------------------------------------------------------------- /examples/app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/heptiolabs/healthcheck" 15 | "github.com/justinas/alice" 16 | 17 | // monitoring 18 | "github.com/prometheus/client_golang/prometheus" 19 | "github.com/prometheus/client_golang/prometheus/promhttp" 20 | 21 | // logging 22 | "github.com/rs/zerolog" 23 | "github.com/rs/zerolog/hlog" 24 | "github.com/rs/zerolog/log" 25 | 26 | // events 27 | raven "github.com/getsentry/raven-go" 28 | ) 29 | 30 | var ( 31 | addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.") 32 | probeAddr = flag.String("probe-address", ":8086", "The address to listen on for probe requests.") 33 | metricsAddr = flag.String("metrics-address", ":9090", "The address to listen on for Prometheus metrics requests.") 34 | 35 | inFlightGauge = prometheus.NewGauge( 36 | prometheus.GaugeOpts{ 37 | Name: "in_flight_requests", 38 | Help: "A gauge of requests currently being served by the wrapped handler.", 39 | }, 40 | ) 41 | 42 | counter = prometheus.NewCounterVec( 43 | prometheus.CounterOpts{ 44 | Name: "http_requests_total", 45 | Help: "A counter for requests to the wrapped handler.", 46 | ConstLabels: map[string]string{ 47 | "version": version, 48 | }, 49 | }, 50 | []string{"code", "method"}, 51 | ) 52 | 53 | duration = prometheus.NewHistogramVec( 54 | prometheus.HistogramOpts{ 55 | Name: "request_duration_seconds", 56 | Help: "A histogram of latencies for requests.", 57 | Buckets: []float64{.25, .5, 1, 2.5, 5, 10}, 58 | ConstLabels: map[string]string{ 59 | "version": version, 60 | }, 61 | }, 62 | []string{"code", "method"}, 63 | ) 64 | 65 | responseSize = prometheus.NewHistogramVec( 66 | prometheus.HistogramOpts{ 67 | Name: "response_size_bytes", 68 | Help: "A histogram of response sizes for requests.", 69 | Buckets: []float64{200, 500, 900, 1500}, 70 | ConstLabels: map[string]string{ 71 | "version": version, 72 | }, 73 | }, 74 | []string{"code", "method"}, 75 | ) 76 | 77 | version string 78 | ) 79 | 80 | func init() { 81 | prometheus.MustRegister(inFlightGauge, counter, duration, responseSize) 82 | } 83 | 84 | func main() { 85 | flag.Parse() 86 | 87 | // logs 88 | log := zerolog.New(os.Stdout).With(). 89 | Timestamp(). 90 | Str("version", version). 91 | Logger() 92 | 93 | // graceful shutdown 94 | quit := make(chan os.Signal) 95 | signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) 96 | 97 | // probes 98 | health := healthcheck.NewHandler() 99 | 100 | // router 101 | r := mux.NewRouter() 102 | c := alice.New(hlog.NewHandler(log), hlog.AccessHandler(accessLogger)) 103 | 104 | s := r.Methods("GET").Subrouter() 105 | s.HandleFunc("/", httpSuccessHandler) 106 | s.HandleFunc("/internal-error", httpErrorHandler) 107 | 108 | srv := &http.Server{Addr: *addr, Handler: c.Then(promRequestHandler(r))} 109 | 110 | go serveMetrics(*metricsAddr) 111 | go serveHTTP(srv) 112 | go serveProbe(*probeAddr, health) 113 | 114 | <-quit 115 | 116 | log.Info().Msg("Shutting down server...") 117 | 118 | // Gracefully shutdown connections 119 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 120 | defer cancel() 121 | 122 | srv.Shutdown(ctx) 123 | } 124 | 125 | func promRequestHandler(handler http.Handler) http.Handler { 126 | return promhttp.InstrumentHandlerInFlight(inFlightGauge, 127 | promhttp.InstrumentHandlerDuration(duration, 128 | promhttp.InstrumentHandlerCounter(counter, 129 | promhttp.InstrumentHandlerResponseSize(responseSize, handler), 130 | ), 131 | ), 132 | ) 133 | } 134 | 135 | func accessLogger(r *http.Request, status, size int, dur time.Duration) { 136 | hlog.FromRequest(r).Info(). 137 | Str("host", r.Host). 138 | Int("status", status). 139 | Int("size", size). 140 | Dur("duration_ms", dur). 141 | Msg("request") 142 | } 143 | 144 | func serveHTTP(srv *http.Server) { 145 | log.Info().Msgf("Server started at %s", srv.Addr) 146 | err := srv.ListenAndServe() 147 | 148 | if err != http.ErrServerClosed { 149 | log.Fatal().Err(err).Msgf("Listen: %s\n", err) 150 | } 151 | } 152 | 153 | func serveProbe(addr string, health healthcheck.Handler) { 154 | log.Info().Msgf("Probe server running at %s", *probeAddr) 155 | http.ListenAndServe(addr, health) 156 | } 157 | 158 | func serveMetrics(addr string) { 159 | log.Info().Msgf("Serving Prometheus metrics on port %s", addr) 160 | 161 | http.Handle("/metrics", promhttp.Handler()) 162 | 163 | if err := http.ListenAndServe(addr, nil); err != nil { 164 | log.Error().Err(err).Msg("Starting Prometheus listener failed") 165 | } 166 | } 167 | 168 | func httpSuccessHandler(w http.ResponseWriter, r *http.Request) { 169 | hostname, err := os.Hostname() 170 | 171 | if err != nil { 172 | fmt.Fprintf(w, "Error getting hostname\n") 173 | return 174 | } 175 | 176 | fmt.Fprintf(w, "Host: %s, Version: %s, Status: Success\n", hostname, version) 177 | } 178 | 179 | func httpErrorHandler(w http.ResponseWriter, r *http.Request) { 180 | hostname, err := os.Hostname() 181 | 182 | if err != nil { 183 | fmt.Fprintf(w, "Error getting hostname\n") 184 | return 185 | } 186 | 187 | w.WriteHeader(http.StatusInternalServerError) 188 | fmt.Fprintf(w, "Host: %s, Version: %s, Status: Internal Service Error\n", hostname, version) 189 | raven.CaptureError(fmt.Errorf("Error triggered, version: %v", version), nil) 190 | } 191 | -------------------------------------------------------------------------------- /examples/elasticsearch-query.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": { 3 | "bool": { 4 | "must": [ 5 | { 6 | "query_string": { 7 | "all_fields": true, 8 | "analyze_wildcard": true, 9 | "query": "status:500 AND kubernetes.labels.app:app AND version:2.0.0" 10 | } 11 | } 12 | ] 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/elasticsearch.md: -------------------------------------------------------------------------------- 1 | Rollback based on an Elasticsearch query 2 | ======================================== 3 | 4 | > Use helm-monitor to rollback a release based on an Elasticsearch query 5 | 6 | ## Prepare 7 | 8 | Make sure to follow the steps described in [README.md](README.md) in order to 9 | start Minikube with Tiller installed and pre-build the application. 10 | 11 | Minikube support the EFK stack via addons, to enable it: 12 | 13 | ```bash 14 | $ minikube addons enable efk 15 | ``` 16 | 17 | If Minikube was already running, you might need to restart it in order to have 18 | the EFK stack up and running: 19 | 20 | ```bash 21 | $ minikube stop 22 | $ minikube start 23 | ``` 24 | 25 | Access Kibana: 26 | 27 | ```bash 28 | $ minikube service kibana-logging -n kube-system 29 | ``` 30 | 31 | ## Upgrade and monitor 32 | 33 | ```bash 34 | # port forward elasticsearch port locally 35 | $ kubectl port-forward -n kube-system $(kubectl get po -n kube-system -l k8s-app=elasticsearch-logging -o jsonpath="{.items[0].metadata.name}") 9200 36 | 37 | # release version 2 38 | $ helm upgrade -i my-app --set image.tag=2.0.0 ./app/charts 39 | 40 | # monitor 41 | $ helm monitor elasticsearch my-app ./elasticsearch-query.json 42 | 43 | # or via Lucene query 44 | $ helm monitor elasticsearch my-app "status:500 AND kubernetes.labels.app:app AND version:2.0.0" 45 | ``` 46 | 47 | Simulate internal server failure: 48 | 49 | ```bash 50 | $ curl $(minikube service my-app --url)internal-error 51 | ``` 52 | 53 | ## Cleanup 54 | 55 | Delete my-app Helm releases: 56 | 57 | ```bash 58 | $ helm del --purge my-app 59 | ``` 60 | -------------------------------------------------------------------------------- /examples/prometheus.md: -------------------------------------------------------------------------------- 1 | Rollback based on a Prometheus query 2 | ==================================== 3 | 4 | > Use helm-monitor to rollback a release based on a Prometheus query. In this 5 | example we run a Prometheus instance and a GoLang application in Minikube, 6 | upgrade then monitor for HTTP failure. If the amount of 5xx reach a certain 7 | limit, then the application get automatically rolled back to its previous state. 8 | 9 | ## Prepare 10 | 11 | Make sure to follow the steps described in [README.md](README.md) in order to 12 | start Minikube with Tiller installed and pre-build the application. 13 | 14 | Install Prometheus: 15 | 16 | ```bash 17 | $ helm install \ 18 | --version 7.0.2 \ 19 | --set server.service.type=LoadBalancer \ 20 | --set server.global.scrape_interval=30s \ 21 | --set alertmanager.enabled=false \ 22 | --set kubeStateMetrics.enabled=false \ 23 | --set nodeExporter.enabled=false \ 24 | --set pushgateway.enabled=false \ 25 | --name prometheus \ 26 | stable/prometheus 27 | ``` 28 | 29 | Access Prometheus: 30 | 31 | ```bash 32 | $ minikube service prometheus 33 | ``` 34 | 35 | ### Upgrade and monitor 36 | 37 | ```bash 38 | # get Prometheus endpoint 39 | $ prometheus=$(minikube service prometheus-server --url) 40 | 41 | # release version 2 42 | $ helm upgrade -i my-app --set image.tag=2.0.0 ./app/charts 43 | 44 | # monitor 45 | $ helm monitor prometheus my-app --prometheus $prometheus 'rate(http_requests_total{code=~"^5.*$",version="2.0.0"}[5m]) > 0' 46 | ``` 47 | 48 | In a new terminal, simulate internal server failure: 49 | 50 | ```bash 51 | $ app=$(minikube service my-app --url) 52 | $ while sleep 1; do curl "$app"/internal-error; done 53 | ``` 54 | 55 | ## Cleanup 56 | 57 | Delete Prometheus and my-app Helm releases: 58 | 59 | ```bash 60 | $ helm del --purge prometheus my-app 61 | ``` 62 | -------------------------------------------------------------------------------- /examples/sentry.md: -------------------------------------------------------------------------------- 1 | Rollback based on Sentry events 2 | =============================== 3 | 4 | > Use helm-monitor to rollback a release based on Sentry events 5 | 6 | ## Prepare 7 | 8 | Make sure to follow the steps described in [README.md](README.md) in order to 9 | start Minikube with Tiller installed and pre-build the application. 10 | 11 | Install Sentry, increase the timeout to let Tiller execute the chart hooks: 12 | 13 | ```bash 14 | $ helm install --version 0.4.1 --name sentry stable/sentry --timeout 3200 15 | $ minikube service sentry-sentry 16 | ``` 17 | 18 | Open the Sentry UI and configure a new project called "my-project". 19 | 20 | Install the example application with the DSN: 21 | ```bash 22 | $ helm upgrade -i my-app \ 23 | --set image.tag=1.0.0 \ 24 | --set env.SENTRY_DSN= \ 25 | --set env.SENTRY_RELEASE=1.0.0 \ 26 | ./app/charts 27 | ``` 28 | 29 | ### Upgrade and monitor 30 | 31 | ```bash 32 | # get the Sentry endpoint 33 | $ sentry=$(minikube service sentry-sentry --url) 34 | 35 | # release version 2 36 | $ helm upgrade -i my-app \ 37 | --set image.tag=2.0.0 \ 38 | --set env.SENTRY_DSN= \ 39 | --set env.SENTRY_RELEASE=2.0.0 \ 40 | ./app/charts 41 | 42 | # monitor 43 | $ helm monitor sentry my-app \ 44 | --api-key \ 45 | --organization sentry \ 46 | --project my-project \ 47 | --sentry $sentry \ 48 | --tag release=2.0.0 \ 49 | 'Error triggered' 50 | ``` 51 | 52 | In a new terminal, simulate internal server failure: 53 | ```bash 54 | $ curl $(minikube service my-app --url)/internal-error 55 | ``` 56 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ContainerSolutions/helm-monitor 2 | 3 | require ( 4 | github.com/BurntSushi/toml v0.3.0 // indirect 5 | github.com/Masterminds/goutils v1.1.0 // indirect 6 | github.com/Masterminds/semver v1.4.2 // indirect 7 | github.com/Masterminds/sprig v2.18.0+incompatible // indirect 8 | github.com/cyphar/filepath-securejoin v0.2.2 // indirect 9 | github.com/davecgh/go-spew v1.1.1 10 | github.com/ghodss/yaml v1.0.0 // indirect 11 | github.com/gobwas/glob v0.2.3 // indirect 12 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect 13 | github.com/golang/protobuf v1.2.0 // indirect 14 | github.com/google/uuid v1.1.1 // indirect 15 | github.com/huandu/xstrings v1.2.0 // indirect 16 | github.com/imdario/mergo v0.3.7 // indirect 17 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 18 | github.com/pkg/errors v0.8.1 // indirect 19 | github.com/spf13/cobra v0.0.3 20 | github.com/spf13/pflag v1.0.2 // indirect 21 | github.com/stretchr/testify v1.3.0 // indirect 22 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 // indirect 23 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d // indirect 24 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect 25 | golang.org/x/text v0.3.0 // indirect 26 | google.golang.org/genproto v0.0.0-20180831171423-11092d34479b // indirect 27 | google.golang.org/grpc v1.7.2 28 | gopkg.in/yaml.v2 v2.2.1 // indirect 29 | k8s.io/apimachinery v0.0.0-20180619225948-e386b2658ed2 // indirect 30 | k8s.io/client-go v10.0.0+incompatible // indirect 31 | k8s.io/helm v2.13.0+incompatible 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.0 h1:e1/Ivsx3Z0FVTV0NSOv/aVgbUWyQuzj7DDnFblkRvsY= 2 | github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= 4 | github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= 5 | github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= 6 | github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= 7 | github.com/Masterminds/sprig v2.18.0+incompatible h1:QoGhlbC6pter1jxKnjMFxT8EqsLuDE6FEcNbWEpw+lI= 8 | github.com/Masterminds/sprig v2.18.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= 9 | github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= 10 | github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= 15 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 16 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= 17 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 18 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 19 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 20 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 21 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 22 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 23 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 24 | github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= 25 | github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= 26 | github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= 27 | github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= 28 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 29 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 30 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 31 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 32 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 33 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 34 | github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= 35 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 36 | github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= 37 | github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 38 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 39 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 40 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 41 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 42 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 43 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I= 44 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 45 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= 46 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 47 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 48 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 49 | google.golang.org/genproto v0.0.0-20180831171423-11092d34479b h1:lohp5blsw53GBXtLyLNaTXPXS9pJ1tiTw61ZHUoE9Qw= 50 | google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 51 | google.golang.org/grpc v1.7.2 h1:Vw1JtR07h6jezLtFKVRNMq5BGqECN1y9dPSEM5f+f7s= 52 | google.golang.org/grpc v1.7.2/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 53 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 54 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 55 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 56 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 57 | k8s.io/apimachinery v0.0.0-20180619225948-e386b2658ed2 h1:NJEj7o7SKxpURej3uJ1QZJZCeRlRj21EatnCK65nrB4= 58 | k8s.io/apimachinery v0.0.0-20180619225948-e386b2658ed2/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= 59 | k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34= 60 | k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= 61 | k8s.io/helm v2.13.0+incompatible h1:d1WBmGGoVb5VZcmQbysDbXGR0Kh/IXPe1SXldrdu19U= 62 | k8s.io/helm v2.13.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= 63 | -------------------------------------------------------------------------------- /helm-monitor-diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ContainerSolutions/helm-monitor/4de8651a9acea23a0717e26bf728ab9a240070e2/helm-monitor-diagram.jpg -------------------------------------------------------------------------------- /helm-monitor-failure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ContainerSolutions/helm-monitor/4de8651a9acea23a0717e26bf728ab9a240070e2/helm-monitor-failure.png -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | current_version=$(sed -n -e 's/version:[ "]*\([^"]*\).*/\1/p' $(dirname $0)/plugin.yaml) 6 | HELM_MONITOR_VERSION=${HELM_MONITOR_VERSION:-$current_version} 7 | 8 | dir=${HELM_PLUGIN_DIR:-"$(helm home)/plugins/helm-monitor"} 9 | os=$(uname -s | tr '[:upper:]' '[:lower:]') 10 | release_file="helm-monitor_${os}_${HELM_MONITOR_VERSION}.tar.gz" 11 | url="https://github.com/ContainerSolutions/helm-monitor/releases/download/v${HELM_MONITOR_VERSION}/${release_file}" 12 | 13 | mkdir -p $dir 14 | 15 | if command -v wget 16 | then 17 | wget -O ${dir}/${release_file} $url 18 | elif command -v curl; then 19 | curl -L -o ${dir}/${release_file} $url 20 | fi 21 | 22 | tar xvf ${dir}/${release_file} -C $dir 23 | 24 | chmod +x ${dir}/helm-monitor 25 | 26 | rm ${dir}/${release_file} 27 | -------------------------------------------------------------------------------- /plugin.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | name: "monitor" 3 | version: "0.4.0" 4 | usage: "monitor and rollback in case of failure based on metrics or logs" 5 | description: |- 6 | Query at a given interval a Prometheus, Elasticsearch or Sentry instance. A 7 | rollback of the release is initiated if the number of result is positive. 8 | ignoreFlags: false 9 | useTunnel: true 10 | command: "$HELM_PLUGIN_DIR/helm-monitor" 11 | hooks: 12 | install: "$HELM_PLUGIN_DIR/install.sh" 13 | --------------------------------------------------------------------------------