├── .gitignore ├── .travis.yml ├── CHANGELOG ├── LICENSE ├── Makefile ├── Readme.md ├── alerts ├── slo.yaml └── slo_test.yaml ├── cmd └── service-level-operator │ ├── flags.go │ └── main.go ├── deploy ├── Readme.md └── manifests │ ├── deployment.yaml │ ├── prometheus.yaml │ ├── rbac.yaml │ └── service.yaml ├── docker ├── dev │ ├── Dockerfile │ ├── docker-compose.yaml │ └── prometheus.yml └── prod │ └── Dockerfile ├── go.mod ├── go.sum ├── hack └── scripts │ ├── build-binary.sh │ ├── build-image.sh │ ├── integration-test.sh │ ├── k8scodegen.sh │ ├── mockgen.sh │ ├── openapicodegen.sh │ ├── test-alerts.sh │ └── unit-test.sh ├── img └── grafana_graphs1.png ├── mocks ├── doc.go ├── github.com │ └── prometheus │ │ └── client_golang │ │ └── api │ │ └── prometheus │ │ └── v1 │ │ └── API.go ├── service │ ├── output │ │ └── Output.go │ └── sli │ │ └── Retriever.go └── thirdparty.go ├── pkg ├── apis │ └── monitoring │ │ ├── register.go │ │ └── v1alpha1 │ │ ├── doc.go │ │ ├── helpers.go │ │ ├── helpers_test.go │ │ ├── openapi_generated.go │ │ ├── register.go │ │ ├── types.go │ │ └── zz_generated.deepcopy.go ├── k8sautogen │ └── client │ │ └── clientset │ │ └── versioned │ │ ├── clientset.go │ │ ├── doc.go │ │ ├── fake │ │ ├── clientset_generated.go │ │ ├── doc.go │ │ └── register.go │ │ ├── scheme │ │ ├── doc.go │ │ └── register.go │ │ └── typed │ │ └── monitoring │ │ └── v1alpha1 │ │ ├── doc.go │ │ ├── fake │ │ ├── doc.go │ │ ├── fake_monitoring_client.go │ │ └── fake_servicelevel.go │ │ ├── generated_expansion.go │ │ ├── monitoring_client.go │ │ └── servicelevel.go ├── log │ ├── dummy.go │ └── log.go ├── operator │ ├── crd.go │ ├── doc.go │ ├── factory.go │ ├── handler.go │ └── handler_test.go └── service │ ├── client │ ├── kubernetes │ │ ├── factory.go │ │ └── fake.go │ └── prometheus │ │ ├── factory.go │ │ ├── factory_test.go │ │ └── fake.go │ ├── configuration │ ├── configuration.go │ └── configuration_test.go │ ├── kubernetes │ ├── crd.go │ ├── kubernetes.go │ └── servicelevel.go │ ├── metrics │ ├── dummy.go │ ├── metrics.go │ ├── prometheus.go │ └── prometheus_test.go │ ├── output │ ├── factory.go │ ├── middleware.go │ ├── output.go │ ├── prometheus.go │ └── prometheus_test.go │ └── sli │ ├── factory.go │ ├── middleware.go │ ├── prometheus.go │ ├── prometheus_test.go │ ├── sli.go │ └── sli_test.go └── test └── manual └── slos.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Binary 14 | bin/ 15 | 16 | # vendor 17 | vendor/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: required 3 | services: 4 | - docker 5 | go: 6 | - "1.13" 7 | 8 | before_install: 9 | # Upgrade default docker 10 | - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 11 | - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 12 | - sudo apt-get update 13 | - sudo apt-get -y install docker-ce 14 | 15 | script: 16 | - make ci 17 | 18 | deploy: 19 | - provider: script 20 | script: docker login -u="${DOCKER_USER}" -p="${DOCKER_PASSWORD}" quay.io && make push 21 | on: 22 | tags: true 23 | 24 | env: 25 | global: 26 | - GO111MODULE: on 27 | - DOCKER_USER: spotahome+serviceleveloperator 28 | # DOCKER_PASSWORD 29 | - secure: "smTgGZsms3W3cjA2aw34Rr1rNjVIHj9N0CwgpOLWuw3j1Nwb4OdRPdHN8qSc4HpCeIVWUU/rdVr2gsXherok+6vQR5y4Wh4dxTODM/4mVxdY67k7URJra4Vktei0K1hD5E4AYAvUb/YZTlBcex83tIkMaeGSYc+w2PzQHa1R1p4PVVEnsu6+O1Qg4i7xErXLmnSVc5tcqpXRhM/cp/XMNiJPKezRVHD5j73ZzN36xJSkNS/xiHw5nA/B/kpzb/NL/sAvknXNqx8+/S0Y3mIsXId4r8LK3YBjV5ALYZvq0mWc/0vZIZ8Fmcf6J+1LQzT+7O9lUuAFKt9LXMsZbiQuG0YEcqix732IqIXicUSOLA/w9TajLNYXQh05L5mAiejHsFZmtDwCDzRi/wvc4d3NkxJt4YsuUu+2Lsd5DGyyuoz2zlzaVoVRSCYwKTIfLU0XdiQ8ilHpI31BN7TK3z+Eh27MfkbJTldyHB2eV0B6mLWxpYICYf5TzlwHtvLg8FUrJ+W7j9Wk63+0W4pY0gzNMSaZunSEuymHjdn94U01aIjqAjMat6HqvjdAs5mYgLmLqC8clOjMW4fJmhdiSXk0NDEAjG35G3OJ9dsImXgaQrHGMM3SZJAhxPEF+MpH0iUuaqC4ExBQtJFz9sQTOue10zOMVRSOoqZQuGmmj2Uy+zk=" 30 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 2 | ## [Unreleased] 3 | 4 | ## [0.3.0] - 2019-10-25 5 | ### Added 6 | - Default SLI input configurations for the SLOs that don't have SLI inputs. 7 | 8 | ## [0.2.0] - 2018-11-14 9 | ### Added 10 | - Grafana dashboard. 11 | 12 | ### Changed 13 | - Move CRD api from `apiVersion: measure.slok.xyz/v1alpha1` to `apiVersion: monitoring.spotahome.com/v1alpha1`. 14 | - Move repository from github.com/slok/service-level-operator to github.com/spotahome/service-level-operator. 15 | 16 | ## [0.1.0] - 2018-10-31 17 | ### Added 18 | - Prometheus metrics for the SLI processing flow. 19 | - Deploy example manifests for Kubernetes. 20 | - Prometheus SLI inputs and SLI result outputs. 21 | - Operator. 22 | - Service level CRD. 23 | 24 | [Unreleased]: https://github.com/spotahome/service-level-operator/compare/v0.3.0...HEAD 25 | [0.3.0]: https://github.com/spotahome/service-level-operator/compare/v0.2.0...v0.3.0 26 | [0.2.0]: https://github.com/spotahome/service-level-operator/compare/v0.1.0...v0.2.0 27 | [0.1.0]: https://github.com/spotahome/service-level-operator/releases/tag/v0.1.0 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [2018] [Spotahome Ltd.] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Name of this service/application 2 | SERVICE_NAME := service-level-operator 3 | 4 | # Shell to use for running scripts 5 | SHELL := $(shell which bash) 6 | 7 | # Get docker path or an empty string 8 | DOCKER := $(shell command -v docker) 9 | 10 | # Get docker-compose path or an empty string 11 | DOCKER_COMPOSE := $(shell command -v docker-compose) 12 | 13 | # Get the main unix group for the user running make (to be used by docker-compose later) 14 | GID := $(shell id -g) 15 | 16 | # Get the unix user id for the user running make (to be used by docker-compose later) 17 | UID := $(shell id -u) 18 | 19 | # Bash history file for container shell 20 | HISTORY_FILE := ~/.bash_history.$(SERVICE_NAME) 21 | 22 | # Version from Git. 23 | VERSION=$(shell git describe --tags --always) 24 | 25 | # Dev direcotry has all the required dev files. 26 | DEV_DIR := ./docker/dev 27 | 28 | # cmds 29 | UNIT_TEST_CMD := ./hack/scripts/unit-test.sh 30 | INTEGRATION_TEST_CMD := ./hack/scripts/integration-test.sh 31 | TEST_ALERTS_CMD := ./hack/scripts/test-alerts.sh 32 | MOCKS_CMD := ./hack/scripts/mockgen.sh 33 | DOCKER_RUN_CMD := docker run \ 34 | -v ${PWD}:/src \ 35 | --rm -it $(SERVICE_NAME) 36 | DOCKER_ALERTS_TEST_RUN_CMD := docker run \ 37 | -v ${PWD}:/prometheus \ 38 | --entrypoint=${TEST_ALERTS_CMD} \ 39 | --rm -it prom/prometheus 40 | BUILD_BINARY_CMD := VERSION=${VERSION} ./hack/scripts/build-binary.sh 41 | BUILD_IMAGE_CMD := VERSION=${VERSION} ./hack/scripts/build-image.sh 42 | DEBUG_CMD := go run ./cmd/service-level-operator/* --debug 43 | DEV_CMD := $(DEBUG_CMD) --development 44 | FAKE_CMD := $(DEV_CMD) --fake 45 | K8S_CODE_GEN_CMD := ./hack/scripts/k8scodegen.sh 46 | OPENAPI_CODE_GEN_CMD := ./hack/scripts/openapicodegen.sh 47 | DEPS_CMD := GO111MODULE=on go mod tidy && GO111MODULE=on go mod vendor 48 | K8S_VERSION := 1.13.12 49 | SET_K8S_DEPS_CMD := GO111MODULE=on go mod edit \ 50 | -require=k8s.io/apiextensions-apiserver@kubernetes-${K8S_VERSION} \ 51 | -require=k8s.io/client-go@kubernetes-${K8S_VERSION} \ 52 | -require=k8s.io/apimachinery@kubernetes-${K8S_VERSION} \ 53 | -require=k8s.io/api@kubernetes-${K8S_VERSION} \ 54 | -require=k8s.io/kubernetes@v${K8S_VERSION} && \ 55 | $(DEPS_CMD) 56 | 57 | 58 | # The default action of this Makefile is to build the development docker image 59 | default: build 60 | 61 | # Test if the dependencies we need to run this Makefile are installed 62 | .PHONY: deps-development 63 | deps-development: 64 | ifndef DOCKER 65 | @echo "Docker is not available. Please install docker" 66 | @exit 1 67 | endif 68 | ifndef DOCKER_COMPOSE 69 | @echo "docker-compose is not available. Please install docker-compose" 70 | @exit 1 71 | endif 72 | 73 | # Build the development docker images 74 | .PHONY: build 75 | build: 76 | docker build -t $(SERVICE_NAME) --build-arg uid=$(UID) --build-arg gid=$(GID) -f $(DEV_DIR)/Dockerfile . 77 | 78 | # run the development stack. 79 | .PHONY: stack 80 | stack: deps-development 81 | cd $(DEV_DIR) && \ 82 | ( docker-compose -p $(SERVICE_NAME) up --build; \ 83 | docker-compose -p $(SERVICE_NAME) stop; \ 84 | docker-compose -p $(SERVICE_NAME) rm -f; ) 85 | 86 | # Build production stuff. 87 | build-binary: build 88 | $(DOCKER_RUN_CMD) /bin/sh -c '$(BUILD_BINARY_CMD)' 89 | 90 | .PHONY: build-image 91 | build-image: 92 | $(BUILD_IMAGE_CMD) 93 | 94 | # Dependencies stuff. 95 | .PHONY: set-k8s-deps 96 | set-k8s-deps: 97 | $(SET_K8S_DEPS_CMD) 98 | 99 | .PHONY: deps 100 | deps: 101 | $(DEPS_CMD) 102 | 103 | k8s-code-gen: 104 | $(K8S_CODE_GEN_CMD) 105 | 106 | openapi-code-gen: 107 | $(OPENAPI_CODE_GEN_CMD) 108 | 109 | # Test stuff in dev 110 | .PHONY: test-alerts 111 | test-alerts: 112 | $(DOCKER_ALERTS_TEST_RUN_CMD) 113 | .PHONY: unit-test 114 | unit-test: build 115 | $(DOCKER_RUN_CMD) /bin/sh -c '$(UNIT_TEST_CMD)' 116 | .PHONY: integration-test 117 | integration-test: build 118 | $(DOCKER_RUN_CMD) /bin/sh -c '$(INTEGRATION_TEST_CMD)' 119 | .PHONY: test 120 | test: integration-test 121 | .PHONY: test 122 | ci: test test-alerts 123 | 124 | # Mocks stuff in dev 125 | .PHONY: mocks 126 | mocks: build 127 | # FIX: Problem using go mod with vektra/mockery. 128 | #$(DOCKER_RUN_CMD) /bin/sh -c '$(MOCKS_CMD)' 129 | $(MOCKS_CMD) 130 | 131 | .PHONY: dev 132 | dev: 133 | $(DEV_CMD) 134 | 135 | 136 | .PHONY: push 137 | push: export PUSH_IMAGE=true 138 | push: build-image 139 | -------------------------------------------------------------------------------- /alerts/slo.yaml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: slo.rules 3 | rules: 4 | - alert: SLOErrorRateTooFast1h 5 | expr: | 6 | ( 7 | increase(service_level_sli_result_error_ratio_total[1h]) 8 | / 9 | increase(service_level_sli_result_count_total[1h]) 10 | ) > (1 - service_level_slo_objective_ratio) * 0.02 11 | and 12 | ( 13 | increase(service_level_sli_result_error_ratio_total[5m]) 14 | / 15 | increase(service_level_sli_result_count_total[5m]) 16 | ) > (1 - service_level_slo_objective_ratio) * 0.02 17 | 18 | labels: 19 | severity: critical 20 | team: a-team 21 | annotations: 22 | summary: The SLO error budget burn rate for 1h is greater than 2% 23 | description: The error rate for 1h in the {{$labels.service_level}}/{{$labels.slo}} SLO error budget is too fast, is greater than the total error budget 2%. 24 | - alert: SLOErrorRateTooFast6h 25 | expr: | 26 | ( 27 | increase(service_level_sli_result_error_ratio_total[6h]) 28 | / 29 | increase(service_level_sli_result_count_total[6h]) 30 | ) > (1 - service_level_slo_objective_ratio) * 0.05 31 | and 32 | ( 33 | increase(service_level_sli_result_error_ratio_total[30m]) 34 | / 35 | increase(service_level_sli_result_count_total[30m]) 36 | ) > (1 - service_level_slo_objective_ratio) * 0.05 37 | labels: 38 | severity: critical 39 | team: a-team 40 | annotations: 41 | summary: The SLO error budget burn rate for 6h is greater than 5% 42 | description: The error rate for 6h in the {{$labels.service_level}}/{{$labels.slo}} SLO error budget is too fast, is greater than the total error budget 5%. 43 | -------------------------------------------------------------------------------- /alerts/slo_test.yaml: -------------------------------------------------------------------------------- 1 | rule_files: 2 | - slo.yaml 3 | 4 | evaluation_interval: 1m 5 | 6 | tests: 7 | # This test will test the alert triggers with 1h of errors bypassing the error budget. 8 | # count ratio total in 60m = count_total * 60m = xxx 9 | # count ratio total in 60m = 1 * 60m = 60 10 | # Error ratio total in 60m = error_total * 60m = xxx 11 | # Error ratio total in 60m = 0.00021* 60m = 0.0126 12 | # Error ration in 60m = Error ratio total / Count ratio total = 0.0021 13 | # Error budget 2% = (error budget * 2%) = xxx 14 | # Error budget 2% = 0.01 * 00.02 = 0.0002 15 | # Should trigger alert? = 0.0021 > 0.0002 = true 16 | - interval: 1m 17 | input_series: 18 | - series: 'service_level_sli_result_error_ratio_total{service_level="sl1",slo="slo1"}' 19 | values: "0+0.00021x120" 20 | - series: 'service_level_sli_result_count_total{service_level="sl1",slo="slo1"}' 21 | values: "0+1x120" 22 | - series: 'service_level_slo_objective_ratio{service_level="sl1",slo="slo1"}' 23 | values: "0.99+0x120" 24 | 25 | alert_rule_test: 26 | - eval_time: 65m 27 | alertname: SLOErrorRateTooFast1h 28 | exp_alerts: 29 | - exp_labels: 30 | severity: critical 31 | team: a-team 32 | service_level: sl1 33 | slo: slo1 34 | exp_annotations: 35 | summary: "The SLO error budget burn rate for 1h is greater than 2%" 36 | description: "The error rate for 1h in the sl1/slo1 SLO error budget is too fast, is greater than the total error budget 2%." 37 | -------------------------------------------------------------------------------- /cmd/service-level-operator/flags.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "k8s.io/client-go/util/homedir" 10 | 11 | "github.com/spotahome/service-level-operator/pkg/operator" 12 | ) 13 | 14 | // defaults 15 | const ( 16 | defMetricsPath = "/metrics" 17 | defListenAddress = ":8080" 18 | defResyncSeconds = 5 19 | defWorkers = 10 20 | ) 21 | 22 | type cmdFlags struct { 23 | fs *flag.FlagSet 24 | 25 | kubeConfig string 26 | resyncSeconds int 27 | workers int 28 | metricsPath string 29 | listenAddress string 30 | labelSelector string 31 | namespace string 32 | defSLISourcePath string 33 | debug bool 34 | development bool 35 | fake bool 36 | } 37 | 38 | func newCmdFlags() *cmdFlags { 39 | c := &cmdFlags{ 40 | fs: flag.NewFlagSet(os.Args[0], flag.ExitOnError), 41 | } 42 | c.init() 43 | 44 | return c 45 | } 46 | 47 | func (c *cmdFlags) init() { 48 | 49 | kubehome := filepath.Join(homedir.HomeDir(), ".kube", "config") 50 | // register flags 51 | c.fs.StringVar(&c.kubeConfig, "kubeconfig", kubehome, "kubernetes configuration path, only used when development mode enabled") 52 | c.fs.StringVar(&c.metricsPath, "metrics-path", defMetricsPath, "the path where the metrics will be served") 53 | c.fs.StringVar(&c.listenAddress, "listen-addr", defListenAddress, "the address where the metrics will be exposed") 54 | c.fs.StringVar(&c.labelSelector, "selector", "", "selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") 55 | c.fs.StringVar(&c.namespace, "namespace", "", "the namespace to filter on, by default all") 56 | c.fs.StringVar(&c.defSLISourcePath, "def-sli-source-path", "", "the path to the default sli sources configuration file") 57 | c.fs.IntVar(&c.resyncSeconds, "resync-seconds", defResyncSeconds, "the number of seconds for the SLO calculation interval") 58 | c.fs.IntVar(&c.workers, "workers", defWorkers, "the number of concurrent workers per controller handling events") 59 | c.fs.BoolVar(&c.development, "development", false, "development flag will allow to run the operator outside a kubernetes cluster") 60 | c.fs.BoolVar(&c.debug, "debug", false, "enable debug mode") 61 | c.fs.BoolVar(&c.fake, "fake", false, "enable faked mode, in faked node external services/dependencies are not needed") 62 | 63 | // Parse flags 64 | c.fs.Parse(os.Args[1:]) 65 | } 66 | 67 | func (c *cmdFlags) toOperatorConfig() operator.Config { 68 | return operator.Config{ 69 | ResyncPeriod: time.Duration(c.resyncSeconds) * time.Second, 70 | ConcurretWorkers: c.workers, 71 | LabelSelector: c.labelSelector, 72 | Namespace: c.namespace, 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /cmd/service-level-operator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/oklog/run" 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 16 | "k8s.io/client-go/kubernetes" 17 | _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" 18 | "k8s.io/client-go/rest" 19 | "k8s.io/client-go/tools/clientcmd" 20 | 21 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 22 | "github.com/spotahome/service-level-operator/pkg/log" 23 | "github.com/spotahome/service-level-operator/pkg/operator" 24 | kubernetesclifactory "github.com/spotahome/service-level-operator/pkg/service/client/kubernetes" 25 | promclifactory "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 26 | "github.com/spotahome/service-level-operator/pkg/service/configuration" 27 | kubernetesservice "github.com/spotahome/service-level-operator/pkg/service/kubernetes" 28 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 29 | ) 30 | 31 | const ( 32 | kubeCliQPS = 100 33 | kubeCliBurst = 100 34 | gracePeriod = 2 * time.Second 35 | ) 36 | 37 | // Main has the main logic of the app. 38 | type Main struct { 39 | flags *cmdFlags 40 | logger log.Logger 41 | } 42 | 43 | // Run runs the main program. 44 | func (m *Main) Run() error { 45 | // Prepare the logger with the correct settings. 46 | jsonLog := true 47 | if m.flags.development { 48 | jsonLog = false 49 | } 50 | m.logger = log.Base(jsonLog) 51 | if m.flags.debug { 52 | m.logger.Set("debug") 53 | } 54 | 55 | if m.flags.fake { 56 | m.logger = m.logger.With("mode", "fake") 57 | m.logger.Warnf("running in faked mode, any external service will be faked") 58 | } 59 | 60 | // Create prometheus registry and metrics service to expose and measure with metrics. 61 | promReg := prometheus.NewRegistry() 62 | metricssvc := metrics.NewPrometheus(promReg) 63 | 64 | // Create services 65 | k8sstdcli, k8scrdcli, k8saexcli, err := m.createKubernetesClients() 66 | if err != nil { 67 | return err 68 | } 69 | k8ssvc := kubernetesservice.New(k8sstdcli, k8scrdcli, k8saexcli, m.logger) 70 | 71 | // Prepare our run entrypoints. 72 | var g run.Group 73 | 74 | // OS signals. 75 | { 76 | sigC := make(chan os.Signal, 1) 77 | exitC := make(chan struct{}) 78 | signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT) 79 | 80 | g.Add( 81 | func() error { 82 | select { 83 | case s := <-sigC: 84 | m.logger.Infof("signal %s received", s) 85 | return nil 86 | case <-exitC: 87 | return nil 88 | } 89 | }, 90 | func(_ error) { 91 | close(exitC) 92 | }, 93 | ) 94 | } 95 | 96 | // Metrics. 97 | { 98 | s := m.createHTTPServer(promReg) 99 | g.Add( 100 | func() error { 101 | m.logger.Infof("metrics server listening on %s", m.flags.listenAddress) 102 | return s.ListenAndServe() 103 | }, 104 | func(_ error) { 105 | m.logger.Infof("draining metrics server connections") 106 | ctx, cancel := context.WithTimeout(context.Background(), gracePeriod) 107 | defer cancel() 108 | err := s.Shutdown(ctx) 109 | if err != nil { 110 | m.logger.Errorf("error while drainning connections on metrics sever") 111 | } 112 | }, 113 | ) 114 | } 115 | 116 | // Operator. 117 | { 118 | 119 | // Load configuration. 120 | var cfgSLISrc *configuration.DefaultSLISource 121 | if m.flags.defSLISourcePath != "" { 122 | f, err := os.Open(m.flags.defSLISourcePath) 123 | if err != nil { 124 | return err 125 | } 126 | defer f.Close() 127 | cfgSLISrc, err = configuration.JSONLoader{}.LoadDefaultSLISource(context.Background(), f) 128 | if err != nil { 129 | return err 130 | } 131 | } 132 | 133 | // Create SLI source client factories. 134 | promCliFactory, err := m.createPrometheusCliFactory(cfgSLISrc) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | cfg := m.flags.toOperatorConfig() 140 | op, err := operator.New(cfg, promReg, promCliFactory, k8ssvc, metricssvc, m.logger) 141 | if err != nil { 142 | return err 143 | } 144 | closeC := make(chan struct{}) 145 | 146 | g.Add( 147 | func() error { 148 | return op.Run(closeC) 149 | }, 150 | func(_ error) { 151 | close(closeC) 152 | }, 153 | ) 154 | } 155 | 156 | // Run everything 157 | return g.Run() 158 | } 159 | 160 | // loadKubernetesConfig loads kubernetes configuration based on flags. 161 | func (m *Main) loadKubernetesConfig() (*rest.Config, error) { 162 | var cfg *rest.Config 163 | // If devel mode then use configuration flag path. 164 | if m.flags.development { 165 | config, err := clientcmd.BuildConfigFromFlags("", m.flags.kubeConfig) 166 | if err != nil { 167 | return nil, fmt.Errorf("could not load configuration: %s", err) 168 | } 169 | cfg = config 170 | } else { 171 | config, err := rest.InClusterConfig() 172 | if err != nil { 173 | return nil, fmt.Errorf("error loading kubernetes configuration inside cluster, check app is running outside kubernetes cluster or run in development mode: %s", err) 174 | } 175 | cfg = config 176 | } 177 | 178 | // Set better cli rate limiter. 179 | cfg.QPS = kubeCliQPS 180 | cfg.Burst = kubeCliBurst 181 | 182 | return cfg, nil 183 | } 184 | 185 | func (m *Main) createKubernetesClients() (kubernetes.Interface, crdcli.Interface, apiextensionscli.Interface, error) { 186 | var factory kubernetesclifactory.ClientFactory 187 | 188 | if m.flags.fake { 189 | factory = kubernetesclifactory.NewFake() 190 | } else { 191 | config, err := m.loadKubernetesConfig() 192 | if err != nil { 193 | return nil, nil, nil, err 194 | } 195 | factory = kubernetesclifactory.NewFactory(config) 196 | } 197 | 198 | stdcli, err := factory.GetSTDClient() 199 | if err != nil { 200 | return nil, nil, nil, err 201 | } 202 | 203 | crdcli, err := factory.GetCRDClient() 204 | if err != nil { 205 | return nil, nil, nil, err 206 | } 207 | 208 | aexcli, err := factory.GetAPIExtensionClient() 209 | if err != nil { 210 | return nil, nil, nil, err 211 | } 212 | 213 | return stdcli, crdcli, aexcli, nil 214 | } 215 | 216 | func (m *Main) createPrometheusCliFactory(cfg *configuration.DefaultSLISource) (promclifactory.ClientFactory, error) { 217 | if m.flags.fake { 218 | return promclifactory.NewFakeFactory(), nil 219 | } 220 | 221 | f := promclifactory.NewBaseFactory() 222 | if cfg != nil && cfg.Prometheus.Address != "" { 223 | err := f.WithDefaultV1APIClient(cfg.Prometheus.Address) 224 | if err != nil { 225 | return nil, err 226 | } 227 | m.logger.Infof("prometheus default SLI source set to: %s", cfg.Prometheus.Address) 228 | } 229 | 230 | return f, nil 231 | } 232 | 233 | // createHTTPServer creates the http server that serves prometheus metrics and healthchecks. 234 | func (m *Main) createHTTPServer(promReg *prometheus.Registry) http.Server { 235 | h := promhttp.HandlerFor(promReg, promhttp.HandlerOpts{}) 236 | mux := http.NewServeMux() 237 | mux.Handle(m.flags.metricsPath, h) 238 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 239 | w.Write([]byte(` 240 | Service level operator 241 | 242 |

Service level operator

243 |

Metrics

244 | 245 | `)) 246 | }) 247 | mux.HandleFunc("/healthz/ready", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`ready`)) }) 248 | mux.HandleFunc("/healthz/live", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`live`)) }) 249 | 250 | return http.Server{ 251 | Handler: mux, 252 | Addr: m.flags.listenAddress, 253 | } 254 | } 255 | 256 | func main() { 257 | m := &Main{flags: newCmdFlags()} 258 | 259 | // Party time! 260 | err := m.Run() 261 | if err != nil { 262 | fmt.Fprintf(os.Stderr, "error running app: %s", err) 263 | os.Exit(1) 264 | } 265 | 266 | fmt.Fprintf(os.Stdout, "see you soon, good bye!") 267 | os.Exit(0) 268 | } 269 | -------------------------------------------------------------------------------- /deploy/Readme.md: -------------------------------------------------------------------------------- 1 | # Delivery 2 | 3 | In deploy/manifests there are example manifests to deploy this operator. 4 | 5 | - Set the correct namespaces on the manifests. 6 | - Set the correct namespace on the service account. 7 | - If you are using [prometheus-operator] check `deploy/manifests/prometheus.yaml` and edit accordingly. 8 | - Image is set to `latest`, this is only the example, it's a bad practice to not use versioned applications. 9 | 10 | [prometheus-operator]: https://github.com/coreos/prometheus-operator 11 | -------------------------------------------------------------------------------- /deploy/manifests/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: service-level-operator 13 | component: app 14 | strategy: 15 | rollingUpdate: 16 | maxUnavailable: 0 17 | template: 18 | metadata: 19 | labels: 20 | app: service-level-operator 21 | component: app 22 | spec: 23 | serviceAccountName: service-level-operator 24 | containers: 25 | - name: app 26 | imagePullPolicy: Always 27 | image: quay.io/spotahome/service-level-operator:latest 28 | ports: 29 | - containerPort: 8080 30 | name: http 31 | protocol: TCP 32 | readinessProbe: 33 | httpGet: 34 | path: /healthz/ready 35 | port: http 36 | livenessProbe: 37 | httpGet: 38 | path: /healthz/live 39 | port: http 40 | resources: 41 | limits: 42 | cpu: 220m 43 | memory: 254Mi 44 | requests: 45 | cpu: 120m 46 | memory: 128Mi 47 | -------------------------------------------------------------------------------- /deploy/manifests/prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | prometheus: myprometheus 9 | spec: 10 | selector: 11 | matchLabels: 12 | app: service-level-operator 13 | component: app 14 | namespaceSelector: 15 | matchNames: 16 | - app-namespace 17 | endpoints: 18 | - port: http 19 | interval: 10s 20 | -------------------------------------------------------------------------------- /deploy/manifests/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | 9 | --- 10 | apiVersion: rbac.authorization.k8s.io/v1 11 | kind: ClusterRole 12 | metadata: 13 | name: service-level-operator 14 | labels: 15 | app: service-level-operator 16 | component: app 17 | rules: 18 | # Register and check CRDs. 19 | - apiGroups: 20 | - apiextensions.k8s.io 21 | resources: 22 | - customresourcedefinitions 23 | verbs: 24 | - "*" 25 | 26 | # Operator logic. 27 | - apiGroups: 28 | - monitoring.spotahome.com 29 | resources: 30 | - servicelevels 31 | - servicelevels/status 32 | verbs: 33 | - "*" 34 | 35 | --- 36 | kind: ClusterRoleBinding 37 | apiVersion: rbac.authorization.k8s.io/v1 38 | metadata: 39 | name: service-level-operator 40 | subjects: 41 | - kind: ServiceAccount 42 | name: service-level-operator 43 | #namespace: test 44 | roleRef: 45 | apiGroup: rbac.authorization.k8s.io 46 | kind: ClusterRole 47 | name: service-level-operator 48 | -------------------------------------------------------------------------------- /deploy/manifests/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | spec: 9 | ports: 10 | - port: 80 11 | protocol: TCP 12 | name: http 13 | targetPort: http 14 | selector: 15 | app: service-level-operator 16 | component: app 17 | -------------------------------------------------------------------------------- /docker/dev/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine 2 | 3 | RUN apk --no-cache add \ 4 | bash \ 5 | git \ 6 | g++ \ 7 | curl \ 8 | openssl \ 9 | openssh-client 10 | 11 | # Mock creator 12 | RUN go get -u github.com/vektra/mockery/.../ 13 | 14 | RUN mkdir /src 15 | 16 | # Create user 17 | ARG uid=1000 18 | ARG gid=1000 19 | RUN addgroup -g $gid service-level-operator && \ 20 | adduser -D -u $uid -G service-level-operator service-level-operator && \ 21 | chown service-level-operator:service-level-operator -R /src && \ 22 | chown service-level-operator:service-level-operator -R /go 23 | USER service-level-operator 24 | 25 | # Fill go mod cache. 26 | RUN mkdir /tmp/cache 27 | COPY go.mod /tmp/cache 28 | COPY go.sum /tmp/cache 29 | RUN cd /tmp/cache && \ 30 | go mod download 31 | 32 | WORKDIR /src 33 | -------------------------------------------------------------------------------- /docker/dev/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | operator: 5 | build: 6 | context: ../.. 7 | dockerfile: docker/dev/Dockerfile 8 | volumes: 9 | - ../..:/src 10 | - ~/.bash_history.service-level-operator:/home/service-level-operator/.bash_history 11 | - ~/.kube:/home/service-level-operator/.kube:ro 12 | - ~/.gitconfig:/home/service-level-operator/.gitconfig:ro 13 | - ~/.ssh:/home/service-level-operator/.ssh 14 | command: ["go", "run", "./cmd/service-level-operator/main.go", "./cmd/service-level-operator/flags.go", "-development", "-debug", "-fake"] 15 | ports: 16 | - "8080:8080" 17 | 18 | prometheus: 19 | image: prom/prometheus 20 | volumes: 21 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 22 | ports: 23 | - 9090:9090 -------------------------------------------------------------------------------- /docker/dev/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 10s 3 | 4 | scrape_configs: 5 | - job_name: service-level-operator 6 | scrape_interval: 10s 7 | static_configs: 8 | - targets: ["operator:8080"] 9 | labels: 10 | mode: fake 11 | environment: dev 12 | -------------------------------------------------------------------------------- /docker/prod/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine AS build-stage 2 | 3 | RUN apk --no-cache add \ 4 | g++ \ 5 | git \ 6 | make 7 | 8 | ARG VERSION 9 | ENV VERSION=${VERSION} 10 | WORKDIR /src 11 | COPY . . 12 | RUN ./hack/scripts/build-binary.sh 13 | 14 | # Final image. 15 | FROM alpine:latest 16 | RUN apk --no-cache add \ 17 | ca-certificates 18 | COPY --from=build-stage /src/bin/service-level-operator /usr/local/bin/service-level-operator 19 | ENTRYPOINT ["/usr/local/bin/service-level-operator"] 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/spotahome/service-level-operator 2 | 3 | require ( 4 | github.com/go-openapi/spec v0.17.0 5 | github.com/oklog/run v1.0.0 6 | github.com/prometheus/client_golang v1.2.1 7 | github.com/prometheus/common v0.7.0 8 | github.com/sirupsen/logrus v1.4.2 9 | github.com/spotahome/kooper v0.6.1-0.20190926114429-1c6a0cfab9a5 10 | github.com/stretchr/testify v1.4.0 11 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 12 | k8s.io/api v0.0.0-20191004102255-dacd7df5a50b // indirect 13 | k8s.io/apiextensions-apiserver v0.0.0-20191004105443-a7d558db75c6 14 | k8s.io/apimachinery v0.0.0-20191004074956-01f8b7d1121a 15 | k8s.io/client-go v0.0.0-20191004102537-eb5b9a8cfde7 16 | k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d 17 | ) 18 | 19 | go 1.13 20 | -------------------------------------------------------------------------------- /hack/scripts/build-binary.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | goos=linux 7 | goarch=amd64 8 | src=./cmd/service-level-operator 9 | out=./bin/service-level-operator 10 | ldf_cmp="-w -extldflags '-static'" 11 | f_ver="-X main.Version=${VERSION:-dev}" 12 | 13 | echo "Building binary at ${out}" 14 | 15 | GOOS=${goos} GOARCH=${goarch} CGO_ENABLED=0 go build -o ${out} --ldflags "${ldf_cmp} ${f_ver}" ${src} -------------------------------------------------------------------------------- /hack/scripts/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | if [ -z ${VERSION} ]; then 6 | echo "VERSION env var needs to be set" 7 | exit 1 8 | fi 9 | 10 | REPOSITORY="quay.io/spotahome/" 11 | IMAGE="service-level-operator" 12 | TARGET_IMAGE=${REPOSITORY}${IMAGE} 13 | 14 | 15 | docker build \ 16 | --build-arg VERSION=${VERSION} \ 17 | -t ${TARGET_IMAGE}:${VERSION} \ 18 | -t ${TARGET_IMAGE}:latest \ 19 | -f ./docker/prod/Dockerfile . 20 | 21 | if [ -n "${PUSH_IMAGE}" ]; then 22 | echo "pushing ${TARGET_IMAGE} images..." 23 | docker push ${TARGET_IMAGE} 24 | fi -------------------------------------------------------------------------------- /hack/scripts/integration-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go test `go list ./... | grep -v vendor` -v -tags='integration' -------------------------------------------------------------------------------- /hack/scripts/k8scodegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | CODE_GENERATOR_IMAGE=quay.io/slok/kube-code-generator:v1.13.5 4 | DIRECTORY=${PWD} 5 | CODE_GENERATOR_PACKAGE=github.com/spotahome/service-level-operator 6 | 7 | docker run --rm -it \ 8 | -v ${DIRECTORY}:/go/src/${CODE_GENERATOR_PACKAGE} \ 9 | -e PROJECT_PACKAGE=${CODE_GENERATOR_PACKAGE} \ 10 | -e CLIENT_GENERATOR_OUT=${CODE_GENERATOR_PACKAGE}/pkg/k8sautogen/client \ 11 | -e APIS_ROOT=${CODE_GENERATOR_PACKAGE}/pkg/apis \ 12 | -e GROUPS_VERSION="monitoring:v1alpha1" \ 13 | -e GENERATION_TARGETS="deepcopy,client" \ 14 | ${CODE_GENERATOR_IMAGE} -------------------------------------------------------------------------------- /hack/scripts/mockgen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go generate ./mocks -------------------------------------------------------------------------------- /hack/scripts/openapicodegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | DIR="$( cd "$( dirname "${0}" )" && pwd )" 4 | ROOT_DIR=${DIR}/../.. 5 | 6 | PROJECT_PACKAGE=github.com/spotahome/service-level-operator 7 | IMAGE=quay.io/slok/kube-code-generator:v1.11.3 8 | 9 | # Execute once per package because we want independent output specs per kind/version. 10 | docker run -it --rm \ 11 | -v ${ROOT_DIR}:/go/src/${PROJECT_PACKAGE} \ 12 | -e CRD_PACKAGES=${PROJECT_PACKAGE}/pkg/apis/monitoring/v1alpha1 \ 13 | -e OPENAPI_OUTPUT_PACKAGE=${PROJECT_PACKAGE}/pkg/apis/monitoring/v1alpha1 \ 14 | ${IMAGE} ./update-openapi.sh -------------------------------------------------------------------------------- /hack/scripts/test-alerts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$(dirname "$(readlink -f $0)")" 4 | ROOT_DIR="${DIR}/../.." 5 | ALERTS=${ROOT_DIR}/alerts 6 | 7 | promtool test rules ${ALERTS}/*_test.yaml -------------------------------------------------------------------------------- /hack/scripts/unit-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go test `go list ./... | grep -v vendor` -v -------------------------------------------------------------------------------- /img/grafana_graphs1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/spotahome/service-level-operator/7eb3345016c14f8af9a437832513d98ac0cd24be/img/grafana_graphs1.png -------------------------------------------------------------------------------- /mocks/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package mocks will have all the mocks of the application, we'll try to use mocking using blackbox 3 | testing and integration tests whenever is possible. 4 | */ 5 | package mocks // import "github.com/spotahome/service-level-operator/mocks" 6 | 7 | // Service mocks. 8 | //go:generate mockery -output ./service/sli -outpkg sli -dir ../pkg/service/sli -name Retriever 9 | //go:generate mockery -output ./service/output -outpkg slo -dir ../pkg/service/output -name Output 10 | 11 | // Third party 12 | //go:generate mockery -output ./github.com/prometheus/client_golang/api/prometheus/v1 -outpkg v1 -dir . -name API 13 | -------------------------------------------------------------------------------- /mocks/github.com/prometheus/client_golang/api/prometheus/v1/API.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package v1 4 | 5 | import api "github.com/prometheus/client_golang/api" 6 | import context "context" 7 | import mock "github.com/stretchr/testify/mock" 8 | 9 | import model "github.com/prometheus/common/model" 10 | import time "time" 11 | import v1 "github.com/prometheus/client_golang/api/prometheus/v1" 12 | 13 | // API is an autogenerated mock type for the API type 14 | type API struct { 15 | mock.Mock 16 | } 17 | 18 | // AlertManagers provides a mock function with given fields: ctx 19 | func (_m *API) AlertManagers(ctx context.Context) (v1.AlertManagersResult, error) { 20 | ret := _m.Called(ctx) 21 | 22 | var r0 v1.AlertManagersResult 23 | if rf, ok := ret.Get(0).(func(context.Context) v1.AlertManagersResult); ok { 24 | r0 = rf(ctx) 25 | } else { 26 | r0 = ret.Get(0).(v1.AlertManagersResult) 27 | } 28 | 29 | var r1 error 30 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 31 | r1 = rf(ctx) 32 | } else { 33 | r1 = ret.Error(1) 34 | } 35 | 36 | return r0, r1 37 | } 38 | 39 | // Alerts provides a mock function with given fields: ctx 40 | func (_m *API) Alerts(ctx context.Context) (v1.AlertsResult, error) { 41 | ret := _m.Called(ctx) 42 | 43 | var r0 v1.AlertsResult 44 | if rf, ok := ret.Get(0).(func(context.Context) v1.AlertsResult); ok { 45 | r0 = rf(ctx) 46 | } else { 47 | r0 = ret.Get(0).(v1.AlertsResult) 48 | } 49 | 50 | var r1 error 51 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 52 | r1 = rf(ctx) 53 | } else { 54 | r1 = ret.Error(1) 55 | } 56 | 57 | return r0, r1 58 | } 59 | 60 | // CleanTombstones provides a mock function with given fields: ctx 61 | func (_m *API) CleanTombstones(ctx context.Context) error { 62 | ret := _m.Called(ctx) 63 | 64 | var r0 error 65 | if rf, ok := ret.Get(0).(func(context.Context) error); ok { 66 | r0 = rf(ctx) 67 | } else { 68 | r0 = ret.Error(0) 69 | } 70 | 71 | return r0 72 | } 73 | 74 | // Config provides a mock function with given fields: ctx 75 | func (_m *API) Config(ctx context.Context) (v1.ConfigResult, error) { 76 | ret := _m.Called(ctx) 77 | 78 | var r0 v1.ConfigResult 79 | if rf, ok := ret.Get(0).(func(context.Context) v1.ConfigResult); ok { 80 | r0 = rf(ctx) 81 | } else { 82 | r0 = ret.Get(0).(v1.ConfigResult) 83 | } 84 | 85 | var r1 error 86 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 87 | r1 = rf(ctx) 88 | } else { 89 | r1 = ret.Error(1) 90 | } 91 | 92 | return r0, r1 93 | } 94 | 95 | // DeleteSeries provides a mock function with given fields: ctx, matches, startTime, endTime 96 | func (_m *API) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { 97 | ret := _m.Called(ctx, matches, startTime, endTime) 98 | 99 | var r0 error 100 | if rf, ok := ret.Get(0).(func(context.Context, []string, time.Time, time.Time) error); ok { 101 | r0 = rf(ctx, matches, startTime, endTime) 102 | } else { 103 | r0 = ret.Error(0) 104 | } 105 | 106 | return r0 107 | } 108 | 109 | // Flags provides a mock function with given fields: ctx 110 | func (_m *API) Flags(ctx context.Context) (v1.FlagsResult, error) { 111 | ret := _m.Called(ctx) 112 | 113 | var r0 v1.FlagsResult 114 | if rf, ok := ret.Get(0).(func(context.Context) v1.FlagsResult); ok { 115 | r0 = rf(ctx) 116 | } else { 117 | if ret.Get(0) != nil { 118 | r0 = ret.Get(0).(v1.FlagsResult) 119 | } 120 | } 121 | 122 | var r1 error 123 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 124 | r1 = rf(ctx) 125 | } else { 126 | r1 = ret.Error(1) 127 | } 128 | 129 | return r0, r1 130 | } 131 | 132 | // LabelNames provides a mock function with given fields: ctx 133 | func (_m *API) LabelNames(ctx context.Context) ([]string, api.Warnings, error) { 134 | ret := _m.Called(ctx) 135 | 136 | var r0 []string 137 | if rf, ok := ret.Get(0).(func(context.Context) []string); ok { 138 | r0 = rf(ctx) 139 | } else { 140 | if ret.Get(0) != nil { 141 | r0 = ret.Get(0).([]string) 142 | } 143 | } 144 | 145 | var r1 api.Warnings 146 | if rf, ok := ret.Get(1).(func(context.Context) api.Warnings); ok { 147 | r1 = rf(ctx) 148 | } else { 149 | if ret.Get(1) != nil { 150 | r1 = ret.Get(1).(api.Warnings) 151 | } 152 | } 153 | 154 | var r2 error 155 | if rf, ok := ret.Get(2).(func(context.Context) error); ok { 156 | r2 = rf(ctx) 157 | } else { 158 | r2 = ret.Error(2) 159 | } 160 | 161 | return r0, r1, r2 162 | } 163 | 164 | // LabelValues provides a mock function with given fields: ctx, label 165 | func (_m *API) LabelValues(ctx context.Context, label string) (model.LabelValues, api.Warnings, error) { 166 | ret := _m.Called(ctx, label) 167 | 168 | var r0 model.LabelValues 169 | if rf, ok := ret.Get(0).(func(context.Context, string) model.LabelValues); ok { 170 | r0 = rf(ctx, label) 171 | } else { 172 | if ret.Get(0) != nil { 173 | r0 = ret.Get(0).(model.LabelValues) 174 | } 175 | } 176 | 177 | var r1 api.Warnings 178 | if rf, ok := ret.Get(1).(func(context.Context, string) api.Warnings); ok { 179 | r1 = rf(ctx, label) 180 | } else { 181 | if ret.Get(1) != nil { 182 | r1 = ret.Get(1).(api.Warnings) 183 | } 184 | } 185 | 186 | var r2 error 187 | if rf, ok := ret.Get(2).(func(context.Context, string) error); ok { 188 | r2 = rf(ctx, label) 189 | } else { 190 | r2 = ret.Error(2) 191 | } 192 | 193 | return r0, r1, r2 194 | } 195 | 196 | // Query provides a mock function with given fields: ctx, query, ts 197 | func (_m *API) Query(ctx context.Context, query string, ts time.Time) (model.Value, api.Warnings, error) { 198 | ret := _m.Called(ctx, query, ts) 199 | 200 | var r0 model.Value 201 | if rf, ok := ret.Get(0).(func(context.Context, string, time.Time) model.Value); ok { 202 | r0 = rf(ctx, query, ts) 203 | } else { 204 | if ret.Get(0) != nil { 205 | r0 = ret.Get(0).(model.Value) 206 | } 207 | } 208 | 209 | var r1 api.Warnings 210 | if rf, ok := ret.Get(1).(func(context.Context, string, time.Time) api.Warnings); ok { 211 | r1 = rf(ctx, query, ts) 212 | } else { 213 | if ret.Get(1) != nil { 214 | r1 = ret.Get(1).(api.Warnings) 215 | } 216 | } 217 | 218 | var r2 error 219 | if rf, ok := ret.Get(2).(func(context.Context, string, time.Time) error); ok { 220 | r2 = rf(ctx, query, ts) 221 | } else { 222 | r2 = ret.Error(2) 223 | } 224 | 225 | return r0, r1, r2 226 | } 227 | 228 | // QueryRange provides a mock function with given fields: ctx, query, r 229 | func (_m *API) QueryRange(ctx context.Context, query string, r v1.Range) (model.Value, api.Warnings, error) { 230 | ret := _m.Called(ctx, query, r) 231 | 232 | var r0 model.Value 233 | if rf, ok := ret.Get(0).(func(context.Context, string, v1.Range) model.Value); ok { 234 | r0 = rf(ctx, query, r) 235 | } else { 236 | if ret.Get(0) != nil { 237 | r0 = ret.Get(0).(model.Value) 238 | } 239 | } 240 | 241 | var r1 api.Warnings 242 | if rf, ok := ret.Get(1).(func(context.Context, string, v1.Range) api.Warnings); ok { 243 | r1 = rf(ctx, query, r) 244 | } else { 245 | if ret.Get(1) != nil { 246 | r1 = ret.Get(1).(api.Warnings) 247 | } 248 | } 249 | 250 | var r2 error 251 | if rf, ok := ret.Get(2).(func(context.Context, string, v1.Range) error); ok { 252 | r2 = rf(ctx, query, r) 253 | } else { 254 | r2 = ret.Error(2) 255 | } 256 | 257 | return r0, r1, r2 258 | } 259 | 260 | // Rules provides a mock function with given fields: ctx 261 | func (_m *API) Rules(ctx context.Context) (v1.RulesResult, error) { 262 | ret := _m.Called(ctx) 263 | 264 | var r0 v1.RulesResult 265 | if rf, ok := ret.Get(0).(func(context.Context) v1.RulesResult); ok { 266 | r0 = rf(ctx) 267 | } else { 268 | r0 = ret.Get(0).(v1.RulesResult) 269 | } 270 | 271 | var r1 error 272 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 273 | r1 = rf(ctx) 274 | } else { 275 | r1 = ret.Error(1) 276 | } 277 | 278 | return r0, r1 279 | } 280 | 281 | // Series provides a mock function with given fields: ctx, matches, startTime, endTime 282 | func (_m *API) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, api.Warnings, error) { 283 | ret := _m.Called(ctx, matches, startTime, endTime) 284 | 285 | var r0 []model.LabelSet 286 | if rf, ok := ret.Get(0).(func(context.Context, []string, time.Time, time.Time) []model.LabelSet); ok { 287 | r0 = rf(ctx, matches, startTime, endTime) 288 | } else { 289 | if ret.Get(0) != nil { 290 | r0 = ret.Get(0).([]model.LabelSet) 291 | } 292 | } 293 | 294 | var r1 api.Warnings 295 | if rf, ok := ret.Get(1).(func(context.Context, []string, time.Time, time.Time) api.Warnings); ok { 296 | r1 = rf(ctx, matches, startTime, endTime) 297 | } else { 298 | if ret.Get(1) != nil { 299 | r1 = ret.Get(1).(api.Warnings) 300 | } 301 | } 302 | 303 | var r2 error 304 | if rf, ok := ret.Get(2).(func(context.Context, []string, time.Time, time.Time) error); ok { 305 | r2 = rf(ctx, matches, startTime, endTime) 306 | } else { 307 | r2 = ret.Error(2) 308 | } 309 | 310 | return r0, r1, r2 311 | } 312 | 313 | // Snapshot provides a mock function with given fields: ctx, skipHead 314 | func (_m *API) Snapshot(ctx context.Context, skipHead bool) (v1.SnapshotResult, error) { 315 | ret := _m.Called(ctx, skipHead) 316 | 317 | var r0 v1.SnapshotResult 318 | if rf, ok := ret.Get(0).(func(context.Context, bool) v1.SnapshotResult); ok { 319 | r0 = rf(ctx, skipHead) 320 | } else { 321 | r0 = ret.Get(0).(v1.SnapshotResult) 322 | } 323 | 324 | var r1 error 325 | if rf, ok := ret.Get(1).(func(context.Context, bool) error); ok { 326 | r1 = rf(ctx, skipHead) 327 | } else { 328 | r1 = ret.Error(1) 329 | } 330 | 331 | return r0, r1 332 | } 333 | 334 | // Targets provides a mock function with given fields: ctx 335 | func (_m *API) Targets(ctx context.Context) (v1.TargetsResult, error) { 336 | ret := _m.Called(ctx) 337 | 338 | var r0 v1.TargetsResult 339 | if rf, ok := ret.Get(0).(func(context.Context) v1.TargetsResult); ok { 340 | r0 = rf(ctx) 341 | } else { 342 | r0 = ret.Get(0).(v1.TargetsResult) 343 | } 344 | 345 | var r1 error 346 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 347 | r1 = rf(ctx) 348 | } else { 349 | r1 = ret.Error(1) 350 | } 351 | 352 | return r0, r1 353 | } 354 | 355 | // TargetsMetadata provides a mock function with given fields: ctx, matchTarget, metric, limit 356 | func (_m *API) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]v1.MetricMetadata, error) { 357 | ret := _m.Called(ctx, matchTarget, metric, limit) 358 | 359 | var r0 []v1.MetricMetadata 360 | if rf, ok := ret.Get(0).(func(context.Context, string, string, string) []v1.MetricMetadata); ok { 361 | r0 = rf(ctx, matchTarget, metric, limit) 362 | } else { 363 | if ret.Get(0) != nil { 364 | r0 = ret.Get(0).([]v1.MetricMetadata) 365 | } 366 | } 367 | 368 | var r1 error 369 | if rf, ok := ret.Get(1).(func(context.Context, string, string, string) error); ok { 370 | r1 = rf(ctx, matchTarget, metric, limit) 371 | } else { 372 | r1 = ret.Error(1) 373 | } 374 | 375 | return r0, r1 376 | } 377 | -------------------------------------------------------------------------------- /mocks/service/output/Output.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package slo 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | 7 | import sli "github.com/spotahome/service-level-operator/pkg/service/sli" 8 | import v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 9 | 10 | // Output is an autogenerated mock type for the Output type 11 | type Output struct { 12 | mock.Mock 13 | } 14 | 15 | // Create provides a mock function with given fields: serviceLevel, _a1, result 16 | func (_m *Output) Create(serviceLevel *v1alpha1.ServiceLevel, _a1 *v1alpha1.SLO, result *sli.Result) error { 17 | ret := _m.Called(serviceLevel, _a1, result) 18 | 19 | var r0 error 20 | if rf, ok := ret.Get(0).(func(*v1alpha1.ServiceLevel, *v1alpha1.SLO, *sli.Result) error); ok { 21 | r0 = rf(serviceLevel, _a1, result) 22 | } else { 23 | r0 = ret.Error(0) 24 | } 25 | 26 | return r0 27 | } 28 | -------------------------------------------------------------------------------- /mocks/service/sli/Retriever.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package sli 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | import sli "github.com/spotahome/service-level-operator/pkg/service/sli" 7 | import v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 8 | 9 | // Retriever is an autogenerated mock type for the Retriever type 10 | type Retriever struct { 11 | mock.Mock 12 | } 13 | 14 | // Retrieve provides a mock function with given fields: _a0 15 | func (_m *Retriever) Retrieve(_a0 *v1alpha1.SLI) (sli.Result, error) { 16 | ret := _m.Called(_a0) 17 | 18 | var r0 sli.Result 19 | if rf, ok := ret.Get(0).(func(*v1alpha1.SLI) sli.Result); ok { 20 | r0 = rf(_a0) 21 | } else { 22 | r0 = ret.Get(0).(sli.Result) 23 | } 24 | 25 | var r1 error 26 | if rf, ok := ret.Get(1).(func(*v1alpha1.SLI) error); ok { 27 | r1 = rf(_a0) 28 | } else { 29 | r1 = ret.Error(1) 30 | } 31 | 32 | return r0, r1 33 | } 34 | -------------------------------------------------------------------------------- /mocks/thirdparty.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 5 | ) 6 | 7 | // Third party resources to create mocks from their interfaces. 8 | 9 | // API is the interface promv1.API. 10 | type API interface{ promv1.API } 11 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/register.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | const ( 4 | // GroupName is the name of the API group. 5 | GroupName = "monitoring.spotahome.com" 6 | ) 7 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | // +k8s:openapi-gen=true 3 | 4 | // Package v1alpha1 is the v1alpha1 version of the API. 5 | // +groupName=monitoring.spotahome.com 6 | package v1alpha1 7 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/helpers.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import "fmt" 4 | 5 | // Validate validates and sets defaults on the ServiceLevel 6 | // Kubernetes resource object. 7 | func (s *ServiceLevel) Validate() error { 8 | 9 | if len(s.Spec.ServiceLevelObjectives) == 0 { 10 | return fmt.Errorf("the number of SLOs on a service level must be more than 0") 11 | } 12 | 13 | // Check if there is an input. 14 | for _, slo := range s.Spec.ServiceLevelObjectives { 15 | err := s.validateSLO(&slo) 16 | if err != nil { 17 | return err 18 | } 19 | } 20 | 21 | return nil 22 | } 23 | 24 | func (s *ServiceLevel) validateSLO(slo *SLO) error { 25 | if slo.Name == "" { 26 | return fmt.Errorf("a SLO must have a name") 27 | } 28 | 29 | if slo.AvailabilityObjectivePercent == 0 { 30 | return fmt.Errorf("the %s SLO must have a availability objective percent", slo.Name) 31 | } 32 | 33 | // Check inputs. 34 | if slo.ServiceLevelIndicator.Prometheus == nil { 35 | return fmt.Errorf("the %s SLO must have at least one input source", slo.Name) 36 | } 37 | 38 | // Check outputs. 39 | if slo.Output.Prometheus == nil { 40 | return fmt.Errorf("the %s SLO must have at least one output source", slo.Name) 41 | } 42 | 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/helpers_test.go: -------------------------------------------------------------------------------- 1 | package v1alpha1_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | 9 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 10 | ) 11 | 12 | func TestServiceLevelValidation(t *testing.T) { 13 | // Setup the different combinations of service level to validate. 14 | goodSL := &monitoringv1alpha1.ServiceLevel{ 15 | ObjectMeta: metav1.ObjectMeta{ 16 | Name: "fake-service0", 17 | }, 18 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 19 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 20 | { 21 | Name: "fake_slo0", 22 | Description: "fake slo 0.", 23 | AvailabilityObjectivePercent: 99.99, 24 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 25 | SLISource: monitoringv1alpha1.SLISource{ 26 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 27 | Address: "http://fake:9090", 28 | TotalQuery: `slo0_total`, 29 | ErrorQuery: `slo0_error`, 30 | }, 31 | }, 32 | }, 33 | Output: monitoringv1alpha1.Output{ 34 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 35 | }, 36 | }, 37 | }, 38 | }, 39 | } 40 | slWithoutSLO := goodSL.DeepCopy() 41 | slWithoutSLO.Spec.ServiceLevelObjectives = []monitoringv1alpha1.SLO{} 42 | slSLOWithoutName := goodSL.DeepCopy() 43 | slSLOWithoutName.Spec.ServiceLevelObjectives[0].Name = "" 44 | slSLOWithoutObjective := goodSL.DeepCopy() 45 | slSLOWithoutObjective.Spec.ServiceLevelObjectives[0].AvailabilityObjectivePercent = 0 46 | slSLOWithoutSLI := goodSL.DeepCopy() 47 | slSLOWithoutSLI.Spec.ServiceLevelObjectives[0].ServiceLevelIndicator.Prometheus = nil 48 | slSLOWithoutOutput := goodSL.DeepCopy() 49 | slSLOWithoutOutput.Spec.ServiceLevelObjectives[0].Output.Prometheus = nil 50 | 51 | tests := []struct { 52 | name string 53 | serviceLevel *monitoringv1alpha1.ServiceLevel 54 | expErr bool 55 | }{ 56 | { 57 | name: "A valid ServiceLevel should be valid.", 58 | serviceLevel: goodSL, 59 | expErr: false, 60 | }, 61 | { 62 | name: "A ServiceLevel without SLOs houldn't be valid.", 63 | serviceLevel: slWithoutSLO, 64 | expErr: true, 65 | }, 66 | { 67 | name: "A ServiceLevel with an SLO without name shouldn't be valid.", 68 | serviceLevel: slSLOWithoutName, 69 | expErr: true, 70 | }, 71 | { 72 | name: "A ServiceLevel with an SLO without objective shouldn't be valid.", 73 | serviceLevel: slSLOWithoutObjective, 74 | expErr: true, 75 | }, 76 | { 77 | name: "A ServiceLevel with an SLO without SLI shouldn't be valid.", 78 | serviceLevel: slSLOWithoutSLI, 79 | expErr: true, 80 | }, 81 | { 82 | name: "A ServiceLevel with an SLO without output shouldn't be valid.", 83 | serviceLevel: slSLOWithoutOutput, 84 | expErr: true, 85 | }, 86 | } 87 | 88 | for _, test := range tests { 89 | t.Run(test.name, func(t *testing.T) { 90 | assert := assert.New(t) 91 | 92 | err := test.serviceLevel.Validate() 93 | 94 | if test.expErr { 95 | assert.Error(err) 96 | } else { 97 | assert.NoError(err) 98 | } 99 | }) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/openapi_generated.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by openapi-gen. DO NOT EDIT. 20 | 21 | // This file was autogenerated by openapi-gen. Do not edit it manually! 22 | 23 | package v1alpha1 24 | 25 | import ( 26 | spec "github.com/go-openapi/spec" 27 | common "k8s.io/kube-openapi/pkg/common" 28 | ) 29 | 30 | func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { 31 | return map[string]common.OpenAPIDefinition{ 32 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.Output": schema_pkg_apis_monitoring_v1alpha1_Output(ref), 33 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusOutputSource": schema_pkg_apis_monitoring_v1alpha1_PrometheusOutputSource(ref), 34 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusSLISource": schema_pkg_apis_monitoring_v1alpha1_PrometheusSLISource(ref), 35 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLI": schema_pkg_apis_monitoring_v1alpha1_SLI(ref), 36 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLISource": schema_pkg_apis_monitoring_v1alpha1_SLISource(ref), 37 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLO": schema_pkg_apis_monitoring_v1alpha1_SLO(ref), 38 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevel": schema_pkg_apis_monitoring_v1alpha1_ServiceLevel(ref), 39 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevelList": schema_pkg_apis_monitoring_v1alpha1_ServiceLevelList(ref), 40 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevelSpec": schema_pkg_apis_monitoring_v1alpha1_ServiceLevelSpec(ref), 41 | } 42 | } 43 | 44 | func schema_pkg_apis_monitoring_v1alpha1_Output(ref common.ReferenceCallback) common.OpenAPIDefinition { 45 | return common.OpenAPIDefinition{ 46 | Schema: spec.Schema{ 47 | SchemaProps: spec.SchemaProps{ 48 | Description: "Output is how the SLO will expose the generated SLO.", 49 | Properties: map[string]spec.Schema{ 50 | "prometheus": { 51 | SchemaProps: spec.SchemaProps{ 52 | Description: "Prometheus is the prometheus format for the SLO output.", 53 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusOutputSource"), 54 | }, 55 | }, 56 | }, 57 | }, 58 | }, 59 | Dependencies: []string{ 60 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusOutputSource"}, 61 | } 62 | } 63 | 64 | func schema_pkg_apis_monitoring_v1alpha1_PrometheusOutputSource(ref common.ReferenceCallback) common.OpenAPIDefinition { 65 | return common.OpenAPIDefinition{ 66 | Schema: spec.Schema{ 67 | SchemaProps: spec.SchemaProps{ 68 | Description: "PrometheusOutputSource is the source of the output in prometheus format.", 69 | Properties: map[string]spec.Schema{ 70 | "labels": { 71 | SchemaProps: spec.SchemaProps{ 72 | Description: "Labels are the labels that will be set to the output metrics of this SLO.", 73 | Type: []string{"object"}, 74 | AdditionalProperties: &spec.SchemaOrBool{ 75 | Schema: &spec.Schema{ 76 | SchemaProps: spec.SchemaProps{ 77 | Type: []string{"string"}, 78 | Format: "", 79 | }, 80 | }, 81 | }, 82 | }, 83 | }, 84 | }, 85 | }, 86 | }, 87 | Dependencies: []string{}, 88 | } 89 | } 90 | 91 | func schema_pkg_apis_monitoring_v1alpha1_PrometheusSLISource(ref common.ReferenceCallback) common.OpenAPIDefinition { 92 | return common.OpenAPIDefinition{ 93 | Schema: spec.Schema{ 94 | SchemaProps: spec.SchemaProps{ 95 | Description: "PrometheusSLISource is the source to get SLIs from a Prometheus backend.", 96 | Properties: map[string]spec.Schema{ 97 | "address": { 98 | SchemaProps: spec.SchemaProps{ 99 | Description: "Address is the address of the Prometheus.", 100 | Type: []string{"string"}, 101 | Format: "", 102 | }, 103 | }, 104 | "totalQuery": { 105 | SchemaProps: spec.SchemaProps{ 106 | Description: "TotalQuery is the query that gets the total that will be the base to get the unavailability of the SLO based on the errorQuery (errorQuery / totalQuery).", 107 | Type: []string{"string"}, 108 | Format: "", 109 | }, 110 | }, 111 | "errorQuery": { 112 | SchemaProps: spec.SchemaProps{ 113 | Description: "ErrorQuery is the query that gets the total errors that then will be divided against the total.", 114 | Type: []string{"string"}, 115 | Format: "", 116 | }, 117 | }, 118 | }, 119 | Required: []string{"address", "totalQuery", "errorQuery"}, 120 | }, 121 | }, 122 | Dependencies: []string{}, 123 | } 124 | } 125 | 126 | func schema_pkg_apis_monitoring_v1alpha1_SLI(ref common.ReferenceCallback) common.OpenAPIDefinition { 127 | return common.OpenAPIDefinition{ 128 | Schema: spec.Schema{ 129 | SchemaProps: spec.SchemaProps{ 130 | Description: "SLI is the SLI to get for the SLO.", 131 | Properties: map[string]spec.Schema{ 132 | "prometheus": { 133 | SchemaProps: spec.SchemaProps{ 134 | Description: "Prometheus is the prometheus SLI source.", 135 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusSLISource"), 136 | }, 137 | }, 138 | }, 139 | }, 140 | }, 141 | Dependencies: []string{ 142 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusSLISource"}, 143 | } 144 | } 145 | 146 | func schema_pkg_apis_monitoring_v1alpha1_SLISource(ref common.ReferenceCallback) common.OpenAPIDefinition { 147 | return common.OpenAPIDefinition{ 148 | Schema: spec.Schema{ 149 | SchemaProps: spec.SchemaProps{ 150 | Description: "SLISource is where the SLI will get from.", 151 | Properties: map[string]spec.Schema{ 152 | "prometheus": { 153 | SchemaProps: spec.SchemaProps{ 154 | Description: "Prometheus is the prometheus SLI source.", 155 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusSLISource"), 156 | }, 157 | }, 158 | }, 159 | }, 160 | }, 161 | Dependencies: []string{ 162 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.PrometheusSLISource"}, 163 | } 164 | } 165 | 166 | func schema_pkg_apis_monitoring_v1alpha1_SLO(ref common.ReferenceCallback) common.OpenAPIDefinition { 167 | return common.OpenAPIDefinition{ 168 | Schema: spec.Schema{ 169 | SchemaProps: spec.SchemaProps{ 170 | Description: "SLO represents a SLO.", 171 | Properties: map[string]spec.Schema{ 172 | "name": { 173 | SchemaProps: spec.SchemaProps{ 174 | Description: "Name of the SLO, must be made of [a-zA-z0-9] and '_'(underscore) characters.", 175 | Type: []string{"string"}, 176 | Format: "", 177 | }, 178 | }, 179 | "description": { 180 | SchemaProps: spec.SchemaProps{ 181 | Description: "Description is a description of the SLO.", 182 | Type: []string{"string"}, 183 | Format: "", 184 | }, 185 | }, 186 | "disable": { 187 | SchemaProps: spec.SchemaProps{ 188 | Description: "Disable will disable the SLO.", 189 | Type: []string{"boolean"}, 190 | Format: "", 191 | }, 192 | }, 193 | "availabilityObjectivePercent": { 194 | SchemaProps: spec.SchemaProps{ 195 | Description: "AvailabilityObjectivePercent is the percentage of availability target for the SLO.", 196 | Type: []string{"number"}, 197 | Format: "double", 198 | }, 199 | }, 200 | "serviceLevelIndicator": { 201 | SchemaProps: spec.SchemaProps{ 202 | Description: "ServiceLevelIndicator is the SLI associated with the SLO.", 203 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLI"), 204 | }, 205 | }, 206 | "output": { 207 | SchemaProps: spec.SchemaProps{ 208 | Description: "Output is the output backedn of the SLO.", 209 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.Output"), 210 | }, 211 | }, 212 | }, 213 | Required: []string{"name", "availabilityObjectivePercent", "serviceLevelIndicator", "output"}, 214 | }, 215 | }, 216 | Dependencies: []string{ 217 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.Output", "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLI"}, 218 | } 219 | } 220 | 221 | func schema_pkg_apis_monitoring_v1alpha1_ServiceLevel(ref common.ReferenceCallback) common.OpenAPIDefinition { 222 | return common.OpenAPIDefinition{ 223 | Schema: spec.Schema{ 224 | SchemaProps: spec.SchemaProps{ 225 | Description: "ServiceLevel represents a service level policy to measure the service level of an application.", 226 | Properties: map[string]spec.Schema{ 227 | "kind": { 228 | SchemaProps: spec.SchemaProps{ 229 | Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", 230 | Type: []string{"string"}, 231 | Format: "", 232 | }, 233 | }, 234 | "apiVersion": { 235 | SchemaProps: spec.SchemaProps{ 236 | Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", 237 | Type: []string{"string"}, 238 | Format: "", 239 | }, 240 | }, 241 | "metadata": { 242 | SchemaProps: spec.SchemaProps{ 243 | Description: "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata", 244 | Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), 245 | }, 246 | }, 247 | "spec": { 248 | SchemaProps: spec.SchemaProps{ 249 | Description: "Specification of the ddesired behaviour of the pod terminator. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status", 250 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevelSpec"), 251 | }, 252 | }, 253 | }, 254 | }, 255 | }, 256 | Dependencies: []string{ 257 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevelSpec", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, 258 | } 259 | } 260 | 261 | func schema_pkg_apis_monitoring_v1alpha1_ServiceLevelList(ref common.ReferenceCallback) common.OpenAPIDefinition { 262 | return common.OpenAPIDefinition{ 263 | Schema: spec.Schema{ 264 | SchemaProps: spec.SchemaProps{ 265 | Description: "ServiceLevelList is a list of ServiceLevel resources", 266 | Properties: map[string]spec.Schema{ 267 | "kind": { 268 | SchemaProps: spec.SchemaProps{ 269 | Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds", 270 | Type: []string{"string"}, 271 | Format: "", 272 | }, 273 | }, 274 | "apiVersion": { 275 | SchemaProps: spec.SchemaProps{ 276 | Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources", 277 | Type: []string{"string"}, 278 | Format: "", 279 | }, 280 | }, 281 | "metadata": { 282 | SchemaProps: spec.SchemaProps{ 283 | Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), 284 | }, 285 | }, 286 | "items": { 287 | SchemaProps: spec.SchemaProps{ 288 | Type: []string{"array"}, 289 | Items: &spec.SchemaOrArray{ 290 | Schema: &spec.Schema{ 291 | SchemaProps: spec.SchemaProps{ 292 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevel"), 293 | }, 294 | }, 295 | }, 296 | }, 297 | }, 298 | }, 299 | Required: []string{"metadata", "items"}, 300 | }, 301 | }, 302 | Dependencies: []string{ 303 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.ServiceLevel", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"}, 304 | } 305 | } 306 | 307 | func schema_pkg_apis_monitoring_v1alpha1_ServiceLevelSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { 308 | return common.OpenAPIDefinition{ 309 | Schema: spec.Schema{ 310 | SchemaProps: spec.SchemaProps{ 311 | Description: "ServiceLevelSpec is the spec for a ServiceLevel resource.", 312 | Properties: map[string]spec.Schema{ 313 | "serviceLevelObjectives": { 314 | SchemaProps: spec.SchemaProps{ 315 | Description: "ServiceLevelObjectives is the list of SLOs of a service/app.", 316 | Type: []string{"array"}, 317 | Items: &spec.SchemaOrArray{ 318 | Schema: &spec.Schema{ 319 | SchemaProps: spec.SchemaProps{ 320 | Ref: ref("github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLO"), 321 | }, 322 | }, 323 | }, 324 | }, 325 | }, 326 | }, 327 | }, 328 | }, 329 | Dependencies: []string{ 330 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1.SLO"}, 331 | } 332 | } 333 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" 5 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | 9 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring" 10 | ) 11 | 12 | const ( 13 | version = "v1alpha1" 14 | ) 15 | 16 | // ServiceLevel constants 17 | const ( 18 | ServiceLevelKind = "ServiceLevel" 19 | ServiceLevelName = "servicelevel" 20 | ServiceLevelNamePlural = "servicelevels" 21 | ServiceLevelScope = apiextensionsv1beta1.NamespaceScoped 22 | ) 23 | 24 | // SchemeGroupVersion is group version used to register these objects 25 | var SchemeGroupVersion = schema.GroupVersion{Group: monitoring.GroupName, Version: version} 26 | 27 | // Kind takes an unqualified kind and returns back a Group qualified GroupKind 28 | func Kind(kind string) schema.GroupKind { 29 | return VersionKind(kind).GroupKind() 30 | } 31 | 32 | // VersionKind takes an unqualified kind and returns back a Group qualified GroupVersionKind 33 | func VersionKind(kind string) schema.GroupVersionKind { 34 | return SchemeGroupVersion.WithKind(kind) 35 | } 36 | 37 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 38 | func Resource(resource string) schema.GroupResource { 39 | return SchemeGroupVersion.WithResource(resource).GroupResource() 40 | } 41 | 42 | var ( 43 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 44 | AddToScheme = SchemeBuilder.AddToScheme 45 | ) 46 | 47 | // Adds the list of known types to Scheme. 48 | func addKnownTypes(scheme *runtime.Scheme) error { 49 | scheme.AddKnownTypes(SchemeGroupVersion, 50 | &ServiceLevel{}, 51 | &ServiceLevelList{}, 52 | ) 53 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +genclient 8 | // +k8s:openapi-gen=true 9 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 10 | 11 | // ServiceLevel represents a service level policy to measure the service level 12 | // of an application. 13 | type ServiceLevel struct { 14 | metav1.TypeMeta `json:",inline"` 15 | // Standard object's metadata. 16 | // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata 17 | // +optional 18 | metav1.ObjectMeta `json:"metadata,omitempty"` 19 | 20 | // Specification of the ddesired behaviour of the pod terminator. 21 | // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status 22 | // +optional 23 | Spec ServiceLevelSpec `json:"spec,omitempty"` 24 | } 25 | 26 | // ServiceLevelSpec is the spec for a ServiceLevel resource. 27 | type ServiceLevelSpec struct { 28 | // ServiceLevelObjectives is the list of SLOs of a service/app. 29 | // +optional 30 | ServiceLevelObjectives []SLO `json:"serviceLevelObjectives,omitempty"` 31 | } 32 | 33 | // SLO represents a SLO. 34 | type SLO struct { 35 | // Name of the SLO, must be made of [a-zA-z0-9] and '_'(underscore) characters. 36 | Name string `json:"name"` 37 | // Description is a description of the SLO. 38 | // +optional 39 | Description string `json:"description,omitempty"` 40 | // Disable will disable the SLO. 41 | Disable bool `json:"disable,omitempty"` 42 | // AvailabilityObjectivePercent is the percentage of availability target for the SLO. 43 | AvailabilityObjectivePercent float64 `json:"availabilityObjectivePercent"` 44 | // ServiceLevelIndicator is the SLI associated with the SLO. 45 | ServiceLevelIndicator SLI `json:"serviceLevelIndicator"` 46 | // Output is the output backedn of the SLO. 47 | Output Output `json:"output"` 48 | } 49 | 50 | // SLI is the SLI to get for the SLO. 51 | type SLI struct { 52 | SLISource `json:",inline"` 53 | } 54 | 55 | // SLISource is where the SLI will get from. 56 | type SLISource struct { 57 | // Prometheus is the prometheus SLI source. 58 | // +optional 59 | Prometheus *PrometheusSLISource `json:"prometheus,omitempty"` 60 | } 61 | 62 | // PrometheusSLISource is the source to get SLIs from a Prometheus backend. 63 | type PrometheusSLISource struct { 64 | // Address is the address of the Prometheus. 65 | Address string `json:"address"` 66 | // TotalQuery is the query that gets the total that will be the base to get the unavailability 67 | // of the SLO based on the errorQuery (errorQuery / totalQuery). 68 | TotalQuery string `json:"totalQuery"` 69 | // ErrorQuery is the query that gets the total errors that then will be divided against the total. 70 | ErrorQuery string `json:"errorQuery"` 71 | } 72 | 73 | // Output is how the SLO will expose the generated SLO. 74 | type Output struct { 75 | //Prometheus is the prometheus format for the SLO output. 76 | // +optional 77 | Prometheus *PrometheusOutputSource `json:"prometheus,omitempty"` 78 | } 79 | 80 | // PrometheusOutputSource is the source of the output in prometheus format. 81 | type PrometheusOutputSource struct { 82 | // Labels are the labels that will be set to the output metrics of this SLO. 83 | // +optional 84 | Labels map[string]string `json:"labels,omitempty"` 85 | } 86 | 87 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 88 | 89 | // ServiceLevelList is a list of ServiceLevel resources 90 | type ServiceLevelList struct { 91 | metav1.TypeMeta `json:",inline"` 92 | metav1.ListMeta `json:"metadata"` 93 | 94 | Items []ServiceLevel `json:"items"` 95 | } 96 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by deepcopy-gen. DO NOT EDIT. 20 | 21 | package v1alpha1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *Output) DeepCopyInto(out *Output) { 29 | *out = *in 30 | if in.Prometheus != nil { 31 | in, out := &in.Prometheus, &out.Prometheus 32 | *out = new(PrometheusOutputSource) 33 | (*in).DeepCopyInto(*out) 34 | } 35 | return 36 | } 37 | 38 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Output. 39 | func (in *Output) DeepCopy() *Output { 40 | if in == nil { 41 | return nil 42 | } 43 | out := new(Output) 44 | in.DeepCopyInto(out) 45 | return out 46 | } 47 | 48 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 49 | func (in *PrometheusOutputSource) DeepCopyInto(out *PrometheusOutputSource) { 50 | *out = *in 51 | if in.Labels != nil { 52 | in, out := &in.Labels, &out.Labels 53 | *out = make(map[string]string, len(*in)) 54 | for key, val := range *in { 55 | (*out)[key] = val 56 | } 57 | } 58 | return 59 | } 60 | 61 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusOutputSource. 62 | func (in *PrometheusOutputSource) DeepCopy() *PrometheusOutputSource { 63 | if in == nil { 64 | return nil 65 | } 66 | out := new(PrometheusOutputSource) 67 | in.DeepCopyInto(out) 68 | return out 69 | } 70 | 71 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 72 | func (in *PrometheusSLISource) DeepCopyInto(out *PrometheusSLISource) { 73 | *out = *in 74 | return 75 | } 76 | 77 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusSLISource. 78 | func (in *PrometheusSLISource) DeepCopy() *PrometheusSLISource { 79 | if in == nil { 80 | return nil 81 | } 82 | out := new(PrometheusSLISource) 83 | in.DeepCopyInto(out) 84 | return out 85 | } 86 | 87 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 88 | func (in *SLI) DeepCopyInto(out *SLI) { 89 | *out = *in 90 | in.SLISource.DeepCopyInto(&out.SLISource) 91 | return 92 | } 93 | 94 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SLI. 95 | func (in *SLI) DeepCopy() *SLI { 96 | if in == nil { 97 | return nil 98 | } 99 | out := new(SLI) 100 | in.DeepCopyInto(out) 101 | return out 102 | } 103 | 104 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 105 | func (in *SLISource) DeepCopyInto(out *SLISource) { 106 | *out = *in 107 | if in.Prometheus != nil { 108 | in, out := &in.Prometheus, &out.Prometheus 109 | *out = new(PrometheusSLISource) 110 | **out = **in 111 | } 112 | return 113 | } 114 | 115 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SLISource. 116 | func (in *SLISource) DeepCopy() *SLISource { 117 | if in == nil { 118 | return nil 119 | } 120 | out := new(SLISource) 121 | in.DeepCopyInto(out) 122 | return out 123 | } 124 | 125 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 126 | func (in *SLO) DeepCopyInto(out *SLO) { 127 | *out = *in 128 | in.ServiceLevelIndicator.DeepCopyInto(&out.ServiceLevelIndicator) 129 | in.Output.DeepCopyInto(&out.Output) 130 | return 131 | } 132 | 133 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SLO. 134 | func (in *SLO) DeepCopy() *SLO { 135 | if in == nil { 136 | return nil 137 | } 138 | out := new(SLO) 139 | in.DeepCopyInto(out) 140 | return out 141 | } 142 | 143 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 144 | func (in *ServiceLevel) DeepCopyInto(out *ServiceLevel) { 145 | *out = *in 146 | out.TypeMeta = in.TypeMeta 147 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 148 | in.Spec.DeepCopyInto(&out.Spec) 149 | return 150 | } 151 | 152 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceLevel. 153 | func (in *ServiceLevel) DeepCopy() *ServiceLevel { 154 | if in == nil { 155 | return nil 156 | } 157 | out := new(ServiceLevel) 158 | in.DeepCopyInto(out) 159 | return out 160 | } 161 | 162 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 163 | func (in *ServiceLevel) DeepCopyObject() runtime.Object { 164 | if c := in.DeepCopy(); c != nil { 165 | return c 166 | } 167 | return nil 168 | } 169 | 170 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 171 | func (in *ServiceLevelList) DeepCopyInto(out *ServiceLevelList) { 172 | *out = *in 173 | out.TypeMeta = in.TypeMeta 174 | out.ListMeta = in.ListMeta 175 | if in.Items != nil { 176 | in, out := &in.Items, &out.Items 177 | *out = make([]ServiceLevel, len(*in)) 178 | for i := range *in { 179 | (*in)[i].DeepCopyInto(&(*out)[i]) 180 | } 181 | } 182 | return 183 | } 184 | 185 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceLevelList. 186 | func (in *ServiceLevelList) DeepCopy() *ServiceLevelList { 187 | if in == nil { 188 | return nil 189 | } 190 | out := new(ServiceLevelList) 191 | in.DeepCopyInto(out) 192 | return out 193 | } 194 | 195 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 196 | func (in *ServiceLevelList) DeepCopyObject() runtime.Object { 197 | if c := in.DeepCopy(); c != nil { 198 | return c 199 | } 200 | return nil 201 | } 202 | 203 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 204 | func (in *ServiceLevelSpec) DeepCopyInto(out *ServiceLevelSpec) { 205 | *out = *in 206 | if in.ServiceLevelObjectives != nil { 207 | in, out := &in.ServiceLevelObjectives, &out.ServiceLevelObjectives 208 | *out = make([]SLO, len(*in)) 209 | for i := range *in { 210 | (*in)[i].DeepCopyInto(&(*out)[i]) 211 | } 212 | } 213 | return 214 | } 215 | 216 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceLevelSpec. 217 | func (in *ServiceLevelSpec) DeepCopy() *ServiceLevelSpec { 218 | if in == nil { 219 | return nil 220 | } 221 | out := new(ServiceLevelSpec) 222 | in.DeepCopyInto(out) 223 | return out 224 | } 225 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/clientset.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package versioned 20 | 21 | import ( 22 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1" 23 | discovery "k8s.io/client-go/discovery" 24 | rest "k8s.io/client-go/rest" 25 | flowcontrol "k8s.io/client-go/util/flowcontrol" 26 | ) 27 | 28 | type Interface interface { 29 | Discovery() discovery.DiscoveryInterface 30 | MonitoringV1alpha1() monitoringv1alpha1.MonitoringV1alpha1Interface 31 | // Deprecated: please explicitly pick a version if possible. 32 | Monitoring() monitoringv1alpha1.MonitoringV1alpha1Interface 33 | } 34 | 35 | // Clientset contains the clients for groups. Each group has exactly one 36 | // version included in a Clientset. 37 | type Clientset struct { 38 | *discovery.DiscoveryClient 39 | monitoringV1alpha1 *monitoringv1alpha1.MonitoringV1alpha1Client 40 | } 41 | 42 | // MonitoringV1alpha1 retrieves the MonitoringV1alpha1Client 43 | func (c *Clientset) MonitoringV1alpha1() monitoringv1alpha1.MonitoringV1alpha1Interface { 44 | return c.monitoringV1alpha1 45 | } 46 | 47 | // Deprecated: Monitoring retrieves the default version of MonitoringClient. 48 | // Please explicitly pick a version. 49 | func (c *Clientset) Monitoring() monitoringv1alpha1.MonitoringV1alpha1Interface { 50 | return c.monitoringV1alpha1 51 | } 52 | 53 | // Discovery retrieves the DiscoveryClient 54 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 55 | if c == nil { 56 | return nil 57 | } 58 | return c.DiscoveryClient 59 | } 60 | 61 | // NewForConfig creates a new Clientset for the given config. 62 | func NewForConfig(c *rest.Config) (*Clientset, error) { 63 | configShallowCopy := *c 64 | if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { 65 | configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) 66 | } 67 | var cs Clientset 68 | var err error 69 | cs.monitoringV1alpha1, err = monitoringv1alpha1.NewForConfig(&configShallowCopy) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return &cs, nil 79 | } 80 | 81 | // NewForConfigOrDie creates a new Clientset for the given config and 82 | // panics if there is an error in the config. 83 | func NewForConfigOrDie(c *rest.Config) *Clientset { 84 | var cs Clientset 85 | cs.monitoringV1alpha1 = monitoringv1alpha1.NewForConfigOrDie(c) 86 | 87 | cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) 88 | return &cs 89 | } 90 | 91 | // New creates a new Clientset for the given RESTClient. 92 | func New(c rest.Interface) *Clientset { 93 | var cs Clientset 94 | cs.monitoringV1alpha1 = monitoringv1alpha1.New(c) 95 | 96 | cs.DiscoveryClient = discovery.NewDiscoveryClient(c) 97 | return &cs 98 | } 99 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated clientset. 20 | package versioned 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/fake/clientset_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | clientset "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 23 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1" 24 | fakemonitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/watch" 27 | "k8s.io/client-go/discovery" 28 | fakediscovery "k8s.io/client-go/discovery/fake" 29 | "k8s.io/client-go/testing" 30 | ) 31 | 32 | // NewSimpleClientset returns a clientset that will respond with the provided objects. 33 | // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, 34 | // without applying any validations and/or defaults. It shouldn't be considered a replacement 35 | // for a real clientset and is mostly useful in simple unit tests. 36 | func NewSimpleClientset(objects ...runtime.Object) *Clientset { 37 | o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) 38 | for _, obj := range objects { 39 | if err := o.Add(obj); err != nil { 40 | panic(err) 41 | } 42 | } 43 | 44 | cs := &Clientset{} 45 | cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} 46 | cs.AddReactor("*", "*", testing.ObjectReaction(o)) 47 | cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { 48 | gvr := action.GetResource() 49 | ns := action.GetNamespace() 50 | watch, err := o.Watch(gvr, ns) 51 | if err != nil { 52 | return false, nil, err 53 | } 54 | return true, watch, nil 55 | }) 56 | 57 | return cs 58 | } 59 | 60 | // Clientset implements clientset.Interface. Meant to be embedded into a 61 | // struct to get a default implementation. This makes faking out just the method 62 | // you want to test easier. 63 | type Clientset struct { 64 | testing.Fake 65 | discovery *fakediscovery.FakeDiscovery 66 | } 67 | 68 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 69 | return c.discovery 70 | } 71 | 72 | var _ clientset.Interface = &Clientset{} 73 | 74 | // MonitoringV1alpha1 retrieves the MonitoringV1alpha1Client 75 | func (c *Clientset) MonitoringV1alpha1() monitoringv1alpha1.MonitoringV1alpha1Interface { 76 | return &fakemonitoringv1alpha1.FakeMonitoringV1alpha1{Fake: &c.Fake} 77 | } 78 | 79 | // Monitoring retrieves the MonitoringV1alpha1Client 80 | func (c *Clientset) Monitoring() monitoringv1alpha1.MonitoringV1alpha1Interface { 81 | return &fakemonitoringv1alpha1.FakeMonitoringV1alpha1{Fake: &c.Fake} 82 | } 83 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated fake clientset. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var scheme = runtime.NewScheme() 31 | var codecs = serializer.NewCodecFactory(scheme) 32 | var parameterCodec = runtime.NewParameterCodec(scheme) 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | monitoringv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package contains the scheme of the automatically generated clientset. 20 | package scheme 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package scheme 20 | 21 | import ( 22 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var Scheme = runtime.NewScheme() 31 | var Codecs = serializer.NewCodecFactory(Scheme) 32 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | monitoringv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(Scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated typed clients. 20 | package v1alpha1 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // Package fake has the automatically generated clients. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake/fake_monitoring_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1" 23 | rest "k8s.io/client-go/rest" 24 | testing "k8s.io/client-go/testing" 25 | ) 26 | 27 | type FakeMonitoringV1alpha1 struct { 28 | *testing.Fake 29 | } 30 | 31 | func (c *FakeMonitoringV1alpha1) ServiceLevels(namespace string) v1alpha1.ServiceLevelInterface { 32 | return &FakeServiceLevels{c, namespace} 33 | } 34 | 35 | // RESTClient returns a RESTClient that is used to communicate 36 | // with API server by this client implementation. 37 | func (c *FakeMonitoringV1alpha1) RESTClient() rest.Interface { 38 | var ret *rest.RESTClient 39 | return ret 40 | } 41 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake/fake_servicelevel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | labels "k8s.io/apimachinery/pkg/labels" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | types "k8s.io/apimachinery/pkg/types" 27 | watch "k8s.io/apimachinery/pkg/watch" 28 | testing "k8s.io/client-go/testing" 29 | ) 30 | 31 | // FakeServiceLevels implements ServiceLevelInterface 32 | type FakeServiceLevels struct { 33 | Fake *FakeMonitoringV1alpha1 34 | ns string 35 | } 36 | 37 | var servicelevelsResource = schema.GroupVersionResource{Group: "monitoring.spotahome.com", Version: "v1alpha1", Resource: "servicelevels"} 38 | 39 | var servicelevelsKind = schema.GroupVersionKind{Group: "monitoring.spotahome.com", Version: "v1alpha1", Kind: "ServiceLevel"} 40 | 41 | // Get takes name of the serviceLevel, and returns the corresponding serviceLevel object, and an error if there is any. 42 | func (c *FakeServiceLevels) Get(name string, options v1.GetOptions) (result *v1alpha1.ServiceLevel, err error) { 43 | obj, err := c.Fake. 44 | Invokes(testing.NewGetAction(servicelevelsResource, c.ns, name), &v1alpha1.ServiceLevel{}) 45 | 46 | if obj == nil { 47 | return nil, err 48 | } 49 | return obj.(*v1alpha1.ServiceLevel), err 50 | } 51 | 52 | // List takes label and field selectors, and returns the list of ServiceLevels that match those selectors. 53 | func (c *FakeServiceLevels) List(opts v1.ListOptions) (result *v1alpha1.ServiceLevelList, err error) { 54 | obj, err := c.Fake. 55 | Invokes(testing.NewListAction(servicelevelsResource, servicelevelsKind, c.ns, opts), &v1alpha1.ServiceLevelList{}) 56 | 57 | if obj == nil { 58 | return nil, err 59 | } 60 | 61 | label, _, _ := testing.ExtractFromListOptions(opts) 62 | if label == nil { 63 | label = labels.Everything() 64 | } 65 | list := &v1alpha1.ServiceLevelList{ListMeta: obj.(*v1alpha1.ServiceLevelList).ListMeta} 66 | for _, item := range obj.(*v1alpha1.ServiceLevelList).Items { 67 | if label.Matches(labels.Set(item.Labels)) { 68 | list.Items = append(list.Items, item) 69 | } 70 | } 71 | return list, err 72 | } 73 | 74 | // Watch returns a watch.Interface that watches the requested serviceLevels. 75 | func (c *FakeServiceLevels) Watch(opts v1.ListOptions) (watch.Interface, error) { 76 | return c.Fake. 77 | InvokesWatch(testing.NewWatchAction(servicelevelsResource, c.ns, opts)) 78 | 79 | } 80 | 81 | // Create takes the representation of a serviceLevel and creates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 82 | func (c *FakeServiceLevels) Create(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 83 | obj, err := c.Fake. 84 | Invokes(testing.NewCreateAction(servicelevelsResource, c.ns, serviceLevel), &v1alpha1.ServiceLevel{}) 85 | 86 | if obj == nil { 87 | return nil, err 88 | } 89 | return obj.(*v1alpha1.ServiceLevel), err 90 | } 91 | 92 | // Update takes the representation of a serviceLevel and updates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 93 | func (c *FakeServiceLevels) Update(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 94 | obj, err := c.Fake. 95 | Invokes(testing.NewUpdateAction(servicelevelsResource, c.ns, serviceLevel), &v1alpha1.ServiceLevel{}) 96 | 97 | if obj == nil { 98 | return nil, err 99 | } 100 | return obj.(*v1alpha1.ServiceLevel), err 101 | } 102 | 103 | // Delete takes name of the serviceLevel and deletes it. Returns an error if one occurs. 104 | func (c *FakeServiceLevels) Delete(name string, options *v1.DeleteOptions) error { 105 | _, err := c.Fake. 106 | Invokes(testing.NewDeleteAction(servicelevelsResource, c.ns, name), &v1alpha1.ServiceLevel{}) 107 | 108 | return err 109 | } 110 | 111 | // DeleteCollection deletes a collection of objects. 112 | func (c *FakeServiceLevels) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { 113 | action := testing.NewDeleteCollectionAction(servicelevelsResource, c.ns, listOptions) 114 | 115 | _, err := c.Fake.Invokes(action, &v1alpha1.ServiceLevelList{}) 116 | return err 117 | } 118 | 119 | // Patch applies the patch and returns the patched serviceLevel. 120 | func (c *FakeServiceLevels) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ServiceLevel, err error) { 121 | obj, err := c.Fake. 122 | Invokes(testing.NewPatchSubresourceAction(servicelevelsResource, c.ns, name, pt, data, subresources...), &v1alpha1.ServiceLevel{}) 123 | 124 | if obj == nil { 125 | return nil, err 126 | } 127 | return obj.(*v1alpha1.ServiceLevel), err 128 | } 129 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | type ServiceLevelExpansion interface{} 22 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/monitoring_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/scheme" 24 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 25 | rest "k8s.io/client-go/rest" 26 | ) 27 | 28 | type MonitoringV1alpha1Interface interface { 29 | RESTClient() rest.Interface 30 | ServiceLevelsGetter 31 | } 32 | 33 | // MonitoringV1alpha1Client is used to interact with features provided by the monitoring.spotahome.com group. 34 | type MonitoringV1alpha1Client struct { 35 | restClient rest.Interface 36 | } 37 | 38 | func (c *MonitoringV1alpha1Client) ServiceLevels(namespace string) ServiceLevelInterface { 39 | return newServiceLevels(c, namespace) 40 | } 41 | 42 | // NewForConfig creates a new MonitoringV1alpha1Client for the given config. 43 | func NewForConfig(c *rest.Config) (*MonitoringV1alpha1Client, error) { 44 | config := *c 45 | if err := setConfigDefaults(&config); err != nil { 46 | return nil, err 47 | } 48 | client, err := rest.RESTClientFor(&config) 49 | if err != nil { 50 | return nil, err 51 | } 52 | return &MonitoringV1alpha1Client{client}, nil 53 | } 54 | 55 | // NewForConfigOrDie creates a new MonitoringV1alpha1Client for the given config and 56 | // panics if there is an error in the config. 57 | func NewForConfigOrDie(c *rest.Config) *MonitoringV1alpha1Client { 58 | client, err := NewForConfig(c) 59 | if err != nil { 60 | panic(err) 61 | } 62 | return client 63 | } 64 | 65 | // New creates a new MonitoringV1alpha1Client for the given RESTClient. 66 | func New(c rest.Interface) *MonitoringV1alpha1Client { 67 | return &MonitoringV1alpha1Client{c} 68 | } 69 | 70 | func setConfigDefaults(config *rest.Config) error { 71 | gv := v1alpha1.SchemeGroupVersion 72 | config.GroupVersion = &gv 73 | config.APIPath = "/apis" 74 | config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} 75 | 76 | if config.UserAgent == "" { 77 | config.UserAgent = rest.DefaultKubernetesUserAgent() 78 | } 79 | 80 | return nil 81 | } 82 | 83 | // RESTClient returns a RESTClient that is used to communicate 84 | // with API server by this client implementation. 85 | func (c *MonitoringV1alpha1Client) RESTClient() rest.Interface { 86 | if c == nil { 87 | return nil 88 | } 89 | return c.restClient 90 | } 91 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/servicelevel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "time" 23 | 24 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 25 | scheme "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/scheme" 26 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | types "k8s.io/apimachinery/pkg/types" 28 | watch "k8s.io/apimachinery/pkg/watch" 29 | rest "k8s.io/client-go/rest" 30 | ) 31 | 32 | // ServiceLevelsGetter has a method to return a ServiceLevelInterface. 33 | // A group's client should implement this interface. 34 | type ServiceLevelsGetter interface { 35 | ServiceLevels(namespace string) ServiceLevelInterface 36 | } 37 | 38 | // ServiceLevelInterface has methods to work with ServiceLevel resources. 39 | type ServiceLevelInterface interface { 40 | Create(*v1alpha1.ServiceLevel) (*v1alpha1.ServiceLevel, error) 41 | Update(*v1alpha1.ServiceLevel) (*v1alpha1.ServiceLevel, error) 42 | Delete(name string, options *v1.DeleteOptions) error 43 | DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error 44 | Get(name string, options v1.GetOptions) (*v1alpha1.ServiceLevel, error) 45 | List(opts v1.ListOptions) (*v1alpha1.ServiceLevelList, error) 46 | Watch(opts v1.ListOptions) (watch.Interface, error) 47 | Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ServiceLevel, err error) 48 | ServiceLevelExpansion 49 | } 50 | 51 | // serviceLevels implements ServiceLevelInterface 52 | type serviceLevels struct { 53 | client rest.Interface 54 | ns string 55 | } 56 | 57 | // newServiceLevels returns a ServiceLevels 58 | func newServiceLevels(c *MonitoringV1alpha1Client, namespace string) *serviceLevels { 59 | return &serviceLevels{ 60 | client: c.RESTClient(), 61 | ns: namespace, 62 | } 63 | } 64 | 65 | // Get takes name of the serviceLevel, and returns the corresponding serviceLevel object, and an error if there is any. 66 | func (c *serviceLevels) Get(name string, options v1.GetOptions) (result *v1alpha1.ServiceLevel, err error) { 67 | result = &v1alpha1.ServiceLevel{} 68 | err = c.client.Get(). 69 | Namespace(c.ns). 70 | Resource("servicelevels"). 71 | Name(name). 72 | VersionedParams(&options, scheme.ParameterCodec). 73 | Do(). 74 | Into(result) 75 | return 76 | } 77 | 78 | // List takes label and field selectors, and returns the list of ServiceLevels that match those selectors. 79 | func (c *serviceLevels) List(opts v1.ListOptions) (result *v1alpha1.ServiceLevelList, err error) { 80 | var timeout time.Duration 81 | if opts.TimeoutSeconds != nil { 82 | timeout = time.Duration(*opts.TimeoutSeconds) * time.Second 83 | } 84 | result = &v1alpha1.ServiceLevelList{} 85 | err = c.client.Get(). 86 | Namespace(c.ns). 87 | Resource("servicelevels"). 88 | VersionedParams(&opts, scheme.ParameterCodec). 89 | Timeout(timeout). 90 | Do(). 91 | Into(result) 92 | return 93 | } 94 | 95 | // Watch returns a watch.Interface that watches the requested serviceLevels. 96 | func (c *serviceLevels) Watch(opts v1.ListOptions) (watch.Interface, error) { 97 | var timeout time.Duration 98 | if opts.TimeoutSeconds != nil { 99 | timeout = time.Duration(*opts.TimeoutSeconds) * time.Second 100 | } 101 | opts.Watch = true 102 | return c.client.Get(). 103 | Namespace(c.ns). 104 | Resource("servicelevels"). 105 | VersionedParams(&opts, scheme.ParameterCodec). 106 | Timeout(timeout). 107 | Watch() 108 | } 109 | 110 | // Create takes the representation of a serviceLevel and creates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 111 | func (c *serviceLevels) Create(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 112 | result = &v1alpha1.ServiceLevel{} 113 | err = c.client.Post(). 114 | Namespace(c.ns). 115 | Resource("servicelevels"). 116 | Body(serviceLevel). 117 | Do(). 118 | Into(result) 119 | return 120 | } 121 | 122 | // Update takes the representation of a serviceLevel and updates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 123 | func (c *serviceLevels) Update(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 124 | result = &v1alpha1.ServiceLevel{} 125 | err = c.client.Put(). 126 | Namespace(c.ns). 127 | Resource("servicelevels"). 128 | Name(serviceLevel.Name). 129 | Body(serviceLevel). 130 | Do(). 131 | Into(result) 132 | return 133 | } 134 | 135 | // Delete takes name of the serviceLevel and deletes it. Returns an error if one occurs. 136 | func (c *serviceLevels) Delete(name string, options *v1.DeleteOptions) error { 137 | return c.client.Delete(). 138 | Namespace(c.ns). 139 | Resource("servicelevels"). 140 | Name(name). 141 | Body(options). 142 | Do(). 143 | Error() 144 | } 145 | 146 | // DeleteCollection deletes a collection of objects. 147 | func (c *serviceLevels) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { 148 | var timeout time.Duration 149 | if listOptions.TimeoutSeconds != nil { 150 | timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second 151 | } 152 | return c.client.Delete(). 153 | Namespace(c.ns). 154 | Resource("servicelevels"). 155 | VersionedParams(&listOptions, scheme.ParameterCodec). 156 | Timeout(timeout). 157 | Body(options). 158 | Do(). 159 | Error() 160 | } 161 | 162 | // Patch applies the patch and returns the patched serviceLevel. 163 | func (c *serviceLevels) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ServiceLevel, err error) { 164 | result = &v1alpha1.ServiceLevel{} 165 | err = c.client.Patch(pt). 166 | Namespace(c.ns). 167 | Resource("servicelevels"). 168 | SubResource(subresources...). 169 | Name(name). 170 | Body(data). 171 | Do(). 172 | Into(result) 173 | return 174 | } 175 | -------------------------------------------------------------------------------- /pkg/log/dummy.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | // Dummy is a dummy logger 4 | var Dummy = dummyLogger{} 5 | 6 | type dummyLogger struct{} 7 | 8 | func (l dummyLogger) Debug(...interface{}) {} 9 | func (l dummyLogger) Debugln(...interface{}) {} 10 | func (l dummyLogger) Debugf(string, ...interface{}) {} 11 | func (l dummyLogger) Info(...interface{}) {} 12 | func (l dummyLogger) Infoln(...interface{}) {} 13 | func (l dummyLogger) Infof(string, ...interface{}) {} 14 | func (l dummyLogger) Warn(...interface{}) {} 15 | func (l dummyLogger) Warnln(...interface{}) {} 16 | func (l dummyLogger) Warnf(string, ...interface{}) {} 17 | func (l dummyLogger) Warningf(format string, args ...interface{}) {} 18 | func (l dummyLogger) Error(...interface{}) {} 19 | func (l dummyLogger) Errorln(...interface{}) {} 20 | func (l dummyLogger) Errorf(string, ...interface{}) {} 21 | func (l dummyLogger) Fatal(...interface{}) {} 22 | func (l dummyLogger) Fatalln(...interface{}) {} 23 | func (l dummyLogger) Fatalf(string, ...interface{}) {} 24 | func (l dummyLogger) Panic(...interface{}) {} 25 | func (l dummyLogger) Panicln(...interface{}) {} 26 | func (l dummyLogger) Panicf(string, ...interface{}) {} 27 | func (l dummyLogger) With(key string, value interface{}) Logger { return l } 28 | func (l dummyLogger) WithField(key string, value interface{}) Logger { return l } 29 | func (l dummyLogger) Set(level Level) error { return nil } 30 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "strings" 7 | 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // Level refers to the level of logging 12 | type Level string 13 | 14 | // Logger is an interface that needs to be implemented in order to log. 15 | type Logger interface { 16 | Debug(...interface{}) 17 | Debugln(...interface{}) 18 | Debugf(string, ...interface{}) 19 | 20 | Info(...interface{}) 21 | Infoln(...interface{}) 22 | Infof(string, ...interface{}) 23 | 24 | Warn(...interface{}) 25 | Warnln(...interface{}) 26 | Warnf(string, ...interface{}) 27 | Warningf(string, ...interface{}) 28 | 29 | Error(...interface{}) 30 | Errorln(...interface{}) 31 | Errorf(string, ...interface{}) 32 | 33 | Fatal(...interface{}) 34 | Fatalln(...interface{}) 35 | Fatalf(string, ...interface{}) 36 | 37 | Panic(...interface{}) 38 | Panicln(...interface{}) 39 | Panicf(string, ...interface{}) 40 | 41 | With(key string, value interface{}) Logger 42 | WithField(key string, value interface{}) Logger 43 | Set(level Level) error 44 | } 45 | 46 | type logger struct { 47 | entry *logrus.Entry 48 | } 49 | 50 | func (l logger) Debug(args ...interface{}) { 51 | l.sourced().Debug(args...) 52 | } 53 | 54 | func (l logger) Debugln(args ...interface{}) { 55 | l.sourced().Debugln(args...) 56 | } 57 | 58 | func (l logger) Debugf(format string, args ...interface{}) { 59 | l.sourced().Debugf(format, args...) 60 | } 61 | 62 | func (l logger) Info(args ...interface{}) { 63 | l.sourced().Info(args...) 64 | } 65 | 66 | func (l logger) Infoln(args ...interface{}) { 67 | l.sourced().Infoln(args...) 68 | } 69 | 70 | func (l logger) Infof(format string, args ...interface{}) { 71 | l.sourced().Infof(format, args...) 72 | } 73 | 74 | func (l logger) Warn(args ...interface{}) { 75 | l.sourced().Warn(args...) 76 | } 77 | 78 | func (l logger) Warnln(args ...interface{}) { 79 | l.sourced().Warnln(args...) 80 | } 81 | 82 | func (l logger) Warnf(format string, args ...interface{}) { 83 | l.sourced().Warnf(format, args...) 84 | } 85 | 86 | func (l logger) Warningf(format string, args ...interface{}) { 87 | l.sourced().Warnf(format, args...) 88 | } 89 | 90 | func (l logger) Error(args ...interface{}) { 91 | l.sourced().Error(args...) 92 | } 93 | 94 | func (l logger) Errorln(args ...interface{}) { 95 | l.sourced().Errorln(args...) 96 | } 97 | 98 | func (l logger) Errorf(format string, args ...interface{}) { 99 | l.sourced().Errorf(format, args...) 100 | } 101 | 102 | func (l logger) Fatal(args ...interface{}) { 103 | l.sourced().Fatal(args...) 104 | } 105 | 106 | func (l logger) Fatalln(args ...interface{}) { 107 | l.sourced().Fatalln(args...) 108 | } 109 | 110 | func (l logger) Fatalf(format string, args ...interface{}) { 111 | l.sourced().Fatalf(format, args...) 112 | } 113 | func (l logger) Panic(args ...interface{}) { 114 | l.sourced().Panic(args...) 115 | } 116 | func (l logger) Panicln(args ...interface{}) { 117 | l.sourced().Panicln(args...) 118 | } 119 | func (l logger) Panicf(format string, args ...interface{}) { 120 | l.sourced().Panicf(format, args...) 121 | } 122 | 123 | func (l logger) With(key string, value interface{}) Logger { 124 | return &logger{l.entry.WithField(key, value)} 125 | } 126 | 127 | func (l logger) WithField(key string, value interface{}) Logger { 128 | return &logger{l.entry.WithField(key, value)} 129 | } 130 | 131 | func (l *logger) Set(level Level) error { 132 | leLev, err := logrus.ParseLevel(string(level)) 133 | if err != nil { 134 | return err 135 | } 136 | l.entry.Logger.Level = leLev 137 | return nil 138 | } 139 | 140 | func (l logger) sourced() *logrus.Entry { 141 | _, file, line, ok := runtime.Caller(3) 142 | if !ok { 143 | file = "" 144 | line = 1 145 | } else { 146 | slash := strings.LastIndex(file, "/") 147 | file = file[slash+1:] 148 | } 149 | return l.entry.WithField("src", fmt.Sprintf("%s:%d", file, line)) 150 | } 151 | 152 | var jsonBaseLogger = func() Logger { 153 | l := logrus.New() 154 | l.Formatter = &logrus.JSONFormatter{} 155 | return &logger{ 156 | entry: &logrus.Entry{ 157 | Logger: l, 158 | }, 159 | } 160 | }() 161 | 162 | var baseLogger = &logger{ 163 | entry: &logrus.Entry{ 164 | Logger: logrus.New(), 165 | }, 166 | } 167 | 168 | // Base returns the base logger 169 | func Base(json bool) Logger { 170 | if json { 171 | return jsonBaseLogger 172 | } 173 | return baseLogger 174 | } 175 | 176 | // Debug logs debug message 177 | func Debug(args ...interface{}) { 178 | baseLogger.sourced().Debug(args...) 179 | } 180 | 181 | // Debugln logs debug message 182 | func Debugln(args ...interface{}) { 183 | baseLogger.sourced().Debugln(args...) 184 | } 185 | 186 | // Debugf logs debug message 187 | func Debugf(format string, args ...interface{}) { 188 | baseLogger.sourced().Debugf(format, args...) 189 | } 190 | 191 | // Info logs info message 192 | func Info(args ...interface{}) { 193 | baseLogger.sourced().Info(args...) 194 | } 195 | 196 | // Infoln logs info message 197 | func Infoln(args ...interface{}) { 198 | baseLogger.sourced().Infoln(args...) 199 | } 200 | 201 | // Infof logs info message 202 | func Infof(format string, args ...interface{}) { 203 | baseLogger.sourced().Infof(format, args...) 204 | } 205 | 206 | // Warn logs warn message 207 | func Warn(args ...interface{}) { 208 | baseLogger.sourced().Warn(args...) 209 | } 210 | 211 | // Warnln logs warn message 212 | func Warnln(args ...interface{}) { 213 | baseLogger.sourced().Warnln(args...) 214 | } 215 | 216 | // Warnf logs warn message 217 | func Warnf(format string, args ...interface{}) { 218 | baseLogger.sourced().Warnf(format, args...) 219 | } 220 | 221 | // Error logs error message 222 | func Error(args ...interface{}) { 223 | baseLogger.sourced().Error(args...) 224 | } 225 | 226 | // Errorln logs error message 227 | func Errorln(args ...interface{}) { 228 | baseLogger.sourced().Errorln(args...) 229 | } 230 | 231 | // Errorf logs error message 232 | func Errorf(format string, args ...interface{}) { 233 | baseLogger.sourced().Errorf(format, args...) 234 | } 235 | 236 | // Fatal logs fatal message 237 | func Fatal(args ...interface{}) { 238 | baseLogger.sourced().Fatal(args...) 239 | } 240 | 241 | // Fatalln logs fatal message 242 | func Fatalln(args ...interface{}) { 243 | baseLogger.sourced().Fatalln(args...) 244 | } 245 | 246 | // Fatalf logs fatal message 247 | func Fatalf(format string, args ...interface{}) { 248 | baseLogger.sourced().Fatalf(format, args...) 249 | } 250 | 251 | // With adds a key:value to the logger 252 | func With(key string, value interface{}) Logger { 253 | return baseLogger.With(key, value) 254 | } 255 | 256 | // WithField adds a key:value to the logger 257 | func WithField(key string, value interface{}) Logger { 258 | return baseLogger.WithField(key, value) 259 | } 260 | 261 | // Set will set the logger level 262 | func Set(level Level) error { 263 | return baseLogger.Set(level) 264 | } 265 | 266 | // Panic logs panic message 267 | func Panic(args ...interface{}) { 268 | baseLogger.Panic(args...) 269 | } 270 | 271 | // Panicln logs panicln message 272 | func Panicln(args ...interface{}) { 273 | baseLogger.Panicln(args...) 274 | } 275 | 276 | // Panicf logs panicln message 277 | func Panicf(format string, args ...interface{}) { 278 | baseLogger.Panicf(format, args...) 279 | } 280 | -------------------------------------------------------------------------------- /pkg/operator/crd.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/apimachinery/pkg/watch" 7 | "k8s.io/client-go/tools/cache" 8 | 9 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 10 | "github.com/spotahome/service-level-operator/pkg/log" 11 | "github.com/spotahome/service-level-operator/pkg/service/kubernetes" 12 | ) 13 | 14 | // serviceLevelCRD is the crd release. 15 | type serviceLevelCRD struct { 16 | cfg Config 17 | service kubernetes.Service 18 | logger log.Logger 19 | } 20 | 21 | func newServiceLevelCRD(cfg Config, service kubernetes.Service, logger log.Logger) *serviceLevelCRD { 22 | logger = logger.With("crd", "servicelevel") 23 | return &serviceLevelCRD{ 24 | cfg: cfg, 25 | service: service, 26 | logger: logger, 27 | } 28 | } 29 | 30 | // Initialize satisfies resource.crd interface. 31 | func (s *serviceLevelCRD) Initialize() error { 32 | crd := kubernetes.CRDConf{ 33 | Kind: monitoringv1alpha1.ServiceLevelKind, 34 | NamePlural: monitoringv1alpha1.ServiceLevelNamePlural, 35 | Group: monitoringv1alpha1.SchemeGroupVersion.Group, 36 | Version: monitoringv1alpha1.SchemeGroupVersion.Version, 37 | Scope: monitoringv1alpha1.ServiceLevelScope, 38 | Categories: []string{"monitoring", "slo"}, 39 | EnableStatusSubresource: true, 40 | } 41 | 42 | return s.service.EnsurePresentCRD(crd) 43 | } 44 | 45 | // GetListerWatcher satisfies resource.crd interface (and retrieve.Retriever). 46 | func (s *serviceLevelCRD) GetListerWatcher() cache.ListerWatcher { 47 | return &cache.ListWatch{ 48 | ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 49 | options.LabelSelector = s.cfg.LabelSelector 50 | return s.service.ListServiceLevels(s.cfg.Namespace, options) 51 | }, 52 | WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 53 | options.LabelSelector = s.cfg.LabelSelector 54 | return s.service.WatchServiceLevels(s.cfg.Namespace, options) 55 | }, 56 | } 57 | } 58 | 59 | // GetObject satisfies resource.crd interface (and retrieve.Retriever). 60 | func (s *serviceLevelCRD) GetObject() runtime.Object { 61 | return &monitoringv1alpha1.ServiceLevel{} 62 | } 63 | -------------------------------------------------------------------------------- /pkg/operator/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | operator handles handles the measurement of the different SLIs and SLOs based on 3 | the metrics described in the CRDs and creating new metrics that will expose the 4 | current SLO measurements of the services. 5 | */ 6 | 7 | package operator 8 | -------------------------------------------------------------------------------- /pkg/operator/factory.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | kmetrics "github.com/spotahome/kooper/monitoring/metrics" 8 | "github.com/spotahome/kooper/operator" 9 | "github.com/spotahome/kooper/operator/controller" 10 | 11 | "github.com/spotahome/service-level-operator/pkg/log" 12 | promcli "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 13 | "github.com/spotahome/service-level-operator/pkg/service/kubernetes" 14 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 15 | "github.com/spotahome/service-level-operator/pkg/service/output" 16 | "github.com/spotahome/service-level-operator/pkg/service/sli" 17 | ) 18 | 19 | const ( 20 | operatorName = "service-level-operator" 21 | jobRetries = 3 22 | ) 23 | 24 | // Config is the configuration for the ci operator. 25 | type Config struct { 26 | // ResyncPeriod is the resync period of the controllers. 27 | ResyncPeriod time.Duration 28 | // ConcurretWorkers are number of workers to handle the events. 29 | ConcurretWorkers int 30 | // LabelSelector is the label selector to filter Kubernetes resources by labels. 31 | LabelSelector string 32 | // Namespace is the namespace to filter Kubernetes resources by a single namespace. 33 | Namespace string 34 | } 35 | 36 | // New returns pod terminator operator. 37 | func New(cfg Config, promreg *prometheus.Registry, promCliFactory promcli.ClientFactory, k8ssvc kubernetes.Service, metricssvc metrics.Service, logger log.Logger) (operator.Operator, error) { 38 | 39 | // Create crd. 40 | ptCRD := newServiceLevelCRD(cfg, k8ssvc, logger) 41 | 42 | // Create services. 43 | promRetriever := sli.NewPrometheus(promCliFactory, logger.WithField("sli-retriever", "prometheus")) 44 | retrieverFact := sli.NewRetrieverFactory( 45 | sli.NewMetricsMiddleware(metricssvc, "prometheus", promRetriever), 46 | ) 47 | 48 | promOutput := output.NewPrometheus(output.PrometheusCfg{}, promreg, logger.WithField("slo-output", "prometheus")) 49 | outputFact := output.NewFactory( 50 | output.NewMetricsMiddleware(metricssvc, "prometheus", promOutput), 51 | ) 52 | 53 | // Create handler. 54 | handler := NewHandler(outputFact, retrieverFact, logger) 55 | 56 | // Create controller. 57 | ctrlCfg := &controller.Config{ 58 | Name: operatorName, 59 | ConcurrentWorkers: cfg.ConcurretWorkers, 60 | ResyncInterval: cfg.ResyncPeriod, 61 | ProcessingJobRetries: jobRetries, 62 | } 63 | 64 | ctrl := controller.New( 65 | ctrlCfg, 66 | handler, 67 | ptCRD, 68 | nil, 69 | nil, 70 | kmetrics.NewPrometheus(promreg), 71 | logger) 72 | 73 | // Assemble CRD and controller to create the operator. 74 | return operator.NewOperator(ptCRD, ctrl, logger), nil 75 | } 76 | -------------------------------------------------------------------------------- /pkg/operator/handler.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "k8s.io/apimachinery/pkg/runtime" 9 | 10 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 11 | "github.com/spotahome/service-level-operator/pkg/log" 12 | "github.com/spotahome/service-level-operator/pkg/service/output" 13 | "github.com/spotahome/service-level-operator/pkg/service/sli" 14 | ) 15 | 16 | // Handler is the Operator handler. 17 | type Handler struct { 18 | outputerFact output.Factory 19 | retrieverFact sli.RetrieverFactory 20 | logger log.Logger 21 | } 22 | 23 | // NewHandler returns a new project handler 24 | func NewHandler(outputerFact output.Factory, retrieverFact sli.RetrieverFactory, logger log.Logger) *Handler { 25 | return &Handler{ 26 | outputerFact: outputerFact, 27 | retrieverFact: retrieverFact, 28 | logger: logger, 29 | } 30 | } 31 | 32 | // Add will ensure the the ci builds and jobs are persisted. 33 | func (h *Handler) Add(_ context.Context, obj runtime.Object) error { 34 | sl, ok := obj.(*monitoringv1alpha1.ServiceLevel) 35 | if !ok { 36 | return fmt.Errorf("can't handle received object, it's not a service level object") 37 | } 38 | 39 | slc := sl.DeepCopy() 40 | 41 | err := slc.Validate() 42 | if err != nil { 43 | return err 44 | } 45 | 46 | var wg sync.WaitGroup 47 | wg.Add(len(slc.Spec.ServiceLevelObjectives)) 48 | 49 | // Retrieve the SLIs. 50 | for _, slo := range slc.Spec.ServiceLevelObjectives { 51 | slo := slo 52 | 53 | go func() { 54 | defer wg.Done() 55 | err := h.processSLO(slc, &slo) 56 | // Don't stop if one of the SLOs errors, the rest should 57 | // be processed independently. 58 | if err != nil { 59 | h.logger.With("sl", sl.Name).With("slo", slo.Name).Errorf("error processing SLO: %s", err) 60 | } 61 | }() 62 | } 63 | 64 | wg.Wait() 65 | return nil 66 | } 67 | 68 | func (h *Handler) processSLO(sl *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO) error { 69 | if slo.Disable { 70 | h.logger.Debugf("ignoring SLO %s", slo.Name) 71 | return nil 72 | } 73 | 74 | retriever, err := h.retrieverFact.GetStrategy(&slo.ServiceLevelIndicator) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | res, err := retriever.Retrieve(&slo.ServiceLevelIndicator) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | outputer, err := h.outputerFact.GetStrategy(slo) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | err = outputer.Create(sl, slo, &res) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | return nil 95 | } 96 | 97 | // Delete handles the deletion of a release. 98 | func (h *Handler) Delete(_ context.Context, name string) error { 99 | h.logger.Debugf("delete received") 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /pkg/operator/handler_test.go: -------------------------------------------------------------------------------- 1 | package operator_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/mock" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | 11 | moutput "github.com/spotahome/service-level-operator/mocks/service/output" 12 | msli "github.com/spotahome/service-level-operator/mocks/service/sli" 13 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 14 | "github.com/spotahome/service-level-operator/pkg/log" 15 | "github.com/spotahome/service-level-operator/pkg/operator" 16 | "github.com/spotahome/service-level-operator/pkg/service/output" 17 | "github.com/spotahome/service-level-operator/pkg/service/sli" 18 | ) 19 | 20 | var ( 21 | sl0 = &monitoringv1alpha1.ServiceLevel{ 22 | ObjectMeta: metav1.ObjectMeta{ 23 | Name: "fake-service0", 24 | Namespace: "fake", 25 | }, 26 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 27 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 28 | { 29 | Name: "slo0", 30 | AvailabilityObjectivePercent: 99.99, 31 | Disable: true, 32 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 33 | SLISource: monitoringv1alpha1.SLISource{ 34 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 35 | Address: "http://127.0.0.1:9090", 36 | TotalQuery: `sum(increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[5m]))`, 37 | ErrorQuery: `sum(increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com", code=~"5.."}[5m]))`, 38 | }, 39 | }, 40 | }, 41 | Output: monitoringv1alpha1.Output{ 42 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 43 | }, 44 | }, 45 | }, 46 | }, 47 | } 48 | 49 | sl1 = &monitoringv1alpha1.ServiceLevel{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Name: "fake-service0", 52 | Namespace: "fake", 53 | }, 54 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 55 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 56 | { 57 | Name: "slo0", 58 | AvailabilityObjectivePercent: 99.95, 59 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 60 | SLISource: monitoringv1alpha1.SLISource{ 61 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 62 | }, 63 | }, 64 | Output: monitoringv1alpha1.Output{ 65 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 66 | }, 67 | }, 68 | { 69 | Name: "slo1", 70 | AvailabilityObjectivePercent: 99.99, 71 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 72 | SLISource: monitoringv1alpha1.SLISource{ 73 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 74 | }, 75 | }, 76 | Output: monitoringv1alpha1.Output{ 77 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 78 | }, 79 | }, 80 | { 81 | Name: "slo2", 82 | AvailabilityObjectivePercent: 99.9, 83 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 84 | SLISource: monitoringv1alpha1.SLISource{ 85 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 86 | }, 87 | }, 88 | Output: monitoringv1alpha1.Output{ 89 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 90 | }, 91 | }, 92 | { 93 | Name: "slo3", 94 | AvailabilityObjectivePercent: 99.9999, 95 | Disable: true, 96 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 97 | SLISource: monitoringv1alpha1.SLISource{ 98 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 99 | }, 100 | }, 101 | Output: monitoringv1alpha1.Output{ 102 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 103 | }, 104 | }, 105 | }, 106 | }, 107 | } 108 | ) 109 | 110 | func TestHandler(t *testing.T) { 111 | tests := []struct { 112 | name string 113 | serviceLevel *monitoringv1alpha1.ServiceLevel 114 | processTimes int 115 | expErr bool 116 | }{ 117 | { 118 | name: "With disabled SLO should not process anything.", 119 | serviceLevel: sl0, 120 | processTimes: 0, 121 | expErr: false, 122 | }, 123 | { 124 | name: "A service level with multiple slos should process all slos.", 125 | serviceLevel: sl1, 126 | processTimes: 3, 127 | expErr: false, 128 | }, 129 | } 130 | 131 | for _, test := range tests { 132 | t.Run(test.name, func(t *testing.T) { 133 | assert := assert.New(t) 134 | 135 | // Mocks. 136 | mout := &moutput.Output{} 137 | moutf := output.MockFactory{Mock: mout} 138 | mret := &msli.Retriever{} 139 | mretf := sli.MockRetrieverFactory{Mock: mret} 140 | 141 | if test.processTimes > 0 { 142 | mout.On("Create", mock.Anything, mock.Anything, mock.Anything).Times(test.processTimes).Return(nil) 143 | mret.On("Retrieve", mock.Anything).Times(test.processTimes).Return(sli.Result{}, nil) 144 | } 145 | 146 | h := operator.NewHandler(moutf, mretf, log.Dummy) 147 | err := h.Add(context.Background(), test.serviceLevel) 148 | 149 | if test.expErr { 150 | assert.Error(err) 151 | } else if assert.NoError(err) { 152 | mout.AssertExpectations(t) 153 | mret.AssertExpectations(t) 154 | } 155 | }) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/service/client/kubernetes/factory.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 5 | "k8s.io/client-go/kubernetes" 6 | "k8s.io/client-go/rest" 7 | 8 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 9 | ) 10 | 11 | // ClientFactory knows how to get Kubernetes clients. 12 | type ClientFactory interface { 13 | // GetSTDClient gets the Kubernetes standard client (pods, services...). 14 | GetSTDClient() (kubernetes.Interface, error) 15 | // GetCRDClient gets the Kubernetes client for the CRDs described in this application. 16 | GetCRDClient() (crdcli.Interface, error) 17 | // GetAPIExtensionClient gets the Kubernetes api extensions client (crds...). 18 | GetAPIExtensionClient() (apiextensionscli.Interface, error) 19 | } 20 | 21 | type factory struct { 22 | restCfg *rest.Config 23 | 24 | stdcli kubernetes.Interface 25 | crdcli crdcli.Interface 26 | aexcli apiextensionscli.Interface 27 | } 28 | 29 | // NewFactory returns a new kubernetes client factory. 30 | func NewFactory(config *rest.Config) ClientFactory { 31 | return &factory{ 32 | restCfg: config, 33 | } 34 | } 35 | 36 | func (f *factory) GetSTDClient() (kubernetes.Interface, error) { 37 | if f.stdcli == nil { 38 | cli, err := kubernetes.NewForConfig(f.restCfg) 39 | if err != nil { 40 | return nil, err 41 | } 42 | f.stdcli = cli 43 | } 44 | return f.stdcli, nil 45 | } 46 | func (f *factory) GetCRDClient() (crdcli.Interface, error) { 47 | if f.crdcli == nil { 48 | cli, err := crdcli.NewForConfig(f.restCfg) 49 | if err != nil { 50 | return nil, err 51 | } 52 | f.crdcli = cli 53 | } 54 | return f.crdcli, nil 55 | } 56 | func (f *factory) GetAPIExtensionClient() (apiextensionscli.Interface, error) { 57 | if f.aexcli == nil { 58 | cli, err := apiextensionscli.NewForConfig(f.restCfg) 59 | if err != nil { 60 | return nil, err 61 | } 62 | f.aexcli = cli 63 | } 64 | return f.aexcli, nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/service/client/kubernetes/fake.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 5 | apiextensionsclifake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "k8s.io/apimachinery/pkg/version" 9 | fakediscovery "k8s.io/client-go/discovery/fake" 10 | "k8s.io/client-go/kubernetes" 11 | kubernetesfake "k8s.io/client-go/kubernetes/fake" 12 | 13 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 14 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 15 | crdclifake "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/fake" 16 | ) 17 | 18 | // fakeFactory is a fake factory that has already loaded faked objects on the Kubernetes clients. 19 | type fakeFactory struct{} 20 | 21 | // NewFake returns the faked Kubernetes clients factory. 22 | func NewFake() ClientFactory { 23 | return &fakeFactory{} 24 | } 25 | 26 | func (f *fakeFactory) GetSTDClient() (kubernetes.Interface, error) { 27 | return kubernetesfake.NewSimpleClientset(stdObjs...), nil 28 | } 29 | func (f *fakeFactory) GetCRDClient() (crdcli.Interface, error) { 30 | return crdclifake.NewSimpleClientset(crdObjs...), nil 31 | } 32 | func (f *fakeFactory) GetAPIExtensionClient() (apiextensionscli.Interface, error) { 33 | cli := apiextensionsclifake.NewSimpleClientset(aexObjs...) 34 | 35 | // Fake cluster version (Required for CRD version checks). 36 | fakeDiscovery, _ := cli.Discovery().(*fakediscovery.FakeDiscovery) 37 | fakeDiscovery.FakedServerVersion = &version.Info{ 38 | GitVersion: "v1.10.5", 39 | } 40 | 41 | return cli, nil 42 | } 43 | 44 | var ( 45 | stdObjs = []runtime.Object{} 46 | 47 | // The field selector doesn't work with a fake K8s client: https://github.com/kubernetes/client-go/issues/326 48 | crdObjs = []runtime.Object{ 49 | &monitoringv1alpha1.ServiceLevel{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Name: "fake-service0", 52 | Namespace: "ns0", 53 | Labels: map[string]string{ 54 | "wrong": "false", 55 | }, 56 | }, 57 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 58 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 59 | { 60 | Name: "fake_slo0", 61 | Description: "fake slo 0.", 62 | AvailabilityObjectivePercent: 99.99, 63 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 64 | SLISource: monitoringv1alpha1.SLISource{ 65 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 66 | Address: "http://fake:9090", 67 | TotalQuery: `slo0_total`, 68 | ErrorQuery: `slo0_error`, 69 | }, 70 | }, 71 | }, 72 | Output: monitoringv1alpha1.Output{ 73 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 74 | Labels: map[string]string{ 75 | "fake": "true", 76 | "team": "fake-team0", 77 | }, 78 | }, 79 | }, 80 | }, 81 | { 82 | Name: "fake_slo1", 83 | Description: "fake slo 1.", 84 | AvailabilityObjectivePercent: 99.9, 85 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 86 | SLISource: monitoringv1alpha1.SLISource{ 87 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 88 | Address: "http://fake:9090", 89 | TotalQuery: `slo1_total`, 90 | ErrorQuery: `slo1_error`, 91 | }, 92 | }, 93 | }, 94 | Output: monitoringv1alpha1.Output{ 95 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 96 | Labels: map[string]string{ 97 | "fake": "true", 98 | "team": "fake-team1", 99 | }, 100 | }, 101 | }, 102 | }, 103 | { 104 | Name: "fake_slo2", 105 | Description: "fake slo 2.", 106 | AvailabilityObjectivePercent: 99.998, 107 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 108 | SLISource: monitoringv1alpha1.SLISource{ 109 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 110 | Address: "http://fake:9090", 111 | TotalQuery: `slo2_total`, 112 | ErrorQuery: `slo2_error`, 113 | }, 114 | }, 115 | }, 116 | Output: monitoringv1alpha1.Output{ 117 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 118 | Labels: map[string]string{ 119 | "fake": "true", 120 | "team": "fake-team2", 121 | }, 122 | }, 123 | }, 124 | }, 125 | }, 126 | }, 127 | }, 128 | &monitoringv1alpha1.ServiceLevel{ 129 | ObjectMeta: metav1.ObjectMeta{ 130 | Name: "fake-service1", 131 | Namespace: "ns1", 132 | Labels: map[string]string{ 133 | "wrong": "false", 134 | }, 135 | }, 136 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 137 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 138 | { 139 | Name: "fake_slo3", 140 | Description: "fake slo 3.", 141 | AvailabilityObjectivePercent: 99, 142 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 143 | SLISource: monitoringv1alpha1.SLISource{ 144 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 145 | Address: "http://fake:9090", 146 | TotalQuery: `slo3_total`, 147 | ErrorQuery: `slo3_error`, 148 | }, 149 | }, 150 | }, 151 | Output: monitoringv1alpha1.Output{ 152 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 153 | Labels: map[string]string{ 154 | "fake": "true", 155 | "team": "fake-team3", 156 | }, 157 | }, 158 | }, 159 | }, 160 | }, 161 | }, 162 | }, 163 | &monitoringv1alpha1.ServiceLevel{ 164 | ObjectMeta: metav1.ObjectMeta{ 165 | Name: "fake-service2-no-output", 166 | Namespace: "ns0", 167 | Labels: map[string]string{ 168 | "wrong": "true", 169 | }, 170 | }, 171 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 172 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 173 | { 174 | Name: "fake_slo4", 175 | Description: "fake slo 4.", 176 | AvailabilityObjectivePercent: 99, 177 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 178 | SLISource: monitoringv1alpha1.SLISource{ 179 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 180 | Address: "http://fake:9090", 181 | TotalQuery: `slo3_total`, 182 | ErrorQuery: `slo3_error`, 183 | }, 184 | }, 185 | }, 186 | Output: monitoringv1alpha1.Output{}, 187 | }, 188 | }, 189 | }, 190 | }, 191 | 192 | &monitoringv1alpha1.ServiceLevel{ 193 | ObjectMeta: metav1.ObjectMeta{ 194 | Name: "fake-service3-no-input", 195 | Namespace: "ns1", 196 | Labels: map[string]string{ 197 | "wrong": "true", 198 | }, 199 | }, 200 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 201 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 202 | { 203 | Name: "fake_slo5", 204 | Description: "fake slo 5.", 205 | AvailabilityObjectivePercent: 99, 206 | ServiceLevelIndicator: monitoringv1alpha1.SLI{}, 207 | Output: monitoringv1alpha1.Output{ 208 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 209 | Labels: map[string]string{ 210 | "wrong": "true", 211 | }, 212 | }, 213 | }, 214 | }, 215 | }, 216 | }, 217 | }, 218 | } 219 | 220 | aexObjs = []runtime.Object{} 221 | ) 222 | -------------------------------------------------------------------------------- /pkg/service/client/prometheus/factory.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/prometheus/client_golang/api" 8 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 9 | ) 10 | 11 | // ClientFactory knows how to get prometheus API clients. 12 | type ClientFactory interface { 13 | // GetV1APIClient returns a new prometheus v1 API client. 14 | // address is the address of the prometheus. 15 | GetV1APIClient(address string) (promv1.API, error) 16 | } 17 | 18 | // BaseFactory returns Prometheus clients based on the address. 19 | // This factory implements a way of returning default Prometheus 20 | // clients in case it was set. 21 | type BaseFactory struct { 22 | v1Clis map[string]api.Client 23 | climu sync.Mutex 24 | } 25 | 26 | // NewBaseFactory returns a new client Basefactory. 27 | func NewBaseFactory() *BaseFactory { 28 | return &BaseFactory{ 29 | v1Clis: map[string]api.Client{}, 30 | } 31 | } 32 | 33 | // GetV1APIClient satisfies ClientFactory interface. 34 | func (f *BaseFactory) GetV1APIClient(address string) (promv1.API, error) { 35 | f.climu.Lock() 36 | defer f.climu.Unlock() 37 | 38 | var err error 39 | cli, ok := f.v1Clis[address] 40 | if !ok { 41 | cli, err = newClient(address) 42 | if err != nil { 43 | return nil, fmt.Errorf("error creating prometheus client: %s", err) 44 | } 45 | f.v1Clis[address] = cli 46 | } 47 | return promv1.NewAPI(cli), nil 48 | } 49 | 50 | // WithDefaultV1APIClient sets a default client for V1 api client. 51 | func (f *BaseFactory) WithDefaultV1APIClient(address string) error { 52 | const defAddressKey = "" 53 | f.climu.Lock() 54 | defer f.climu.Unlock() 55 | 56 | dc, err := newClient(address) 57 | if err != nil { 58 | return fmt.Errorf("error creating prometheus client: %s", err) 59 | } 60 | f.v1Clis[defAddressKey] = dc 61 | 62 | return nil 63 | } 64 | 65 | func newClient(address string) (api.Client, error) { 66 | if address == "" { 67 | return nil, fmt.Errorf("address can't be empty") 68 | } 69 | 70 | return api.NewClient(api.Config{Address: address}) 71 | } 72 | 73 | // MockFactory returns a predefined prometheus v1 API client. 74 | type MockFactory struct { 75 | Cli promv1.API 76 | } 77 | 78 | // GetV1APIClient satisfies ClientFactory interface. 79 | func (m *MockFactory) GetV1APIClient(_ string) (promv1.API, error) { 80 | return m.Cli, nil 81 | } 82 | -------------------------------------------------------------------------------- /pkg/service/client/prometheus/factory_test.go: -------------------------------------------------------------------------------- 1 | package prometheus_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 9 | ) 10 | 11 | func TestBaseFactoryV1Client(t *testing.T) { 12 | tests := map[string]struct { 13 | cli func() *prometheus.BaseFactory 14 | address string 15 | expErr bool 16 | }{ 17 | "A regular client address should be returned without error.": { 18 | cli: func() *prometheus.BaseFactory { 19 | return prometheus.NewBaseFactory() 20 | }, 21 | address: "http://127.0.0.1:9090", 22 | }, 23 | 24 | "Getting a missing address client should error.": { 25 | cli: func() *prometheus.BaseFactory { 26 | return prometheus.NewBaseFactory() 27 | }, 28 | address: "", 29 | expErr: true, 30 | }, 31 | 32 | "Getting a missing address client with a default client it should not error.": { 33 | cli: func() *prometheus.BaseFactory { 34 | f := prometheus.NewBaseFactory() 35 | f.WithDefaultV1APIClient("http://127.0.0.1:9090") 36 | return f 37 | }, 38 | address: "", 39 | expErr: false, 40 | }, 41 | } 42 | 43 | for name, test := range tests { 44 | t.Run(name, func(t *testing.T) { 45 | assert := assert.New(t) 46 | 47 | f := test.cli() 48 | _, err := f.GetV1APIClient(test.address) 49 | 50 | if test.expErr { 51 | assert.Error(err) 52 | } else { 53 | assert.NoError(err) 54 | } 55 | }) 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /pkg/service/client/prometheus/fake.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/api" 9 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 10 | "github.com/prometheus/common/model" 11 | ) 12 | 13 | var ( 14 | slo3CallCount int 15 | ) 16 | 17 | type fakeFactory struct { 18 | } 19 | 20 | // NewFakeFactory returns a new fake factory. 21 | func NewFakeFactory() ClientFactory { 22 | return &fakeFactory{} 23 | } 24 | 25 | // GetV1APIClient satisfies ClientFactory interface. 26 | func (f *fakeFactory) GetV1APIClient(_ string) (promv1.API, error) { 27 | return &fakeAPICli{ 28 | queryFuncs: map[string]func() float64{ 29 | "slo0_total": func() float64 { return 100 }, 30 | "slo0_error": func() float64 { return 1 }, 31 | "slo1_total": func() float64 { return 1000 }, 32 | "slo1_error": func() float64 { return 1 }, 33 | "slo2_total": func() float64 { return 100000 }, 34 | "slo2_error": func() float64 { return 12 }, 35 | "slo3_total": func() float64 { return 10000 }, 36 | "slo3_error": func() float64 { 37 | // Every 2 calls return error. 38 | slo3CallCount++ 39 | if slo3CallCount%2 == 0 { 40 | return 1 41 | } 42 | return 0 43 | }, 44 | }, 45 | }, nil 46 | } 47 | 48 | // fakeAPICli is a faked http client. 49 | type fakeAPICli struct { 50 | queryFuncs map[string]func() float64 51 | } 52 | 53 | func (f *fakeAPICli) Query(_ context.Context, query string, ts time.Time) (model.Value, api.Warnings, error) { 54 | 55 | fn, ok := f.queryFuncs[query] 56 | if !ok { 57 | return nil, nil, fmt.Errorf("not faked result") 58 | } 59 | 60 | return model.Vector{ 61 | &model.Sample{ 62 | Metric: model.Metric{}, 63 | Timestamp: model.Time(time.Now().UTC().Nanosecond()), 64 | Value: model.SampleValue(fn()), 65 | }, 66 | }, nil, nil 67 | } 68 | 69 | func (f *fakeAPICli) Alerts(ctx context.Context) (promv1.AlertsResult, error) { 70 | return promv1.AlertsResult{}, nil 71 | } 72 | func (f *fakeAPICli) AlertManagers(_ context.Context) (promv1.AlertManagersResult, error) { 73 | return promv1.AlertManagersResult{}, nil 74 | } 75 | func (f *fakeAPICli) CleanTombstones(_ context.Context) error { 76 | return nil 77 | } 78 | func (f *fakeAPICli) Config(_ context.Context) (promv1.ConfigResult, error) { 79 | return promv1.ConfigResult{}, nil 80 | } 81 | func (f *fakeAPICli) DeleteSeries(_ context.Context, matches []string, startTime time.Time, endTime time.Time) error { 82 | return nil 83 | } 84 | func (f *fakeAPICli) Flags(_ context.Context) (promv1.FlagsResult, error) { 85 | return promv1.FlagsResult{}, nil 86 | } 87 | func (f *fakeAPICli) LabelNames(ctx context.Context) ([]string, api.Warnings, error) { 88 | return nil, nil, nil 89 | } 90 | func (f *fakeAPICli) LabelValues(_ context.Context, label string) (model.LabelValues, api.Warnings, error) { 91 | return model.LabelValues{}, nil, nil 92 | } 93 | func (f *fakeAPICli) QueryRange(_ context.Context, query string, r promv1.Range) (model.Value, api.Warnings, error) { 94 | return nil, nil, nil 95 | } 96 | func (f *fakeAPICli) Series(_ context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, api.Warnings, error) { 97 | return []model.LabelSet{}, nil, nil 98 | } 99 | func (f *fakeAPICli) Snapshot(_ context.Context, skipHead bool) (promv1.SnapshotResult, error) { 100 | return promv1.SnapshotResult{}, nil 101 | } 102 | func (f *fakeAPICli) Rules(ctx context.Context) (promv1.RulesResult, error) { 103 | return promv1.RulesResult{}, nil 104 | } 105 | func (f *fakeAPICli) Targets(_ context.Context) (promv1.TargetsResult, error) { 106 | return promv1.TargetsResult{}, nil 107 | } 108 | func (f *fakeAPICli) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]promv1.MetricMetadata, error) { 109 | return nil, nil 110 | } 111 | -------------------------------------------------------------------------------- /pkg/service/configuration/configuration.go: -------------------------------------------------------------------------------- 1 | package configuration 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "io/ioutil" 8 | ) 9 | 10 | // DefaultSLISource is a configuration object with the default 11 | // endpoints. 12 | type DefaultSLISource struct { 13 | Prometheus PrometheusSLISource `json:"prometheus,omitempty"` 14 | } 15 | 16 | // PrometheusSLISource is the default prometheus source. 17 | type PrometheusSLISource struct { 18 | Address string `json:"address,omitempty"` 19 | } 20 | 21 | 22 | // Loader knows how to load configuration based on different formats. 23 | // At this moment configuration is not versioned, the configuration 24 | // is so simple that if it grows we could refactor and add version, 25 | // in this case not versioned configuration could be loaded as v1. 26 | type Loader interface { 27 | // LoadDefaultSLISource will load the default sli source configuration . 28 | LoadDefaultSLISource(ctx context.Context, r io.Reader) (*DefaultSLISource, error) 29 | } 30 | 31 | // JSONLoader knows how to load application configuration. 32 | type JSONLoader struct{} 33 | 34 | // LoadDefaultSLISource satisfies Loader interface by loading in JSON format. 35 | func (j JSONLoader) LoadDefaultSLISource(_ context.Context, r io.Reader) (*DefaultSLISource, error) { 36 | bs, err := ioutil.ReadAll(r) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | cfg := &DefaultSLISource{} 42 | err = json.Unmarshal(bs, cfg) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | return cfg, nil 48 | } 49 | -------------------------------------------------------------------------------- /pkg/service/configuration/configuration_test.go: -------------------------------------------------------------------------------- 1 | package configuration_test 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/spotahome/service-level-operator/pkg/service/configuration" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestJSONLoaderLoadDefaultSLISource(t *testing.T) { 13 | tests := map[string]struct { 14 | jsonConfig string 15 | expConfig *configuration.DefaultSLISource 16 | expErr bool 17 | }{ 18 | "Correct JSON configuration should be loaded without error.": { 19 | jsonConfig: `{"prometheus": {"address": "http://test:9090"}}`, 20 | expConfig: &configuration.DefaultSLISource{ 21 | Prometheus: configuration.PrometheusSLISource{ 22 | Address: "http://test:9090", 23 | }, 24 | }, 25 | }, 26 | 27 | "A malformed JSON should error.": { 28 | jsonConfig: `{"prometheus":`, 29 | expErr: true, 30 | }, 31 | } 32 | 33 | for name, test := range tests { 34 | t.Run(name, func(t *testing.T) { 35 | assert := assert.New(t) 36 | 37 | r := strings.NewReader(test.jsonConfig) 38 | gotConfig, err := configuration.JSONLoader{}.LoadDefaultSLISource(context.TODO(), r) 39 | if test.expErr { 40 | assert.Error(err) 41 | } else if assert.NoError(err) { 42 | assert.Equal(test.expConfig, gotConfig) 43 | } 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/service/kubernetes/crd.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | koopercrd "github.com/spotahome/kooper/client/crd" 5 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 6 | 7 | "github.com/spotahome/service-level-operator/pkg/log" 8 | ) 9 | 10 | // CRDConf is the configuration of the crd. 11 | type CRDConf = koopercrd.Conf 12 | 13 | // CRD is the CRD service that knows how to interact with k8s to manage them. 14 | type CRD interface { 15 | // EnsurePresentCRD will create the custom resource and wait to be ready 16 | // if there is not already present. 17 | EnsurePresentCRD(conf CRDConf) error 18 | } 19 | 20 | // crdService is the CRD service implementation using API calls to kubernetes. 21 | type crd struct { 22 | crdCli koopercrd.Interface 23 | logger log.Logger 24 | } 25 | 26 | // NewCRD returns a new CRD KubeService. 27 | func NewCRD(aeClient apiextensionscli.Interface, logger log.Logger) CRD { 28 | logger = logger.With("service", "k8s.crd") 29 | crdCli := koopercrd.NewClient(aeClient, logger) 30 | 31 | return &crd{ 32 | crdCli: crdCli, 33 | logger: logger, 34 | } 35 | } 36 | 37 | // EnsurePresentCRD satisfies workspace.Service interface. 38 | func (c *crd) EnsurePresentCRD(conf CRDConf) error { 39 | return c.crdCli.EnsurePresent(conf) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/service/kubernetes/kubernetes.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 5 | "k8s.io/client-go/kubernetes" 6 | 7 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 8 | "github.com/spotahome/service-level-operator/pkg/log" 9 | ) 10 | 11 | // Service is the service used to interact with the Kubernetes 12 | // objects. 13 | type Service interface { 14 | ServiceLevel 15 | CRD 16 | } 17 | 18 | type service struct { 19 | ServiceLevel 20 | CRD 21 | } 22 | 23 | // New returns a new Kubernetes service. 24 | func New(stdcli kubernetes.Interface, crdcli crdcli.Interface, apiextcli apiextensionscli.Interface, logger log.Logger) Service { 25 | return &service{ 26 | ServiceLevel: NewServiceLevel(crdcli, logger), 27 | CRD: NewCRD(apiextcli, logger), 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/service/kubernetes/servicelevel.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/watch" 6 | 7 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 8 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 9 | "github.com/spotahome/service-level-operator/pkg/log" 10 | ) 11 | 12 | // ServiceLevel knows how to interact with Kubernetes on the 13 | // ServiceLevel CRs 14 | type ServiceLevel interface { 15 | // ListServiceLevels will list the service levels. 16 | ListServiceLevels(namespace string, opts metav1.ListOptions) (*monitoringv1alpha1.ServiceLevelList, error) 17 | // ListServiceLevels will list the service levels. 18 | WatchServiceLevels(namespace string, opt metav1.ListOptions) (watch.Interface, error) 19 | } 20 | 21 | type serviceLevel struct { 22 | cli crdcli.Interface 23 | logger log.Logger 24 | } 25 | 26 | // NewServiceLevel returns a new service level service. 27 | func NewServiceLevel(crdcli crdcli.Interface, logger log.Logger) ServiceLevel { 28 | return &serviceLevel{ 29 | cli: crdcli, 30 | logger: logger, 31 | } 32 | } 33 | 34 | func (s *serviceLevel) ListServiceLevels(namespace string, opts metav1.ListOptions) (*monitoringv1alpha1.ServiceLevelList, error) { 35 | return s.cli.MonitoringV1alpha1().ServiceLevels(namespace).List(opts) 36 | } 37 | func (s *serviceLevel) WatchServiceLevels(namespace string, opts metav1.ListOptions) (watch.Interface, error) { 38 | return s.cli.MonitoringV1alpha1().ServiceLevels(namespace).Watch(opts) 39 | } 40 | -------------------------------------------------------------------------------- /pkg/service/metrics/dummy.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Dummy is a Dummy implementation of the metrics service. 10 | var Dummy = &dummy{} 11 | 12 | type dummy struct{} 13 | 14 | func (dummy) ObserveSLIRetrieveDuration(_ *monitoringv1alpha1.SLI, _ string, startTime time.Time) {} 15 | func (dummy) IncSLIRetrieveError(_ *monitoringv1alpha1.SLI, _ string) {} 16 | func (dummy) ObserveOuputCreateDuration(_ *monitoringv1alpha1.SLO, _ string, startTime time.Time) {} 17 | func (dummy) IncOuputCreateError(_ *monitoringv1alpha1.SLO, _ string) {} 18 | -------------------------------------------------------------------------------- /pkg/service/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Service knows how to monitoring the different parts, flows and processes 10 | // of the application to give more insights and improve the observability 11 | // of the application. 12 | type Service interface { 13 | // ObserveSLIRetrieveDuration will monitoring the duration of the process of gathering the group of 14 | // SLIs for a SLO. 15 | ObserveSLIRetrieveDuration(sli *monitoringv1alpha1.SLI, kind string, startTime time.Time) 16 | // IncSLIRetrieveError will increment the number of errors on the retrieval of the SLIs. 17 | IncSLIRetrieveError(sli *monitoringv1alpha1.SLI, kind string) 18 | // ObserveOuputCreateDuration monitorings the duration of the process of creating the output for the SLO 19 | ObserveOuputCreateDuration(slo *monitoringv1alpha1.SLO, kind string, startTime time.Time) 20 | // IncOuputCreateError will increment the number of errors on the SLO output creation. 21 | IncOuputCreateError(slo *monitoringv1alpha1.SLO, kind string) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/service/metrics/prometheus.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 8 | ) 9 | 10 | const ( 11 | promNamespace = "service_level" 12 | promSubsystem = "processing" 13 | ) 14 | 15 | var ( 16 | buckets = prometheus.DefBuckets 17 | ) 18 | 19 | type prometheusService struct { 20 | sliRetrieveHistogram *prometheus.HistogramVec 21 | sliRetrieveErrCounter *prometheus.CounterVec 22 | outputCreateHistogram *prometheus.HistogramVec 23 | outputCreateErrCounter *prometheus.CounterVec 24 | 25 | reg prometheus.Registerer 26 | } 27 | 28 | // NewPrometheus returns a new metrics.Service implementation that 29 | // knows how to monitor gusing Prometheus as backend. 30 | func NewPrometheus(reg prometheus.Registerer) Service { 31 | p := &prometheusService{ 32 | sliRetrieveHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 33 | Namespace: promNamespace, 34 | Subsystem: promSubsystem, 35 | Name: "sli_retrieve_duration_seconds", 36 | Help: "The duration seconds to retrieve the SLIs.", 37 | Buckets: buckets, 38 | }, []string{"kind"}), 39 | 40 | sliRetrieveErrCounter: prometheus.NewCounterVec(prometheus.CounterOpts{ 41 | Namespace: promNamespace, 42 | Subsystem: promSubsystem, 43 | Name: "sli_retrieve_failures_total", 44 | Help: "Total number sli retrieval failures.", 45 | }, []string{"kind"}), 46 | 47 | outputCreateHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 48 | Namespace: promNamespace, 49 | Subsystem: promSubsystem, 50 | Name: "output_create_duration_seconds", 51 | Help: "The duration seconds to create the output of the SLI and SLO results.", 52 | Buckets: buckets, 53 | }, []string{"kind"}), 54 | 55 | outputCreateErrCounter: prometheus.NewCounterVec(prometheus.CounterOpts{ 56 | Namespace: promNamespace, 57 | Subsystem: promSubsystem, 58 | Name: "output_create_failures_total", 59 | Help: "Total number SLI and SLO output creation failures.", 60 | }, []string{"kind"}), 61 | 62 | reg: reg, 63 | } 64 | 65 | p.registerMetrics() 66 | 67 | return p 68 | } 69 | 70 | func (p prometheusService) registerMetrics() { 71 | p.reg.MustRegister( 72 | p.sliRetrieveHistogram, 73 | p.sliRetrieveErrCounter, 74 | p.outputCreateHistogram, 75 | p.outputCreateErrCounter, 76 | ) 77 | } 78 | 79 | // ObserveSLIRetrieveDuration satisfies metrics.Service interface. 80 | func (p prometheusService) ObserveSLIRetrieveDuration(_ *monitoringv1alpha1.SLI, kind string, startTime time.Time) { 81 | p.sliRetrieveHistogram.WithLabelValues(kind).Observe(time.Since(startTime).Seconds()) 82 | } 83 | 84 | // IncSLIRetrieveError satisfies metrics.Service interface. 85 | func (p prometheusService) IncSLIRetrieveError(_ *monitoringv1alpha1.SLI, kind string) { 86 | p.sliRetrieveErrCounter.WithLabelValues(kind).Inc() 87 | } 88 | 89 | // ObserveOuputCreateDuration satisfies metrics.Service interface. 90 | func (p prometheusService) ObserveOuputCreateDuration(_ *monitoringv1alpha1.SLO, kind string, startTime time.Time) { 91 | p.outputCreateHistogram.WithLabelValues(kind).Observe(time.Since(startTime).Seconds()) 92 | } 93 | 94 | // IncOuputCreateError satisfies metrics.Service interface. 95 | func (p prometheusService) IncOuputCreateError(_ *monitoringv1alpha1.SLO, kind string) { 96 | p.outputCreateErrCounter.WithLabelValues(kind).Inc() 97 | } 98 | -------------------------------------------------------------------------------- /pkg/service/metrics/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package metrics_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | "github.com/stretchr/testify/assert" 12 | 13 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 14 | ) 15 | 16 | func TestPrometheusMetrics(t *testing.T) { 17 | kind := "test" 18 | 19 | tests := []struct { 20 | name string 21 | addMetrics func(metrics.Service) 22 | expMetrics []string 23 | expCode int 24 | }{ 25 | { 26 | name: "Measuring SLO realted metrics should expose SLO processing metrics on the prometheus endpoint.", 27 | addMetrics: func(s metrics.Service) { 28 | now := time.Now() 29 | s.IncOuputCreateError(nil, kind) 30 | s.IncOuputCreateError(nil, kind) 31 | s.ObserveOuputCreateDuration(nil, kind, now.Add(-6*time.Second)) 32 | s.ObserveOuputCreateDuration(nil, kind, now.Add(-27*time.Millisecond)) 33 | }, 34 | expMetrics: []string{ 35 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.005"} 0`, 36 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.01"} 0`, 37 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.025"} 0`, 38 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.05"} 1`, 39 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.1"} 1`, 40 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.25"} 1`, 41 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.5"} 1`, 42 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="1"} 1`, 43 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="2.5"} 1`, 44 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="5"} 1`, 45 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="10"} 2`, 46 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="+Inf"} 2`, 47 | `service_level_processing_output_create_duration_seconds_count{kind="test"} 2`, 48 | 49 | `service_level_processing_output_create_failures_total{kind="test"} 2`, 50 | }, 51 | expCode: 200, 52 | }, 53 | { 54 | name: "Measuring SLI realted metrics should expose SLI processing metrics on the prometheus endpoint.", 55 | addMetrics: func(s metrics.Service) { 56 | now := time.Now() 57 | s.IncSLIRetrieveError(nil, kind) 58 | s.IncSLIRetrieveError(nil, kind) 59 | s.IncSLIRetrieveError(nil, kind) 60 | s.ObserveSLIRetrieveDuration(nil, kind, now.Add(-3*time.Second)) 61 | s.ObserveSLIRetrieveDuration(nil, kind, now.Add(-15*time.Millisecond)) 62 | s.ObserveSLIRetrieveDuration(nil, kind, now.Add(-567*time.Millisecond)) 63 | }, 64 | expMetrics: []string{ 65 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.005"} 0`, 66 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.01"} 0`, 67 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.025"} 1`, 68 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.05"} 1`, 69 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.1"} 1`, 70 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.25"} 1`, 71 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.5"} 1`, 72 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="1"} 2`, 73 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="2.5"} 2`, 74 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="5"} 3`, 75 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="10"} 3`, 76 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="+Inf"} 3`, 77 | `service_level_processing_sli_retrieve_duration_seconds_count{kind="test"} 3`, 78 | 79 | `service_level_processing_sli_retrieve_failures_total{kind="test"} 3`, 80 | }, 81 | expCode: 200, 82 | }, 83 | } 84 | 85 | for _, test := range tests { 86 | t.Run(test.name, func(t *testing.T) { 87 | assert := assert.New(t) 88 | 89 | reg := prometheus.NewRegistry() 90 | m := metrics.NewPrometheus(reg) 91 | 92 | // Add desired metrics 93 | test.addMetrics(m) 94 | 95 | // Ask prometheus for the metrics 96 | h := promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) 97 | r := httptest.NewRequest("GET", "/metrics", nil) 98 | w := httptest.NewRecorder() 99 | h.ServeHTTP(w, r) 100 | resp := w.Result() 101 | 102 | // Check all metrics are present. 103 | if assert.Equal(test.expCode, resp.StatusCode) { 104 | body, _ := ioutil.ReadAll(resp.Body) 105 | for _, expMetric := range test.expMetrics { 106 | assert.Contains(string(body), expMetric, "metric not present on the result of metrics service") 107 | } 108 | } 109 | }) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /pkg/service/output/factory.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Factory is a factory that knows how to get the correct 10 | // Output strategy based on the SLO output source. 11 | type Factory interface { 12 | // GetGetStrategy returns a output based on the SLO source. 13 | GetStrategy(*monitoringv1alpha1.SLO) (Output, error) 14 | } 15 | 16 | // factory doesn't create objects per se, it only knows 17 | // what strategy to return based on the passed SLI. 18 | type factory struct { 19 | promOutput Output 20 | } 21 | 22 | // NewFactory returns a new output factory. 23 | func NewFactory(promOutput Output) Factory { 24 | return &factory{ 25 | promOutput: promOutput, 26 | } 27 | } 28 | 29 | // GetStrategy satsifies OutputFactory interface. 30 | func (f factory) GetStrategy(s *monitoringv1alpha1.SLO) (Output, error) { 31 | if s.Output.Prometheus != nil { 32 | return f.promOutput, nil 33 | } 34 | 35 | return nil, fmt.Errorf("%s unsupported output kind", s.Name) 36 | } 37 | 38 | // MockFactory returns the mocked output strategy. 39 | type MockFactory struct { 40 | Mock Output 41 | } 42 | 43 | // GetStrategy satisfies Factory interface. 44 | func (m MockFactory) GetStrategy(_ *monitoringv1alpha1.SLO) (Output, error) { 45 | return m.Mock, nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/service/output/middleware.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 8 | "github.com/spotahome/service-level-operator/pkg/service/sli" 9 | ) 10 | 11 | // metricsMiddleware will monitoring the calls to the SLO output. 12 | type metricsMiddleware struct { 13 | kind string 14 | metricssvc metrics.Service 15 | next Output 16 | } 17 | 18 | // NewMetricsMiddleware returns a new metrics middleware that wraps a Output SLO 19 | // service and monitorings with metrics. 20 | func NewMetricsMiddleware(metricssvc metrics.Service, kind string, next Output) Output { 21 | return metricsMiddleware{ 22 | kind: kind, 23 | metricssvc: metricssvc, 24 | next: next, 25 | } 26 | } 27 | 28 | // Create satisfies slo.Output interface. 29 | func (m metricsMiddleware) Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) (err error) { 30 | defer func(t time.Time) { 31 | m.metricssvc.ObserveOuputCreateDuration(slo, m.kind, t) 32 | if err != nil { 33 | m.metricssvc.IncOuputCreateError(slo, m.kind) 34 | } 35 | }(time.Now()) 36 | return m.next.Create(serviceLevel, slo, result) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/service/output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 5 | "github.com/spotahome/service-level-operator/pkg/log" 6 | "github.com/spotahome/service-level-operator/pkg/service/sli" 7 | ) 8 | 9 | // Output knows how expose/send/create the output of a SLO and SLI result. 10 | type Output interface { 11 | // Create will create the SLI result and the SLO on the specific format. 12 | // It receives the SLI's SLO and it's result. 13 | Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) error 14 | } 15 | 16 | type logger struct { 17 | logger log.Logger 18 | } 19 | 20 | // NewLogger returns a new output logger service that will output the SLOs on 21 | // the specified logger. 22 | func NewLogger(l log.Logger) Output { 23 | return &logger{ 24 | logger: l, 25 | } 26 | } 27 | 28 | // Create will log the result on the console. 29 | func (l *logger) Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) error { 30 | errorRat, err := result.ErrorRatio() 31 | if err != nil { 32 | return err 33 | } 34 | l.logger.With("id", serviceLevel.Name). 35 | With("slo", slo.Name). 36 | With("availability-target", slo.AvailabilityObjectivePercent). 37 | Infof("SLI error ratio: %f", errorRat) 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/service/output/prometheus.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | 10 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 11 | "github.com/spotahome/service-level-operator/pkg/log" 12 | "github.com/spotahome/service-level-operator/pkg/service/sli" 13 | ) 14 | 15 | const ( 16 | promNS = "service_level" 17 | promSLOSubsystem = "slo" 18 | promSLISubsystem = "sli" 19 | defExpireDuration = 90 * time.Second 20 | ) 21 | 22 | // metricValue is an internal type to store the counters 23 | // of the metrics so when the collector is called it creates 24 | // the metrics based on this values. 25 | type metricValue struct { 26 | serviceLevel *monitoringv1alpha1.ServiceLevel 27 | slo *monitoringv1alpha1.SLO 28 | errorSum float64 29 | countSum float64 30 | objective float64 31 | expire time.Time // expire is the time where this metric will expire unless it's refreshed. 32 | } 33 | 34 | // PrometheusCfg is the configuration of the Prometheus Output. 35 | type PrometheusCfg struct { 36 | // ExpireDuration is the time a metric will expire if is not refreshed. 37 | ExpireDuration time.Duration 38 | } 39 | 40 | // Validate will validate the cfg setting safe defaults. 41 | func (p *PrometheusCfg) Validate() { 42 | if p.ExpireDuration == 0 { 43 | p.ExpireDuration = defExpireDuration 44 | } 45 | } 46 | 47 | // Prometheus knows how to set the output of the SLO on a Prometheus backend. 48 | // The way it works this output is creating two main counters, one that increments 49 | // the error and other that increments the full ratio. 50 | // Example: 51 | // error ratio: 0 + 0 + 0.001 + 0.1 + 0.01 = 0.111 52 | // full ratio: 1 + 1 + 1 + 1 + 1 = 5 53 | // 54 | // You could get the total availability ratio with 1-(0.111/5) = 0.9778 55 | // In other words the availability of all this time is: 97.78% 56 | // 57 | // Under the hood this service is a prometheus collector, it will send to 58 | // prometheus dynamic metrics (because of dynamic labels) when the collect 59 | // process is called. This is made by storing the internal counters and 60 | // generating the metrics when the collect process is callend on each scrape. 61 | type prometheusOutput struct { 62 | cfg PrometheusCfg 63 | metricValuesMu sync.Mutex 64 | metricValues map[string]*metricValue 65 | reg prometheus.Registerer 66 | logger log.Logger 67 | } 68 | 69 | // NewPrometheus returns a new Prometheus output. 70 | func NewPrometheus(cfg PrometheusCfg, reg prometheus.Registerer, logger log.Logger) Output { 71 | cfg.Validate() 72 | 73 | p := &prometheusOutput{ 74 | cfg: cfg, 75 | metricValues: map[string]*metricValue{}, 76 | reg: reg, 77 | logger: logger, 78 | } 79 | 80 | // Autoregister as collector of SLO metrics for prometheus. 81 | p.reg.MustRegister(p) 82 | 83 | return p 84 | } 85 | 86 | // Create satisfies output interface. By setting the correct values on the different 87 | // metrics of the SLO. 88 | func (p *prometheusOutput) Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) error { 89 | p.metricValuesMu.Lock() 90 | defer p.metricValuesMu.Unlock() 91 | 92 | // Get the current metrics for the SLO. 93 | sloID := fmt.Sprintf("%s-%s-%s", serviceLevel.Namespace, serviceLevel.Name, slo.Name) 94 | if _, ok := p.metricValues[sloID]; !ok { 95 | p.metricValues[sloID] = &metricValue{} 96 | } 97 | 98 | // Add metric values. 99 | errRat, err := result.ErrorRatio() 100 | if err != nil { 101 | return err 102 | } 103 | 104 | // Check it's a possitive number, this shouldn't be necessary but for 105 | // safety we do it. 106 | if errRat < 0 { 107 | errRat = 0 108 | } 109 | 110 | metric := p.metricValues[sloID] 111 | metric.serviceLevel = serviceLevel 112 | metric.slo = slo 113 | metric.errorSum += errRat 114 | metric.countSum++ 115 | // Objective is in % so we convert to ratio (0-1). 116 | metric.objective = slo.AvailabilityObjectivePercent / 100 117 | // Refresh the metric expiration. 118 | metric.expire = time.Now().Add(p.cfg.ExpireDuration) 119 | 120 | return nil 121 | } 122 | 123 | // Describe satisfies prometheus.Collector interface. 124 | func (p *prometheusOutput) Describe(chan<- *prometheus.Desc) {} 125 | 126 | // Collect satisfies prometheus.Collector interface. 127 | func (p *prometheusOutput) Collect(ch chan<- prometheus.Metric) { 128 | p.metricValuesMu.Lock() 129 | defer p.metricValuesMu.Unlock() 130 | p.logger.Debugf("start collecting all service level metrics") 131 | 132 | for id, metric := range p.metricValues { 133 | metric := metric 134 | 135 | // If metric has expired then remove from the map. 136 | if time.Now().After(metric.expire) { 137 | p.logger.With("slo", metric.slo.Name).With("service-level", metric.serviceLevel.Name).Infof("metric expired, removing") 138 | delete(p.metricValues, id) 139 | continue 140 | } 141 | 142 | ns := metric.serviceLevel.Namespace 143 | slName := metric.serviceLevel.Name 144 | sloName := metric.slo.Name 145 | var labels map[string]string 146 | // Check just in case. 147 | if metric.slo.Output.Prometheus != nil && metric.slo.Output.Prometheus.Labels != nil { 148 | labels = metric.slo.Output.Prometheus.Labels 149 | } 150 | 151 | ch <- p.getSLIErrorMetric(ns, slName, sloName, labels, metric.errorSum) 152 | ch <- p.getSLICountMetric(ns, slName, sloName, labels, metric.countSum) 153 | ch <- p.getSLOObjectiveMetric(ns, slName, sloName, labels, metric.objective) 154 | } 155 | 156 | // Collect all SLOs metric. 157 | p.logger.Debugf("finished collecting all the service level metrics") 158 | } 159 | 160 | func (p *prometheusOutput) getSLIErrorMetric(ns, serviceLevel, slo string, constLabels prometheus.Labels, value float64) prometheus.Metric { 161 | return prometheus.MustNewConstMetric( 162 | prometheus.NewDesc( 163 | prometheus.BuildFQName(promNS, promSLISubsystem, "result_error_ratio_total"), 164 | "Is the error or failure ratio of an SLI result.", 165 | []string{"namespace", "service_level", "slo"}, 166 | constLabels, 167 | ), 168 | prometheus.CounterValue, 169 | value, 170 | ns, serviceLevel, slo, 171 | ) 172 | } 173 | 174 | func (p *prometheusOutput) getSLICountMetric(ns, serviceLevel, slo string, constLabels prometheus.Labels, value float64) prometheus.Metric { 175 | return prometheus.MustNewConstMetric( 176 | prometheus.NewDesc( 177 | prometheus.BuildFQName(promNS, promSLISubsystem, "result_count_total"), 178 | "Is the number of times an SLI result has been processed.", 179 | []string{"namespace", "service_level", "slo"}, 180 | constLabels, 181 | ), 182 | prometheus.CounterValue, 183 | value, 184 | ns, serviceLevel, slo, 185 | ) 186 | } 187 | 188 | func (p *prometheusOutput) getSLOObjectiveMetric(ns, serviceLevel, slo string, constLabels prometheus.Labels, value float64) prometheus.Metric { 189 | return prometheus.MustNewConstMetric( 190 | prometheus.NewDesc( 191 | prometheus.BuildFQName(promNS, promSLOSubsystem, "objective_ratio"), 192 | "Is the objective of the SLO in ratio unit.", 193 | []string{"namespace", "service_level", "slo"}, 194 | constLabels, 195 | ), 196 | prometheus.GaugeValue, 197 | value, 198 | ns, serviceLevel, slo, 199 | ) 200 | } 201 | -------------------------------------------------------------------------------- /pkg/service/output/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package output_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | "github.com/stretchr/testify/assert" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | 14 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 15 | "github.com/spotahome/service-level-operator/pkg/log" 16 | "github.com/spotahome/service-level-operator/pkg/service/output" 17 | "github.com/spotahome/service-level-operator/pkg/service/sli" 18 | ) 19 | 20 | var ( 21 | sl0 = &monitoringv1alpha1.ServiceLevel{ 22 | ObjectMeta: metav1.ObjectMeta{ 23 | Name: "sl0-test", 24 | Namespace: "ns0", 25 | }, 26 | } 27 | sl1 = &monitoringv1alpha1.ServiceLevel{ 28 | ObjectMeta: metav1.ObjectMeta{ 29 | Name: "sl1-test", 30 | Namespace: "ns1", 31 | }, 32 | } 33 | slo00 = &monitoringv1alpha1.SLO{ 34 | Name: "slo00-test", 35 | AvailabilityObjectivePercent: 99.999, 36 | Output: monitoringv1alpha1.Output{ 37 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 38 | }, 39 | } 40 | slo01 = &monitoringv1alpha1.SLO{ 41 | Name: "slo01-test", 42 | AvailabilityObjectivePercent: 99.98, 43 | Output: monitoringv1alpha1.Output{ 44 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 45 | }, 46 | } 47 | slo10 = &monitoringv1alpha1.SLO{ 48 | Name: "slo10-test", 49 | AvailabilityObjectivePercent: 99.99978, 50 | Output: monitoringv1alpha1.Output{ 51 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 52 | }, 53 | } 54 | slo11 = &monitoringv1alpha1.SLO{ 55 | Name: "slo11-test", 56 | AvailabilityObjectivePercent: 95.9981, 57 | Output: monitoringv1alpha1.Output{ 58 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 59 | Labels: map[string]string{ 60 | "env": "test", 61 | "team": "team1", 62 | }, 63 | }, 64 | }, 65 | } 66 | ) 67 | 68 | func TestPrometheusOutput(t *testing.T) { 69 | tests := []struct { 70 | name string 71 | cfg output.PrometheusCfg 72 | createResults func(output output.Output) 73 | expMetrics []string 74 | expMissingMetrics []string 75 | }{ 76 | { 77 | name: "Creating a output result should expose all the required metrics", 78 | createResults: func(output output.Output) { 79 | output.Create(sl0, slo00, &sli.Result{ 80 | TotalQ: 1000000, 81 | ErrorQ: 122, 82 | }) 83 | }, 84 | expMetrics: []string{ 85 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.000122`, 86 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 1`, 87 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 88 | }, 89 | }, 90 | { 91 | name: "Expired metrics shouldn't be exposed", 92 | cfg: output.PrometheusCfg{ 93 | ExpireDuration: 500 * time.Microsecond, 94 | }, 95 | createResults: func(output output.Output) { 96 | output.Create(sl0, slo00, &sli.Result{ 97 | TotalQ: 1000000, 98 | ErrorQ: 122, 99 | }) 100 | time.Sleep(1 * time.Millisecond) 101 | }, 102 | expMissingMetrics: []string{ 103 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.000122`, 104 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 1`, 105 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 106 | }, 107 | }, 108 | { 109 | name: "Creating a output result should expose all the required metrics (multiple adds on same SLO).", 110 | createResults: func(output output.Output) { 111 | slis := []*sli.Result{ 112 | &sli.Result{TotalQ: 1000000, ErrorQ: 122}, 113 | &sli.Result{TotalQ: 999, ErrorQ: 1}, 114 | &sli.Result{TotalQ: 812392, ErrorQ: 94}, 115 | &sli.Result{TotalQ: 83, ErrorQ: 83}, 116 | &sli.Result{TotalQ: 11223, ErrorQ: 11222}, 117 | &sli.Result{TotalQ: 9999999999, ErrorQ: 2}, 118 | &sli.Result{TotalQ: 1245, ErrorQ: 0}, 119 | &sli.Result{TotalQ: 9019, ErrorQ: 1001}, 120 | } 121 | for _, sli := range slis { 122 | output.Create(sl0, slo00, sli) 123 | } 124 | }, 125 | expMetrics: []string{ 126 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 2.112137520556389`, 127 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 8`, 128 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 129 | }, 130 | }, 131 | { 132 | name: "Creating a output result should expose all the required metrics (multiple SLOs).", 133 | createResults: func(output output.Output) { 134 | output.Create(sl0, slo00, &sli.Result{ 135 | TotalQ: 1000000, 136 | ErrorQ: 122, 137 | }) 138 | output.Create(sl0, slo01, &sli.Result{ 139 | TotalQ: 1011, 140 | ErrorQ: 340, 141 | }) 142 | output.Create(sl1, slo10, &sli.Result{ 143 | TotalQ: 9212, 144 | ErrorQ: 1, 145 | }) 146 | output.Create(sl1, slo10, &sli.Result{ 147 | TotalQ: 3456, 148 | ErrorQ: 3, 149 | }) 150 | output.Create(sl1, slo11, &sli.Result{ 151 | TotalQ: 998, 152 | ErrorQ: 7, 153 | }) 154 | }, 155 | expMetrics: []string{ 156 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.000122`, 157 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 1`, 158 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 159 | 160 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo01-test"} 0.3363006923837784`, 161 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo01-test"} 1`, 162 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo01-test"} 0.9998`, 163 | 164 | `service_level_sli_result_error_ratio_total{namespace="ns1",service_level="sl1-test",slo="slo10-test"} 0.0009766096154773965`, 165 | `service_level_sli_result_count_total{namespace="ns1",service_level="sl1-test",slo="slo10-test"} 2`, 166 | `service_level_slo_objective_ratio{namespace="ns1",service_level="sl1-test",slo="slo10-test"} 0.9999978`, 167 | 168 | `service_level_sli_result_error_ratio_total{env="test",namespace="ns1",service_level="sl1-test",slo="slo11-test",team="team1"} 0.0070140280561122245`, 169 | `service_level_sli_result_count_total{env="test",namespace="ns1",service_level="sl1-test",slo="slo11-test",team="team1"} 1`, 170 | `service_level_slo_objective_ratio{env="test",namespace="ns1",service_level="sl1-test",slo="slo11-test",team="team1"} 0.959981`, 171 | }, 172 | }, 173 | } 174 | 175 | for _, test := range tests { 176 | t.Run(test.name, func(t *testing.T) { 177 | assert := assert.New(t) 178 | promReg := prometheus.NewRegistry() 179 | 180 | output := output.NewPrometheus(test.cfg, promReg, log.Dummy) 181 | test.createResults(output) 182 | 183 | // Check metrics 184 | h := promhttp.HandlerFor(promReg, promhttp.HandlerOpts{}) 185 | w := httptest.NewRecorder() 186 | req := httptest.NewRequest("GET", "/metrics", nil) 187 | h.ServeHTTP(w, req) 188 | 189 | metrics, _ := ioutil.ReadAll(w.Result().Body) 190 | for _, expMetric := range test.expMetrics { 191 | assert.Contains(string(metrics), expMetric) 192 | } 193 | for _, expMissingMetric := range test.expMissingMetrics { 194 | assert.NotContains(string(metrics), expMissingMetric) 195 | } 196 | }) 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /pkg/service/sli/factory.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "errors" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // RetrieverFactory is a factory that knows how to get the correct 10 | // Retriever based on the SLI source. 11 | type RetrieverFactory interface { 12 | // GetRetriever returns a retriever based on the SLI source. 13 | GetStrategy(*monitoringv1alpha1.SLI) (Retriever, error) 14 | } 15 | 16 | // retrieverFactory doesn't create objects per se, it only knows 17 | // what strategy to return based on the passed SLI. 18 | type retrieverFactory struct { 19 | promRetriever Retriever 20 | } 21 | 22 | // NewRetrieverFactory returns a new retriever factory. 23 | func NewRetrieverFactory(promRetriever Retriever) RetrieverFactory { 24 | return &retrieverFactory{ 25 | promRetriever: promRetriever, 26 | } 27 | } 28 | 29 | // GetRetriever satsifies RetrieverFactory interface. 30 | func (r retrieverFactory) GetStrategy(s *monitoringv1alpha1.SLI) (Retriever, error) { 31 | if s.Prometheus != nil { 32 | return r.promRetriever, nil 33 | } 34 | 35 | return nil, errors.New("unsupported retriever kind") 36 | } 37 | 38 | // MockRetrieverFactory returns the mocked retriever strategy. 39 | type MockRetrieverFactory struct { 40 | Mock Retriever 41 | } 42 | 43 | // GetStrategy satisfies RetrieverFactory interface. 44 | func (m MockRetrieverFactory) GetStrategy(_ *monitoringv1alpha1.SLI) (Retriever, error) { 45 | return m.Mock, nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/service/sli/middleware.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 8 | ) 9 | 10 | // metricsMiddleware will monitoring the calls to the SLI Retriever. 11 | type metricsMiddleware struct { 12 | kind string 13 | metricssvc metrics.Service 14 | next Retriever 15 | } 16 | 17 | // NewMetricsMiddleware returns a new metrics middleware that wraps a Retriever SLI 18 | // service and monitorings with metrics. 19 | func NewMetricsMiddleware(metricssvc metrics.Service, kind string, next Retriever) Retriever { 20 | return metricsMiddleware{ 21 | kind: kind, 22 | metricssvc: metricssvc, 23 | next: next, 24 | } 25 | } 26 | 27 | // Retrieve satisfies sli.Retriever interface. 28 | func (m metricsMiddleware) Retrieve(sli *monitoringv1alpha1.SLI) (result Result, err error) { 29 | defer func(t time.Time) { 30 | m.metricssvc.ObserveSLIRetrieveDuration(sli, m.kind, t) 31 | if err != nil { 32 | m.metricssvc.IncSLIRetrieveError(sli, m.kind) 33 | } 34 | }(time.Now()) 35 | return m.next.Retrieve(sli) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/service/sli/prometheus.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "golang.org/x/sync/errgroup" 9 | 10 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 11 | "github.com/prometheus/common/model" 12 | 13 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 14 | "github.com/spotahome/service-level-operator/pkg/log" 15 | promcli "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 16 | ) 17 | 18 | const promCliTimeout = 2 * time.Second 19 | 20 | // prometheus knows how to get SLIs from a prometheus backend. 21 | type prometheus struct { 22 | cliFactory promcli.ClientFactory 23 | logger log.Logger 24 | } 25 | 26 | // NewPrometheus returns a new prometheus SLI service. 27 | func NewPrometheus(promCliFactory promcli.ClientFactory, logger log.Logger) Retriever { 28 | return &prometheus{ 29 | cliFactory: promCliFactory, 30 | logger: logger, 31 | } 32 | } 33 | 34 | // Retrieve satisfies Service interface.. 35 | func (p *prometheus) Retrieve(sli *monitoringv1alpha1.SLI) (Result, error) { 36 | cli, err := p.cliFactory.GetV1APIClient(sli.Prometheus.Address) 37 | if err != nil { 38 | return Result{}, err 39 | } 40 | 41 | // Get both metrics. 42 | res := Result{} 43 | 44 | promclictx, cancel := context.WithTimeout(context.Background(), promCliTimeout) 45 | defer cancel() 46 | 47 | // Make queries concurrently. 48 | g, ctx := errgroup.WithContext(promclictx) 49 | g.Go(func() error { 50 | res.TotalQ, err = p.getVectorMetric(ctx, cli, sli.Prometheus.TotalQuery) 51 | return err 52 | }) 53 | g.Go(func() error { 54 | res.ErrorQ, err = p.getVectorMetric(ctx, cli, sli.Prometheus.ErrorQuery) 55 | return err 56 | }) 57 | 58 | // Wait for the first error or until all of them have finished. 59 | err = g.Wait() 60 | if err != nil { 61 | return Result{}, err 62 | } 63 | 64 | return res, nil 65 | } 66 | 67 | func (p *prometheus) getVectorMetric(ctx context.Context, cli promv1.API, query string) (float64, error) { 68 | // Make the query. 69 | val, _, err := cli.Query(ctx, query, time.Now()) 70 | if err != nil { 71 | return 0, err 72 | } 73 | 74 | if val == nil { 75 | return 0, fmt.Errorf("nil value received from prometheus") 76 | } 77 | 78 | // Only vectors are valid metrics. 79 | if val.Type() != model.ValVector { 80 | return 0, fmt.Errorf("received metric needs to be a vector, received: %s", val.Type()) 81 | } 82 | mtr := val.(model.Vector) 83 | 84 | // If we obtain no metric then for us is 0. 85 | if len(mtr) == 0 { 86 | return 0, nil 87 | } 88 | 89 | // More than one metric should be an error. 90 | if len(mtr) != 1 { 91 | return 0, fmt.Errorf("wrong samples length, should not be more than 1, got: %d", len(mtr)) 92 | } 93 | 94 | return float64(mtr[0].Value), nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/service/sli/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package sli_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/prometheus/common/model" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/mock" 10 | 11 | mpromv1 "github.com/spotahome/service-level-operator/mocks/github.com/prometheus/client_golang/api/prometheus/v1" 12 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 13 | "github.com/spotahome/service-level-operator/pkg/log" 14 | prometheusvc "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 15 | "github.com/spotahome/service-level-operator/pkg/service/sli" 16 | ) 17 | 18 | func TestPrometheusRetrieve(t *testing.T) { 19 | sli0 := &monitoringv1alpha1.SLI{ 20 | SLISource: monitoringv1alpha1.SLISource{ 21 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 22 | TotalQuery: "test_total_query", 23 | ErrorQuery: "test_error_query", 24 | }, 25 | }, 26 | } 27 | vector2 := model.Vector{ 28 | &model.Sample{ 29 | Metric: model.Metric{}, 30 | Value: model.SampleValue(2), 31 | }, 32 | } 33 | vector100 := model.Vector{ 34 | &model.Sample{ 35 | Metric: model.Metric{}, 36 | Value: model.SampleValue(100), 37 | }, 38 | } 39 | 40 | tests := []struct { 41 | name string 42 | sli *monitoringv1alpha1.SLI 43 | 44 | totalQueryResult model.Value 45 | totalQueryErr error 46 | errorQueryResult model.Value 47 | errorQueryErr error 48 | 49 | expResult sli.Result 50 | expErr bool 51 | }{ 52 | { 53 | name: "If no result from prometheus it should fail.", 54 | sli: sli0, 55 | expErr: true, 56 | }, 57 | { 58 | name: "Failing total query should make the retrieval fail.", 59 | sli: sli0, 60 | totalQueryResult: vector100, 61 | totalQueryErr: errors.New("wanted error"), 62 | errorQueryResult: vector2, 63 | expErr: true, 64 | }, 65 | { 66 | name: "Failing error query should make the retrieval fail.", 67 | sli: sli0, 68 | totalQueryResult: vector100, 69 | errorQueryResult: vector2, 70 | errorQueryErr: errors.New("wanted error"), 71 | expErr: true, 72 | }, 73 | { 74 | name: "If the query doesn't return a vector it should fail.", 75 | sli: sli0, 76 | totalQueryResult: &model.Scalar{ 77 | Value: model.SampleValue(2), 78 | }, 79 | errorQueryResult: vector2, 80 | expErr: true, 81 | }, 82 | { 83 | name: "If the query returns more than one metric it should fail.", 84 | sli: sli0, 85 | totalQueryResult: model.Vector{ 86 | &model.Sample{ 87 | Value: model.SampleValue(1), 88 | }, 89 | &model.Sample{ 90 | Value: model.SampleValue(2), 91 | }, 92 | }, 93 | errorQueryResult: vector2, 94 | expErr: true, 95 | }, 96 | { 97 | name: "If the query returns 0 metrics it should treat as a 0 value.", 98 | sli: sli0, 99 | totalQueryResult: vector2, 100 | errorQueryResult: model.Vector{}, 101 | expErr: false, 102 | expResult: sli.Result{ 103 | TotalQ: 2, 104 | ErrorQ: 0, 105 | }, 106 | }, 107 | { 108 | name: "Quering prometheus for total and error metrics should return a correct result", 109 | sli: sli0, 110 | totalQueryResult: vector100, 111 | errorQueryResult: vector2, 112 | expResult: sli.Result{ 113 | TotalQ: 100, 114 | ErrorQ: 2, 115 | }, 116 | }, 117 | } 118 | 119 | for _, test := range tests { 120 | t.Run(test.name, func(t *testing.T) { 121 | assert := assert.New(t) 122 | 123 | // Mocks. 124 | mapi := &mpromv1.API{} 125 | mpromfactory := &prometheusvc.MockFactory{Cli: mapi} 126 | mapi.On("Query", mock.Anything, test.sli.Prometheus.TotalQuery, mock.Anything).Return(test.totalQueryResult, nil, test.errorQueryErr) 127 | mapi.On("Query", mock.Anything, test.sli.Prometheus.ErrorQuery, mock.Anything).Return(test.errorQueryResult, nil, test.totalQueryErr) 128 | 129 | retriever := sli.NewPrometheus(mpromfactory, log.Dummy) 130 | res, err := retriever.Retrieve(test.sli) 131 | 132 | if test.expErr { 133 | assert.Error(err) 134 | } else if assert.NoError(err) { 135 | assert.Equal(test.expResult, res) 136 | } 137 | }) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/service/sli/sli.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "fmt" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Result is the result of getting a SLI from a backend. 10 | type Result struct { 11 | // TotalQ is the result of applying the total query. 12 | TotalQ float64 13 | // ErrorQ is the result of applying the error query. 14 | ErrorQ float64 15 | } 16 | 17 | // AvailabilityRatio returns the availability of an SLI result in 18 | // ratio unit (0-1). 19 | func (r *Result) AvailabilityRatio() (float64, error) { 20 | if r.TotalQ < r.ErrorQ { 21 | return 0, fmt.Errorf("%f can't be higher than %f", r.ErrorQ, r.TotalQ) 22 | } 23 | 24 | // If no total then everything ok. 25 | if r.TotalQ <= 0 { 26 | return 1, nil 27 | } 28 | 29 | eRat, err := r.ErrorRatio() 30 | if err != nil { 31 | return 0, err 32 | } 33 | 34 | return 1 - eRat, nil 35 | } 36 | 37 | // ErrorRatio returns the error of an SLI result in. 38 | // ratio unit (0-1). 39 | func (r *Result) ErrorRatio() (float64, error) { 40 | if r.TotalQ < r.ErrorQ { 41 | return 0, fmt.Errorf("%f can't be higher than %f", r.ErrorQ, r.TotalQ) 42 | } 43 | 44 | // If no total then everything ok. 45 | if r.TotalQ <= 0 { 46 | return 0, nil 47 | } 48 | 49 | return r.ErrorQ / r.TotalQ, nil 50 | } 51 | 52 | // Retriever knows how to get SLIs from different backends. 53 | type Retriever interface { 54 | // Retrieve returns the result of a SLI retrieved from the implemented backend. 55 | Retrieve(*monitoringv1alpha1.SLI) (Result, error) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/service/sli/sli_test.go: -------------------------------------------------------------------------------- 1 | package sli_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/spotahome/service-level-operator/pkg/service/sli" 9 | ) 10 | 11 | func TestSLIResult(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | errorQ float64 15 | totalQ float64 16 | expAvailability float64 17 | expAvailabiityErr bool 18 | expError float64 19 | expErrorErr bool 20 | }{ 21 | { 22 | name: "Not having a total quantity should return everything ok.", 23 | expAvailability: 1, 24 | expError: 0, 25 | }, 26 | { 27 | name: "Having more errors than total should be impossible.", 28 | errorQ: 600, 29 | totalQ: 300, 30 | expErrorErr: true, 31 | expAvailabiityErr: true, 32 | }, 33 | { 34 | name: "If half of the total are errors then the ratio of availability and error should be 0.5.", 35 | errorQ: 300, 36 | totalQ: 600, 37 | expAvailability: 0.5, 38 | expError: 0.5, 39 | }, 40 | { 41 | name: "If a 33% of errors then the ratios should be 0.33 and 0.66.", 42 | errorQ: 33, 43 | totalQ: 100, 44 | expAvailability: 0.6699999999999999, 45 | expError: 0.33, 46 | }, 47 | { 48 | name: "In small quantities the ratios should be correctly calculated.", 49 | errorQ: 4, 50 | totalQ: 10, 51 | expAvailability: 0.6, 52 | expError: 0.4, 53 | }, 54 | { 55 | name: "In big quantities the ratios should be correctly calculated.", 56 | errorQ: 240, 57 | totalQ: 10000000, 58 | expAvailability: 0.999976, 59 | expError: 0.000024, 60 | }, 61 | } 62 | 63 | for _, test := range tests { 64 | t.Run(test.name, func(t *testing.T) { 65 | assert := assert.New(t) 66 | 67 | res := sli.Result{ 68 | TotalQ: test.totalQ, 69 | ErrorQ: test.errorQ, 70 | } 71 | 72 | av, err := res.AvailabilityRatio() 73 | if test.expAvailabiityErr { 74 | assert.Error(err) 75 | } else if assert.NoError(err) { 76 | assert.Equal(test.expAvailability, av) 77 | } 78 | 79 | dw, err := res.ErrorRatio() 80 | if test.expErrorErr { 81 | assert.Error(err) 82 | } else if assert.NoError(err) { 83 | assert.Equal(test.expError, dw) 84 | } 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/manual/slos.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.spotahome.com/v1alpha1 2 | kind: ServiceLevel 3 | metadata: 4 | name: awesome-service 5 | spec: 6 | serviceLevelObjectives: 7 | # A typical 5xx request SLO. 8 | - name: "9999_http_request_lt_500" 9 | description: 99.99% of requests must be served with <500 status code. 10 | disable: false 11 | availabilityObjectivePercent: 99.99 12 | serviceLevelIndicator: 13 | prometheus: 14 | address: http://127.0.0.1:9091 15 | totalQuery: | 16 | sum( 17 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 18 | errorQuery: | 19 | sum( 20 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com", code=~"5.."}[2m])) 21 | output: 22 | prometheus: {} 23 | 24 | # Latency example. 25 | - name: "90_http_request_latency_lt_250ms" 26 | description: 90% of requests must be served with in less than 250ms. 27 | disable: false 28 | availabilityObjectivePercent: 90 29 | serviceLevelIndicator: 30 | prometheus: 31 | address: http://127.0.0.1:9091 32 | totalQuery: | 33 | sum( 34 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 35 | errorQuery: | 36 | sum( 37 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 38 | - 39 | sum( 40 | increase(skipper_serve_host_duration_seconds_bucket{host="www_spotahome_com", le="0.25"}[2m])) 41 | output: 42 | prometheus: {} 43 | 44 | # Same latency example as previous one but with different latency bucket. Is normal to have 45 | # multiple SLOs on different latency limits. 46 | - name: "99_http_request_latency_lt_500ms" 47 | description: 99% of requests must be served with in less than 500ms. 48 | disable: false 49 | availabilityObjectivePercent: 99 50 | serviceLevelIndicator: 51 | prometheus: 52 | address: http://127.0.0.1:9091 53 | totalQuery: | 54 | sum( 55 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 56 | errorQuery: | 57 | sum( 58 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 59 | - 60 | sum( 61 | increase(skipper_serve_host_duration_seconds_bucket{host="www_spotahome_com", le="0.5"}[2m])) 62 | output: 63 | prometheus: {} 64 | 65 | # Same as previous one except the availability objective. This way we can see that 66 | # with the same SLIs our SLO changes drastically. 67 | - name: "90_http_request_latency_lt_500ms" 68 | description: 90% of requests must be served with in less than 500ms. 69 | disable: false 70 | availabilityObjectivePercent: 90 71 | serviceLevelIndicator: 72 | prometheus: 73 | address: http://127.0.0.1:9091 74 | totalQuery: | 75 | sum( 76 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 77 | errorQuery: | 78 | sum( 79 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 80 | - 81 | sum( 82 | increase(skipper_serve_host_duration_seconds_bucket{host="www_spotahome_com", le="0.5"}[2m])) 83 | output: 84 | prometheus: 85 | labels: 86 | env: production 87 | team: a-team 88 | --------------------------------------------------------------------------------