├── .gitignore ├── .travis.yml ├── CHANGELOG ├── LICENSE ├── Makefile ├── Readme.md ├── alerts ├── slo.yaml └── slo_test.yaml ├── cmd └── service-level-operator │ ├── flags.go │ └── main.go ├── deploy ├── Readme.md └── manifests │ ├── deployment.yaml │ ├── prometheus.yaml │ ├── rbac.yaml │ └── service.yaml ├── docker ├── dev │ ├── Dockerfile │ ├── docker-compose.yaml │ └── prometheus.yml └── prod │ └── Dockerfile ├── go.mod ├── go.sum ├── hack └── scripts │ ├── build-binary.sh │ ├── build-image.sh │ ├── integration-test.sh │ ├── k8scodegen.sh │ ├── mockgen.sh │ ├── openapicodegen.sh │ ├── test-alerts.sh │ └── unit-test.sh ├── img └── grafana_graphs1.png ├── mocks ├── doc.go ├── github.com │ └── prometheus │ │ └── client_golang │ │ └── api │ │ └── prometheus │ │ └── v1 │ │ └── API.go ├── service │ ├── output │ │ └── Output.go │ └── sli │ │ └── Retriever.go └── thirdparty.go ├── pkg ├── apis │ └── monitoring │ │ ├── register.go │ │ └── v1alpha1 │ │ ├── doc.go │ │ ├── helpers.go │ │ ├── helpers_test.go │ │ ├── openapi_generated.go │ │ ├── register.go │ │ ├── types.go │ │ └── zz_generated.deepcopy.go ├── k8sautogen │ └── client │ │ └── clientset │ │ └── versioned │ │ ├── clientset.go │ │ ├── doc.go │ │ ├── fake │ │ ├── clientset_generated.go │ │ ├── doc.go │ │ └── register.go │ │ ├── scheme │ │ ├── doc.go │ │ └── register.go │ │ └── typed │ │ └── monitoring │ │ └── v1alpha1 │ │ ├── doc.go │ │ ├── fake │ │ ├── doc.go │ │ ├── fake_monitoring_client.go │ │ └── fake_servicelevel.go │ │ ├── generated_expansion.go │ │ ├── monitoring_client.go │ │ └── servicelevel.go ├── log │ ├── dummy.go │ └── log.go ├── operator │ ├── crd.go │ ├── doc.go │ ├── factory.go │ ├── handler.go │ └── handler_test.go └── service │ ├── client │ ├── kubernetes │ │ ├── factory.go │ │ └── fake.go │ └── prometheus │ │ ├── factory.go │ │ ├── factory_test.go │ │ └── fake.go │ ├── configuration │ ├── configuration.go │ └── configuration_test.go │ ├── kubernetes │ ├── crd.go │ ├── kubernetes.go │ └── servicelevel.go │ ├── metrics │ ├── dummy.go │ ├── metrics.go │ ├── prometheus.go │ └── prometheus_test.go │ ├── output │ ├── factory.go │ ├── middleware.go │ ├── output.go │ ├── prometheus.go │ └── prometheus_test.go │ └── sli │ ├── factory.go │ ├── middleware.go │ ├── prometheus.go │ ├── prometheus_test.go │ ├── sli.go │ └── sli_test.go └── test └── manual └── slos.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Binary 14 | bin/ 15 | 16 | # vendor 17 | vendor/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: required 3 | services: 4 | - docker 5 | go: 6 | - "1.13" 7 | 8 | before_install: 9 | # Upgrade default docker 10 | - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - 11 | - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" 12 | - sudo apt-get update 13 | - sudo apt-get -y install docker-ce 14 | 15 | script: 16 | - make ci 17 | 18 | deploy: 19 | - provider: script 20 | script: docker login -u="${DOCKER_USER}" -p="${DOCKER_PASSWORD}" quay.io && make push 21 | on: 22 | tags: true 23 | 24 | env: 25 | global: 26 | - GO111MODULE: on 27 | - DOCKER_USER: spotahome+serviceleveloperator 28 | # DOCKER_PASSWORD 29 | - secure: "smTgGZsms3W3cjA2aw34Rr1rNjVIHj9N0CwgpOLWuw3j1Nwb4OdRPdHN8qSc4HpCeIVWUU/rdVr2gsXherok+6vQR5y4Wh4dxTODM/4mVxdY67k7URJra4Vktei0K1hD5E4AYAvUb/YZTlBcex83tIkMaeGSYc+w2PzQHa1R1p4PVVEnsu6+O1Qg4i7xErXLmnSVc5tcqpXRhM/cp/XMNiJPKezRVHD5j73ZzN36xJSkNS/xiHw5nA/B/kpzb/NL/sAvknXNqx8+/S0Y3mIsXId4r8LK3YBjV5ALYZvq0mWc/0vZIZ8Fmcf6J+1LQzT+7O9lUuAFKt9LXMsZbiQuG0YEcqix732IqIXicUSOLA/w9TajLNYXQh05L5mAiejHsFZmtDwCDzRi/wvc4d3NkxJt4YsuUu+2Lsd5DGyyuoz2zlzaVoVRSCYwKTIfLU0XdiQ8ilHpI31BN7TK3z+Eh27MfkbJTldyHB2eV0B6mLWxpYICYf5TzlwHtvLg8FUrJ+W7j9Wk63+0W4pY0gzNMSaZunSEuymHjdn94U01aIjqAjMat6HqvjdAs5mYgLmLqC8clOjMW4fJmhdiSXk0NDEAjG35G3OJ9dsImXgaQrHGMM3SZJAhxPEF+MpH0iUuaqC4ExBQtJFz9sQTOue10zOMVRSOoqZQuGmmj2Uy+zk=" 30 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 2 | ## [Unreleased] 3 | 4 | ## [0.3.0] - 2019-10-25 5 | ### Added 6 | - Default SLI input configurations for the SLOs that don't have SLI inputs. 7 | 8 | ## [0.2.0] - 2018-11-14 9 | ### Added 10 | - Grafana dashboard. 11 | 12 | ### Changed 13 | - Move CRD api from `apiVersion: measure.slok.xyz/v1alpha1` to `apiVersion: monitoring.spotahome.com/v1alpha1`. 14 | - Move repository from github.com/slok/service-level-operator to github.com/spotahome/service-level-operator. 15 | 16 | ## [0.1.0] - 2018-10-31 17 | ### Added 18 | - Prometheus metrics for the SLI processing flow. 19 | - Deploy example manifests for Kubernetes. 20 | - Prometheus SLI inputs and SLI result outputs. 21 | - Operator. 22 | - Service level CRD. 23 | 24 | [Unreleased]: https://github.com/spotahome/service-level-operator/compare/v0.3.0...HEAD 25 | [0.3.0]: https://github.com/spotahome/service-level-operator/compare/v0.2.0...v0.3.0 26 | [0.2.0]: https://github.com/spotahome/service-level-operator/compare/v0.1.0...v0.2.0 27 | [0.1.0]: https://github.com/spotahome/service-level-operator/releases/tag/v0.1.0 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [2018] [Spotahome Ltd.] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Name of this service/application 2 | SERVICE_NAME := service-level-operator 3 | 4 | # Shell to use for running scripts 5 | SHELL := $(shell which bash) 6 | 7 | # Get docker path or an empty string 8 | DOCKER := $(shell command -v docker) 9 | 10 | # Get docker-compose path or an empty string 11 | DOCKER_COMPOSE := $(shell command -v docker-compose) 12 | 13 | # Get the main unix group for the user running make (to be used by docker-compose later) 14 | GID := $(shell id -g) 15 | 16 | # Get the unix user id for the user running make (to be used by docker-compose later) 17 | UID := $(shell id -u) 18 | 19 | # Bash history file for container shell 20 | HISTORY_FILE := ~/.bash_history.$(SERVICE_NAME) 21 | 22 | # Version from Git. 23 | VERSION=$(shell git describe --tags --always) 24 | 25 | # Dev direcotry has all the required dev files. 26 | DEV_DIR := ./docker/dev 27 | 28 | # cmds 29 | UNIT_TEST_CMD := ./hack/scripts/unit-test.sh 30 | INTEGRATION_TEST_CMD := ./hack/scripts/integration-test.sh 31 | TEST_ALERTS_CMD := ./hack/scripts/test-alerts.sh 32 | MOCKS_CMD := ./hack/scripts/mockgen.sh 33 | DOCKER_RUN_CMD := docker run \ 34 | -v ${PWD}:/src \ 35 | --rm -it $(SERVICE_NAME) 36 | DOCKER_ALERTS_TEST_RUN_CMD := docker run \ 37 | -v ${PWD}:/prometheus \ 38 | --entrypoint=${TEST_ALERTS_CMD} \ 39 | --rm -it prom/prometheus 40 | BUILD_BINARY_CMD := VERSION=${VERSION} ./hack/scripts/build-binary.sh 41 | BUILD_IMAGE_CMD := VERSION=${VERSION} ./hack/scripts/build-image.sh 42 | DEBUG_CMD := go run ./cmd/service-level-operator/* --debug 43 | DEV_CMD := $(DEBUG_CMD) --development 44 | FAKE_CMD := $(DEV_CMD) --fake 45 | K8S_CODE_GEN_CMD := ./hack/scripts/k8scodegen.sh 46 | OPENAPI_CODE_GEN_CMD := ./hack/scripts/openapicodegen.sh 47 | DEPS_CMD := GO111MODULE=on go mod tidy && GO111MODULE=on go mod vendor 48 | K8S_VERSION := 1.13.12 49 | SET_K8S_DEPS_CMD := GO111MODULE=on go mod edit \ 50 | -require=k8s.io/apiextensions-apiserver@kubernetes-${K8S_VERSION} \ 51 | -require=k8s.io/client-go@kubernetes-${K8S_VERSION} \ 52 | -require=k8s.io/apimachinery@kubernetes-${K8S_VERSION} \ 53 | -require=k8s.io/api@kubernetes-${K8S_VERSION} \ 54 | -require=k8s.io/kubernetes@v${K8S_VERSION} && \ 55 | $(DEPS_CMD) 56 | 57 | 58 | # The default action of this Makefile is to build the development docker image 59 | default: build 60 | 61 | # Test if the dependencies we need to run this Makefile are installed 62 | .PHONY: deps-development 63 | deps-development: 64 | ifndef DOCKER 65 | @echo "Docker is not available. Please install docker" 66 | @exit 1 67 | endif 68 | ifndef DOCKER_COMPOSE 69 | @echo "docker-compose is not available. Please install docker-compose" 70 | @exit 1 71 | endif 72 | 73 | # Build the development docker images 74 | .PHONY: build 75 | build: 76 | docker build -t $(SERVICE_NAME) --build-arg uid=$(UID) --build-arg gid=$(GID) -f $(DEV_DIR)/Dockerfile . 77 | 78 | # run the development stack. 79 | .PHONY: stack 80 | stack: deps-development 81 | cd $(DEV_DIR) && \ 82 | ( docker-compose -p $(SERVICE_NAME) up --build; \ 83 | docker-compose -p $(SERVICE_NAME) stop; \ 84 | docker-compose -p $(SERVICE_NAME) rm -f; ) 85 | 86 | # Build production stuff. 87 | build-binary: build 88 | $(DOCKER_RUN_CMD) /bin/sh -c '$(BUILD_BINARY_CMD)' 89 | 90 | .PHONY: build-image 91 | build-image: 92 | $(BUILD_IMAGE_CMD) 93 | 94 | # Dependencies stuff. 95 | .PHONY: set-k8s-deps 96 | set-k8s-deps: 97 | $(SET_K8S_DEPS_CMD) 98 | 99 | .PHONY: deps 100 | deps: 101 | $(DEPS_CMD) 102 | 103 | k8s-code-gen: 104 | $(K8S_CODE_GEN_CMD) 105 | 106 | openapi-code-gen: 107 | $(OPENAPI_CODE_GEN_CMD) 108 | 109 | # Test stuff in dev 110 | .PHONY: test-alerts 111 | test-alerts: 112 | $(DOCKER_ALERTS_TEST_RUN_CMD) 113 | .PHONY: unit-test 114 | unit-test: build 115 | $(DOCKER_RUN_CMD) /bin/sh -c '$(UNIT_TEST_CMD)' 116 | .PHONY: integration-test 117 | integration-test: build 118 | $(DOCKER_RUN_CMD) /bin/sh -c '$(INTEGRATION_TEST_CMD)' 119 | .PHONY: test 120 | test: integration-test 121 | .PHONY: test 122 | ci: test test-alerts 123 | 124 | # Mocks stuff in dev 125 | .PHONY: mocks 126 | mocks: build 127 | # FIX: Problem using go mod with vektra/mockery. 128 | #$(DOCKER_RUN_CMD) /bin/sh -c '$(MOCKS_CMD)' 129 | $(MOCKS_CMD) 130 | 131 | .PHONY: dev 132 | dev: 133 | $(DEV_CMD) 134 | 135 | 136 | .PHONY: push 137 | push: export PUSH_IMAGE=true 138 | push: build-image 139 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # service-level-operator [![Build Status][travis-image]][travis-url] [![Go Report Card][goreport-image]][goreport-url] [![docker image][quay-image]][quay-url] 2 | 3 | Service level operator abstracts and automates the service level of Kubernetes applications by generation SLI & SLOs to be consumed easily by dashboards and alerts and allow that the SLI/SLO's live with the application flow. 4 | 5 | This operator interacts with Kubernetes using the CRDs as a way to define application service levels and generating output service level metrics. 6 | 7 | Although this operator is though to interact with different backends and generate different output backends, at this moment only uses [Prometheus] as input and output backend. 8 | 9 | ![grafana graphs](img/grafana_graphs1.png) 10 | 11 | ## Example 12 | 13 | For this example the output and input backend will be [Prometheus]. 14 | 15 | First you will need to define a CRD with your service SLI & SLOs. In this case we have a service that has an SLO on 99.99 availability, and the SLI is that 5xx are considered errors. 16 | 17 | ```yaml 18 | apiVersion: monitoring.spotahome.com/v1alpha1 19 | kind: ServiceLevel 20 | metadata: 21 | name: awesome-service 22 | spec: 23 | serviceLevelObjectives: 24 | - name: "9999_http_request_lt_500" 25 | description: 99.99% of requests must be served with <500 status code. 26 | disable: false 27 | availabilityObjectivePercent: 99.99 28 | serviceLevelIndicator: 29 | prometheus: 30 | address: http://myprometheus:9090 31 | totalQuery: sum(increase(http_request_total{host="awesome_service_io"}[2m])) 32 | errorQuery: sum(increase(http_request_total{host="awesome_service_io", code=~"5.."}[2m])) 33 | output: 34 | prometheus: 35 | labels: 36 | team: a-team 37 | iteration: "3" 38 | ``` 39 | 40 | The Operator will generate the SLI and SLO in this prometheus format: 41 | 42 | ```text 43 | # HELP service_level_sli_result_count_total Is the number of times an SLI result has been processed. 44 | # TYPE service_level_sli_result_count_total counter 45 | service_level_sli_result_count_total{service_level="awesome-service",slo="9999_http_request_lt_500"} 1708 46 | # HELP service_level_sli_result_error_ratio_total Is the error or failure ratio of an SLI result. 47 | # TYPE service_level_sli_result_error_ratio_total counter 48 | service_level_sli_result_error_ratio_total{service_level="awesome-service",slo="9999_http_request_lt_500"} 0.40508550763795764 49 | # HELP service_level_slo_objective_ratio Is the objective of the SLO in ratio unit. 50 | # TYPE service_level_slo_objective_ratio gauge 51 | service_level_slo_objective_ratio{service_level="awesome-service",slo="9999_http_request_lt_500"} 0.9998999999999999 52 | ``` 53 | 54 | ## How does it work 55 | 56 | The operator will query and create new metrics based on the SLOs caulculations at regular intervals (see `--resync-seconds` flag). 57 | 58 | The approach that has been taken to generate the SLI results is based on [how Google uses and manages SLIs, SLOs and error budgets][sre-book-slo] 59 | 60 | In the manifest the SLI is made of 2 prometheus metrics: 61 | 62 | - The total of requests: `sum(increase(http_request_total{host="awesome_service_io"}[2m]))` 63 | - The total number of failed requests: `sum(increase(http_request_total{host="awesome_service_io", code=~"5.."}[2m]))` 64 | 65 | By expresing what are the total count on SLI result processing and the error ratio processed the operator will generate the SLO metrics for this service. 66 | 67 | Like is seen in the above output the operator generates 3 metrics: 68 | 69 | - `service_level_sli_result_error_ratio_total`: The _downtime/error_ ratio (0-1) of the service. 70 | - `service_level_sli_result_count_total`: The total count of SLI processed total, in other words, what would be the ratio if the service would be 100% correct all the time becasue ratios are from 0 to 1. 71 | - `service_level_slo_objective_ratio`: The objective of the SLO in ratio. This metrics is't processed at all (only changed to ratio unit), but is important to create error budget quries, alerts... 72 | 73 | With these metrics we can build availability graphs based on % and error budget burns. 74 | 75 | The approach of using counters (instead of gauges) to store the total counts and the error/downtime total gives us the ability to get SLO/SLI rates, increments, speed... in the different time ranges (check query examples section) and is safer in case of missed scrapes, SLI calculation errors... In other words this approach gives us flexibility and safety. 76 | 77 | Is important to note that like every metrics this is not exact and is a aproximation (good one but an approximation after all) 78 | 79 | ## Grafana dashboard 80 | 81 | There is a [grafana dashboard][grafana-dashboard] to show the SLO's status. 82 | 83 | ## Supported input/output backends 84 | 85 | ### Input (SLI sources) 86 | 87 | Inputs for SLIs can be declared at two levels. 88 | 89 | At SLO level (this way we can use different endpoint for each SLO) 90 | 91 | ```yaml 92 | ... 93 | serviceLevelObjectives: 94 | - name: "my_slok" 95 | ... 96 | serviceLevelIndicator: 97 | prometheus: 98 | address: http://myprometheus:9090 99 | ... 100 | ``` 101 | 102 | Also, if any of the SLOs does not have a default input, setting a default SLI source configuration when running the operator will fallback to these. 103 | 104 | The flag is `--def-sli-source-path` and the file format is this: 105 | 106 | ```json 107 | { 108 | "prometheus": { 109 | "address": "http://127.0.0.1:9090" 110 | } 111 | } 112 | ``` 113 | 114 | Example: 115 | 116 | ```bash 117 | --def-sli-source-path <(echo '{"prometheus": {"address": "http://127.0.0.1:12345"}}') 118 | ``` 119 | 120 | List of supported SLI sources: 121 | 122 | - [Prometheus] 123 | 124 | ### Output 125 | 126 | Outputs are how the SLO metrics will be exported. Here is a list of supported output backends: 127 | 128 | - [Prometheus] 129 | 130 | ## Query examples 131 | 132 | ### Availability level rate 133 | 134 | This will output the availability rate of a service based. 135 | 136 | ```text 137 | 1 - ( 138 | rate(service_level_sli_result_error_ratio_total[1m]) 139 | / 140 | rate(service_level_sli_result_count_total[1m]) 141 | ) * 100 142 | ``` 143 | 144 | ### Availability level in the last 24h 145 | 146 | This will output the availability rate of a service based. 147 | 148 | ```text 149 | 1 - ( 150 | increase(service_level_sli_result_error_ratio_total[24h]) 151 | / 152 | increase(service_level_sli_result_count_total[24h]) 153 | ) * 100 154 | ``` 155 | 156 | ### Error budget burn rate 157 | 158 | The way this operator abstracts the SLI results it's easy to get the error budget burn rate without a time range projection, this is because the calculation is constant and based on ratios (0-1) instead of duration, rps, processed messages... 159 | 160 | To know the error budget burn rate we need to get the errors ratio in a interval (eg `5m`): 161 | 162 | ```text 163 | increase(service_level_sli_result_error_ratio_total{service_level="${service_level}", slo="${slo}"}[5m]) 164 | / 165 | increase(service_level_sli_result_count_total{service_level="${service_level}", slo="${slo}"}[${5m}] 166 | ``` 167 | 168 | And to get the maximum burn rate that we can afford so we don't consume all the error budget would be: 169 | 170 | ```text 171 | (1 - service_level_slo_objective_ratio{service_level="${service_level}", slo="${slo}"}) 172 | ``` 173 | 174 | This query gets the max error budget ratio that we can afford (eg: for a 99.99% SLO would be `0.0001` ratio). 175 | 176 | With those 2 queries we know the error ratio (error burn rate) and the max error ratio that we can afford (error budget). 177 | 178 | ### Error budget with a 30d projection and burndown chart 179 | 180 | Calculating the burndown charts is a little bit more tricky. 181 | 182 | #### Context 183 | 184 | - Taking the previous example we are calculating error budget based on 1 month, this are 43200m (30 \* 24 \* 60). 185 | - Our SLO objective is 99.99 (in ratio: 0.9998999999999999) 186 | - Error budget is based in a 100% for 30d that decrements when availability is less than 99.99% (like the SLO specifies). 187 | 188 | #### Query 189 | 190 | ```text 191 | ( 192 | ( 193 | (1 - service_level_slo_objective_ratio) * 43200 * increase(service_level_sli_result_count_total[1m]) 194 | - 195 | increase(service_level_sli_result_error_ratio_total[${range}]) 196 | ) 197 | / 198 | ( 199 | (1 - service_level_slo_objective_ratio) * 43200 * increase(service_level_sli_result_count_total[1m]) 200 | ) 201 | ) * 100 202 | ``` 203 | 204 | Let's decompose the query. 205 | 206 | #### Query explanation 207 | 208 | `(1 - service_level_slo_objective_ratio) * 43200 * increase(service_level_sli_result_count_total[1m])` is the total ratio measured in 1m (sucess + failures) multiplied by the number of minutes in a month and the error budget ratio(1-0.9998999999999999). In other words this is the total (sum) number of error budget for 1 month we have. 209 | 210 | `increase(service_level_sli_result_error_ratio_total[${range}])` this is the SLO error sum that we had in \${range} (range changes over time, the first day of the month will be 1d, the 15th of the month will be 15d). 211 | 212 | So `(1 - service_level_slo_objective_ratio) * 43200 * increase(service_level_sli_result_count_total[1m]) - increase(service_level_sli_result_error_ratio_total[${range}])` returns the number of remaining error budget we have after `${range}`. 213 | 214 | If we take that last part and divide for the total error budget we have for the month (`(1 - service_level_slo_objective_ratio) * 43200 * increase(service_level_sli_result_count_total[1m])`) this returns us a ratio of the error budget consumed. Multiply by 100 and we have the percent of error budget consumed after `${range}`. 215 | 216 | ## Prometheus alerts 217 | 218 | The operator gives the SLIs and SLOs in the same format so we could create 1 alert for all of our SLOs, or be more specific and filter by labels. 219 | 220 | ### Multiple Burn Rate Alerts (SRE workbook) 221 | 222 | This Alert follows Google's SRE approach for alerting based on SLO burn rate and error budget , specifically the one on the [SRE workbook][sre-workbook] Chapter 5.4 (Alert on burn rate), the 5th approach (Multiple burn rate alerts). 223 | 224 | ```yaml 225 | groups: 226 | - name: slo.rules 227 | rules: 228 | - alert: SLOErrorRateTooFast1h 229 | expr: | 230 | ( 231 | increase(service_level_sli_result_error_ratio_total[1h]) 232 | / 233 | increase(service_level_sli_result_count_total[1h]) 234 | ) > (1 - service_level_slo_objective_ratio) * 14.6 235 | labels: 236 | severity: critical 237 | team: a-team 238 | annotations: 239 | summary: The monthly SLO error budget consumed for 1h is greater than 2% 240 | description: The error rate for 1h in the {{$labels.service_level}}/{{$labels.slo}} SLO error budget is being consumed too fast, is greater than 2% monthly budget. 241 | - alert: SLOErrorRateTooFast6h 242 | expr: | 243 | ( 244 | increase(service_level_sli_result_error_ratio_total[6h]) 245 | / 246 | increase(service_level_sli_result_count_total[6h]) 247 | ) > (1 - service_level_slo_objective_ratio) * 6 248 | labels: 249 | severity: critical 250 | team: a-team 251 | annotations: 252 | summary: The monthly SLO error budget consumed for 6h is greater than 5% 253 | description: The error rate for 6h in the {{$labels.service_level}}/{{$labels.slo}} SLO error budget is being consumed too fast, is greater than 5% monthly budget. 254 | ``` 255 | 256 | This alert will trigger if the error budget consumed in 1h is greater than the 2% for 30 days or in 6h if greater than 5%. This numbers are the recomended ones by Google as a baseline based on their experience over the years. 257 | 258 | | SLO monthly budget burned | time range | burn rate to consume this percentage | 259 | | ------------------------- | ---------- | ------------------------------------ | 260 | | 2% | 1h | 730 \* 2 / 100 = 14.6 | 261 | | 5% | 6h | 730 / 6 \* 5 / 100 = 6 | 262 | | 10% | 3d | 30 / 3 \* 10 / 100 = 1 | 263 | 264 | ### Multiwindow, Multi-Burn-Rate Alerts (SRE workbook) 265 | 266 | This alert kind is extracted from the [SRE workbook][sre-workbook] Chapter 5.4 (Alert on burn rate), the 6th approach (Multiwindow, Multi-burn-rate alerts) 267 | 268 | Our previous alerts could happen that a big error rate peak in 5m could be enough to bypass the SLO threshold for 60m, so we can add a second check to the previous alert to check if the error rate countinues by passing the SLO error budget threshold. For example checking the past 5m or 10m. 269 | 270 | Check the alert [here][multiwindow-alert] 271 | 272 | [travis-image]: https://travis-ci.org/spotahome/service-level-operator.svg?branch=master 273 | [travis-url]: https://travis-ci.org/spotahome/service-level-operator 274 | [goreport-image]: https://goreportcard.com/badge/github.com/spotahome/service-level-operator 275 | [goreport-url]: https://goreportcard.com/report/github.com/spotahome/service-level-operator 276 | [quay-image]: https://quay.io/repository/spotahome/service-level-operator/status 277 | [quay-url]: https://quay.io/repository/spotahome/service-level-operator 278 | [sre-book-slo]: https://landing.google.com/sre/book/chapters/service-level-objectives.html 279 | [prometheus]: https://prometheus.io/ 280 | [grafana-dashboard]: https://grafana.com/dashboards/8793 281 | [sre-workbook]: https://books.google.es/books?id=fElmDwAAQBAJ 282 | [multiwindow-alert]: alerts/slo.yaml 283 | -------------------------------------------------------------------------------- /alerts/slo.yaml: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: slo.rules 3 | rules: 4 | - alert: SLOErrorRateTooFast1h 5 | expr: | 6 | ( 7 | increase(service_level_sli_result_error_ratio_total[1h]) 8 | / 9 | increase(service_level_sli_result_count_total[1h]) 10 | ) > (1 - service_level_slo_objective_ratio) * 0.02 11 | and 12 | ( 13 | increase(service_level_sli_result_error_ratio_total[5m]) 14 | / 15 | increase(service_level_sli_result_count_total[5m]) 16 | ) > (1 - service_level_slo_objective_ratio) * 0.02 17 | 18 | labels: 19 | severity: critical 20 | team: a-team 21 | annotations: 22 | summary: The SLO error budget burn rate for 1h is greater than 2% 23 | description: The error rate for 1h in the {{$labels.service_level}}/{{$labels.slo}} SLO error budget is too fast, is greater than the total error budget 2%. 24 | - alert: SLOErrorRateTooFast6h 25 | expr: | 26 | ( 27 | increase(service_level_sli_result_error_ratio_total[6h]) 28 | / 29 | increase(service_level_sli_result_count_total[6h]) 30 | ) > (1 - service_level_slo_objective_ratio) * 0.05 31 | and 32 | ( 33 | increase(service_level_sli_result_error_ratio_total[30m]) 34 | / 35 | increase(service_level_sli_result_count_total[30m]) 36 | ) > (1 - service_level_slo_objective_ratio) * 0.05 37 | labels: 38 | severity: critical 39 | team: a-team 40 | annotations: 41 | summary: The SLO error budget burn rate for 6h is greater than 5% 42 | description: The error rate for 6h in the {{$labels.service_level}}/{{$labels.slo}} SLO error budget is too fast, is greater than the total error budget 5%. 43 | -------------------------------------------------------------------------------- /alerts/slo_test.yaml: -------------------------------------------------------------------------------- 1 | rule_files: 2 | - slo.yaml 3 | 4 | evaluation_interval: 1m 5 | 6 | tests: 7 | # This test will test the alert triggers with 1h of errors bypassing the error budget. 8 | # count ratio total in 60m = count_total * 60m = xxx 9 | # count ratio total in 60m = 1 * 60m = 60 10 | # Error ratio total in 60m = error_total * 60m = xxx 11 | # Error ratio total in 60m = 0.00021* 60m = 0.0126 12 | # Error ration in 60m = Error ratio total / Count ratio total = 0.0021 13 | # Error budget 2% = (error budget * 2%) = xxx 14 | # Error budget 2% = 0.01 * 00.02 = 0.0002 15 | # Should trigger alert? = 0.0021 > 0.0002 = true 16 | - interval: 1m 17 | input_series: 18 | - series: 'service_level_sli_result_error_ratio_total{service_level="sl1",slo="slo1"}' 19 | values: "0+0.00021x120" 20 | - series: 'service_level_sli_result_count_total{service_level="sl1",slo="slo1"}' 21 | values: "0+1x120" 22 | - series: 'service_level_slo_objective_ratio{service_level="sl1",slo="slo1"}' 23 | values: "0.99+0x120" 24 | 25 | alert_rule_test: 26 | - eval_time: 65m 27 | alertname: SLOErrorRateTooFast1h 28 | exp_alerts: 29 | - exp_labels: 30 | severity: critical 31 | team: a-team 32 | service_level: sl1 33 | slo: slo1 34 | exp_annotations: 35 | summary: "The SLO error budget burn rate for 1h is greater than 2%" 36 | description: "The error rate for 1h in the sl1/slo1 SLO error budget is too fast, is greater than the total error budget 2%." 37 | -------------------------------------------------------------------------------- /cmd/service-level-operator/flags.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "path/filepath" 7 | "time" 8 | 9 | "k8s.io/client-go/util/homedir" 10 | 11 | "github.com/spotahome/service-level-operator/pkg/operator" 12 | ) 13 | 14 | // defaults 15 | const ( 16 | defMetricsPath = "/metrics" 17 | defListenAddress = ":8080" 18 | defResyncSeconds = 5 19 | defWorkers = 10 20 | ) 21 | 22 | type cmdFlags struct { 23 | fs *flag.FlagSet 24 | 25 | kubeConfig string 26 | resyncSeconds int 27 | workers int 28 | metricsPath string 29 | listenAddress string 30 | labelSelector string 31 | namespace string 32 | defSLISourcePath string 33 | debug bool 34 | development bool 35 | fake bool 36 | } 37 | 38 | func newCmdFlags() *cmdFlags { 39 | c := &cmdFlags{ 40 | fs: flag.NewFlagSet(os.Args[0], flag.ExitOnError), 41 | } 42 | c.init() 43 | 44 | return c 45 | } 46 | 47 | func (c *cmdFlags) init() { 48 | 49 | kubehome := filepath.Join(homedir.HomeDir(), ".kube", "config") 50 | // register flags 51 | c.fs.StringVar(&c.kubeConfig, "kubeconfig", kubehome, "kubernetes configuration path, only used when development mode enabled") 52 | c.fs.StringVar(&c.metricsPath, "metrics-path", defMetricsPath, "the path where the metrics will be served") 53 | c.fs.StringVar(&c.listenAddress, "listen-addr", defListenAddress, "the address where the metrics will be exposed") 54 | c.fs.StringVar(&c.labelSelector, "selector", "", "selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2)") 55 | c.fs.StringVar(&c.namespace, "namespace", "", "the namespace to filter on, by default all") 56 | c.fs.StringVar(&c.defSLISourcePath, "def-sli-source-path", "", "the path to the default sli sources configuration file") 57 | c.fs.IntVar(&c.resyncSeconds, "resync-seconds", defResyncSeconds, "the number of seconds for the SLO calculation interval") 58 | c.fs.IntVar(&c.workers, "workers", defWorkers, "the number of concurrent workers per controller handling events") 59 | c.fs.BoolVar(&c.development, "development", false, "development flag will allow to run the operator outside a kubernetes cluster") 60 | c.fs.BoolVar(&c.debug, "debug", false, "enable debug mode") 61 | c.fs.BoolVar(&c.fake, "fake", false, "enable faked mode, in faked node external services/dependencies are not needed") 62 | 63 | // Parse flags 64 | c.fs.Parse(os.Args[1:]) 65 | } 66 | 67 | func (c *cmdFlags) toOperatorConfig() operator.Config { 68 | return operator.Config{ 69 | ResyncPeriod: time.Duration(c.resyncSeconds) * time.Second, 70 | ConcurretWorkers: c.workers, 71 | LabelSelector: c.labelSelector, 72 | Namespace: c.namespace, 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /cmd/service-level-operator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/oklog/run" 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 16 | "k8s.io/client-go/kubernetes" 17 | _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" 18 | "k8s.io/client-go/rest" 19 | "k8s.io/client-go/tools/clientcmd" 20 | 21 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 22 | "github.com/spotahome/service-level-operator/pkg/log" 23 | "github.com/spotahome/service-level-operator/pkg/operator" 24 | kubernetesclifactory "github.com/spotahome/service-level-operator/pkg/service/client/kubernetes" 25 | promclifactory "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 26 | "github.com/spotahome/service-level-operator/pkg/service/configuration" 27 | kubernetesservice "github.com/spotahome/service-level-operator/pkg/service/kubernetes" 28 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 29 | ) 30 | 31 | const ( 32 | kubeCliQPS = 100 33 | kubeCliBurst = 100 34 | gracePeriod = 2 * time.Second 35 | ) 36 | 37 | // Main has the main logic of the app. 38 | type Main struct { 39 | flags *cmdFlags 40 | logger log.Logger 41 | } 42 | 43 | // Run runs the main program. 44 | func (m *Main) Run() error { 45 | // Prepare the logger with the correct settings. 46 | jsonLog := true 47 | if m.flags.development { 48 | jsonLog = false 49 | } 50 | m.logger = log.Base(jsonLog) 51 | if m.flags.debug { 52 | m.logger.Set("debug") 53 | } 54 | 55 | if m.flags.fake { 56 | m.logger = m.logger.With("mode", "fake") 57 | m.logger.Warnf("running in faked mode, any external service will be faked") 58 | } 59 | 60 | // Create prometheus registry and metrics service to expose and measure with metrics. 61 | promReg := prometheus.NewRegistry() 62 | metricssvc := metrics.NewPrometheus(promReg) 63 | 64 | // Create services 65 | k8sstdcli, k8scrdcli, k8saexcli, err := m.createKubernetesClients() 66 | if err != nil { 67 | return err 68 | } 69 | k8ssvc := kubernetesservice.New(k8sstdcli, k8scrdcli, k8saexcli, m.logger) 70 | 71 | // Prepare our run entrypoints. 72 | var g run.Group 73 | 74 | // OS signals. 75 | { 76 | sigC := make(chan os.Signal, 1) 77 | exitC := make(chan struct{}) 78 | signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT) 79 | 80 | g.Add( 81 | func() error { 82 | select { 83 | case s := <-sigC: 84 | m.logger.Infof("signal %s received", s) 85 | return nil 86 | case <-exitC: 87 | return nil 88 | } 89 | }, 90 | func(_ error) { 91 | close(exitC) 92 | }, 93 | ) 94 | } 95 | 96 | // Metrics. 97 | { 98 | s := m.createHTTPServer(promReg) 99 | g.Add( 100 | func() error { 101 | m.logger.Infof("metrics server listening on %s", m.flags.listenAddress) 102 | return s.ListenAndServe() 103 | }, 104 | func(_ error) { 105 | m.logger.Infof("draining metrics server connections") 106 | ctx, cancel := context.WithTimeout(context.Background(), gracePeriod) 107 | defer cancel() 108 | err := s.Shutdown(ctx) 109 | if err != nil { 110 | m.logger.Errorf("error while drainning connections on metrics sever") 111 | } 112 | }, 113 | ) 114 | } 115 | 116 | // Operator. 117 | { 118 | 119 | // Load configuration. 120 | var cfgSLISrc *configuration.DefaultSLISource 121 | if m.flags.defSLISourcePath != "" { 122 | f, err := os.Open(m.flags.defSLISourcePath) 123 | if err != nil { 124 | return err 125 | } 126 | defer f.Close() 127 | cfgSLISrc, err = configuration.JSONLoader{}.LoadDefaultSLISource(context.Background(), f) 128 | if err != nil { 129 | return err 130 | } 131 | } 132 | 133 | // Create SLI source client factories. 134 | promCliFactory, err := m.createPrometheusCliFactory(cfgSLISrc) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | cfg := m.flags.toOperatorConfig() 140 | op, err := operator.New(cfg, promReg, promCliFactory, k8ssvc, metricssvc, m.logger) 141 | if err != nil { 142 | return err 143 | } 144 | closeC := make(chan struct{}) 145 | 146 | g.Add( 147 | func() error { 148 | return op.Run(closeC) 149 | }, 150 | func(_ error) { 151 | close(closeC) 152 | }, 153 | ) 154 | } 155 | 156 | // Run everything 157 | return g.Run() 158 | } 159 | 160 | // loadKubernetesConfig loads kubernetes configuration based on flags. 161 | func (m *Main) loadKubernetesConfig() (*rest.Config, error) { 162 | var cfg *rest.Config 163 | // If devel mode then use configuration flag path. 164 | if m.flags.development { 165 | config, err := clientcmd.BuildConfigFromFlags("", m.flags.kubeConfig) 166 | if err != nil { 167 | return nil, fmt.Errorf("could not load configuration: %s", err) 168 | } 169 | cfg = config 170 | } else { 171 | config, err := rest.InClusterConfig() 172 | if err != nil { 173 | return nil, fmt.Errorf("error loading kubernetes configuration inside cluster, check app is running outside kubernetes cluster or run in development mode: %s", err) 174 | } 175 | cfg = config 176 | } 177 | 178 | // Set better cli rate limiter. 179 | cfg.QPS = kubeCliQPS 180 | cfg.Burst = kubeCliBurst 181 | 182 | return cfg, nil 183 | } 184 | 185 | func (m *Main) createKubernetesClients() (kubernetes.Interface, crdcli.Interface, apiextensionscli.Interface, error) { 186 | var factory kubernetesclifactory.ClientFactory 187 | 188 | if m.flags.fake { 189 | factory = kubernetesclifactory.NewFake() 190 | } else { 191 | config, err := m.loadKubernetesConfig() 192 | if err != nil { 193 | return nil, nil, nil, err 194 | } 195 | factory = kubernetesclifactory.NewFactory(config) 196 | } 197 | 198 | stdcli, err := factory.GetSTDClient() 199 | if err != nil { 200 | return nil, nil, nil, err 201 | } 202 | 203 | crdcli, err := factory.GetCRDClient() 204 | if err != nil { 205 | return nil, nil, nil, err 206 | } 207 | 208 | aexcli, err := factory.GetAPIExtensionClient() 209 | if err != nil { 210 | return nil, nil, nil, err 211 | } 212 | 213 | return stdcli, crdcli, aexcli, nil 214 | } 215 | 216 | func (m *Main) createPrometheusCliFactory(cfg *configuration.DefaultSLISource) (promclifactory.ClientFactory, error) { 217 | if m.flags.fake { 218 | return promclifactory.NewFakeFactory(), nil 219 | } 220 | 221 | f := promclifactory.NewBaseFactory() 222 | if cfg != nil && cfg.Prometheus.Address != "" { 223 | err := f.WithDefaultV1APIClient(cfg.Prometheus.Address) 224 | if err != nil { 225 | return nil, err 226 | } 227 | m.logger.Infof("prometheus default SLI source set to: %s", cfg.Prometheus.Address) 228 | } 229 | 230 | return f, nil 231 | } 232 | 233 | // createHTTPServer creates the http server that serves prometheus metrics and healthchecks. 234 | func (m *Main) createHTTPServer(promReg *prometheus.Registry) http.Server { 235 | h := promhttp.HandlerFor(promReg, promhttp.HandlerOpts{}) 236 | mux := http.NewServeMux() 237 | mux.Handle(m.flags.metricsPath, h) 238 | mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 239 | w.Write([]byte(` 240 | Service level operator 241 | 242 |

Service level operator

243 |

Metrics

244 | 245 | `)) 246 | }) 247 | mux.HandleFunc("/healthz/ready", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`ready`)) }) 248 | mux.HandleFunc("/healthz/live", func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`live`)) }) 249 | 250 | return http.Server{ 251 | Handler: mux, 252 | Addr: m.flags.listenAddress, 253 | } 254 | } 255 | 256 | func main() { 257 | m := &Main{flags: newCmdFlags()} 258 | 259 | // Party time! 260 | err := m.Run() 261 | if err != nil { 262 | fmt.Fprintf(os.Stderr, "error running app: %s", err) 263 | os.Exit(1) 264 | } 265 | 266 | fmt.Fprintf(os.Stdout, "see you soon, good bye!") 267 | os.Exit(0) 268 | } 269 | -------------------------------------------------------------------------------- /deploy/Readme.md: -------------------------------------------------------------------------------- 1 | # Delivery 2 | 3 | In deploy/manifests there are example manifests to deploy this operator. 4 | 5 | - Set the correct namespaces on the manifests. 6 | - Set the correct namespace on the service account. 7 | - If you are using [prometheus-operator] check `deploy/manifests/prometheus.yaml` and edit accordingly. 8 | - Image is set to `latest`, this is only the example, it's a bad practice to not use versioned applications. 9 | 10 | [prometheus-operator]: https://github.com/coreos/prometheus-operator 11 | -------------------------------------------------------------------------------- /deploy/manifests/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: service-level-operator 13 | component: app 14 | strategy: 15 | rollingUpdate: 16 | maxUnavailable: 0 17 | template: 18 | metadata: 19 | labels: 20 | app: service-level-operator 21 | component: app 22 | spec: 23 | serviceAccountName: service-level-operator 24 | containers: 25 | - name: app 26 | imagePullPolicy: Always 27 | image: quay.io/spotahome/service-level-operator:latest 28 | ports: 29 | - containerPort: 8080 30 | name: http 31 | protocol: TCP 32 | readinessProbe: 33 | httpGet: 34 | path: /healthz/ready 35 | port: http 36 | livenessProbe: 37 | httpGet: 38 | path: /healthz/live 39 | port: http 40 | resources: 41 | limits: 42 | cpu: 220m 43 | memory: 254Mi 44 | requests: 45 | cpu: 120m 46 | memory: 128Mi 47 | -------------------------------------------------------------------------------- /deploy/manifests/prometheus.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.coreos.com/v1 2 | kind: ServiceMonitor 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | prometheus: myprometheus 9 | spec: 10 | selector: 11 | matchLabels: 12 | app: service-level-operator 13 | component: app 14 | namespaceSelector: 15 | matchNames: 16 | - app-namespace 17 | endpoints: 18 | - port: http 19 | interval: 10s 20 | -------------------------------------------------------------------------------- /deploy/manifests/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | 9 | --- 10 | apiVersion: rbac.authorization.k8s.io/v1 11 | kind: ClusterRole 12 | metadata: 13 | name: service-level-operator 14 | labels: 15 | app: service-level-operator 16 | component: app 17 | rules: 18 | # Register and check CRDs. 19 | - apiGroups: 20 | - apiextensions.k8s.io 21 | resources: 22 | - customresourcedefinitions 23 | verbs: 24 | - "*" 25 | 26 | # Operator logic. 27 | - apiGroups: 28 | - monitoring.spotahome.com 29 | resources: 30 | - servicelevels 31 | - servicelevels/status 32 | verbs: 33 | - "*" 34 | 35 | --- 36 | kind: ClusterRoleBinding 37 | apiVersion: rbac.authorization.k8s.io/v1 38 | metadata: 39 | name: service-level-operator 40 | subjects: 41 | - kind: ServiceAccount 42 | name: service-level-operator 43 | #namespace: test 44 | roleRef: 45 | apiGroup: rbac.authorization.k8s.io 46 | kind: ClusterRole 47 | name: service-level-operator 48 | -------------------------------------------------------------------------------- /deploy/manifests/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: service-level-operator 5 | labels: 6 | app: service-level-operator 7 | component: app 8 | spec: 9 | ports: 10 | - port: 80 11 | protocol: TCP 12 | name: http 13 | targetPort: http 14 | selector: 15 | app: service-level-operator 16 | component: app 17 | -------------------------------------------------------------------------------- /docker/dev/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine 2 | 3 | RUN apk --no-cache add \ 4 | bash \ 5 | git \ 6 | g++ \ 7 | curl \ 8 | openssl \ 9 | openssh-client 10 | 11 | # Mock creator 12 | RUN go get -u github.com/vektra/mockery/.../ 13 | 14 | RUN mkdir /src 15 | 16 | # Create user 17 | ARG uid=1000 18 | ARG gid=1000 19 | RUN addgroup -g $gid service-level-operator && \ 20 | adduser -D -u $uid -G service-level-operator service-level-operator && \ 21 | chown service-level-operator:service-level-operator -R /src && \ 22 | chown service-level-operator:service-level-operator -R /go 23 | USER service-level-operator 24 | 25 | # Fill go mod cache. 26 | RUN mkdir /tmp/cache 27 | COPY go.mod /tmp/cache 28 | COPY go.sum /tmp/cache 29 | RUN cd /tmp/cache && \ 30 | go mod download 31 | 32 | WORKDIR /src 33 | -------------------------------------------------------------------------------- /docker/dev/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | operator: 5 | build: 6 | context: ../.. 7 | dockerfile: docker/dev/Dockerfile 8 | volumes: 9 | - ../..:/src 10 | - ~/.bash_history.service-level-operator:/home/service-level-operator/.bash_history 11 | - ~/.kube:/home/service-level-operator/.kube:ro 12 | - ~/.gitconfig:/home/service-level-operator/.gitconfig:ro 13 | - ~/.ssh:/home/service-level-operator/.ssh 14 | command: ["go", "run", "./cmd/service-level-operator/main.go", "./cmd/service-level-operator/flags.go", "-development", "-debug", "-fake"] 15 | ports: 16 | - "8080:8080" 17 | 18 | prometheus: 19 | image: prom/prometheus 20 | volumes: 21 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 22 | ports: 23 | - 9090:9090 -------------------------------------------------------------------------------- /docker/dev/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 10s 3 | 4 | scrape_configs: 5 | - job_name: service-level-operator 6 | scrape_interval: 10s 7 | static_configs: 8 | - targets: ["operator:8080"] 9 | labels: 10 | mode: fake 11 | environment: dev 12 | -------------------------------------------------------------------------------- /docker/prod/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13-alpine AS build-stage 2 | 3 | RUN apk --no-cache add \ 4 | g++ \ 5 | git \ 6 | make 7 | 8 | ARG VERSION 9 | ENV VERSION=${VERSION} 10 | WORKDIR /src 11 | COPY . . 12 | RUN ./hack/scripts/build-binary.sh 13 | 14 | # Final image. 15 | FROM alpine:latest 16 | RUN apk --no-cache add \ 17 | ca-certificates 18 | COPY --from=build-stage /src/bin/service-level-operator /usr/local/bin/service-level-operator 19 | ENTRYPOINT ["/usr/local/bin/service-level-operator"] 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/spotahome/service-level-operator 2 | 3 | require ( 4 | github.com/go-openapi/spec v0.17.0 5 | github.com/oklog/run v1.0.0 6 | github.com/prometheus/client_golang v1.2.1 7 | github.com/prometheus/common v0.7.0 8 | github.com/sirupsen/logrus v1.4.2 9 | github.com/spotahome/kooper v0.6.1-0.20190926114429-1c6a0cfab9a5 10 | github.com/stretchr/testify v1.4.0 11 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 12 | k8s.io/api v0.0.0-20191004102255-dacd7df5a50b // indirect 13 | k8s.io/apiextensions-apiserver v0.0.0-20191004105443-a7d558db75c6 14 | k8s.io/apimachinery v0.0.0-20191004074956-01f8b7d1121a 15 | k8s.io/client-go v0.0.0-20191004102537-eb5b9a8cfde7 16 | k8s.io/kube-openapi v0.0.0-20190918143330-0270cf2f1c1d 17 | ) 18 | 19 | go 1.13 20 | -------------------------------------------------------------------------------- /hack/scripts/build-binary.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | goos=linux 7 | goarch=amd64 8 | src=./cmd/service-level-operator 9 | out=./bin/service-level-operator 10 | ldf_cmp="-w -extldflags '-static'" 11 | f_ver="-X main.Version=${VERSION:-dev}" 12 | 13 | echo "Building binary at ${out}" 14 | 15 | GOOS=${goos} GOARCH=${goarch} CGO_ENABLED=0 go build -o ${out} --ldflags "${ldf_cmp} ${f_ver}" ${src} -------------------------------------------------------------------------------- /hack/scripts/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | if [ -z ${VERSION} ]; then 6 | echo "VERSION env var needs to be set" 7 | exit 1 8 | fi 9 | 10 | REPOSITORY="quay.io/spotahome/" 11 | IMAGE="service-level-operator" 12 | TARGET_IMAGE=${REPOSITORY}${IMAGE} 13 | 14 | 15 | docker build \ 16 | --build-arg VERSION=${VERSION} \ 17 | -t ${TARGET_IMAGE}:${VERSION} \ 18 | -t ${TARGET_IMAGE}:latest \ 19 | -f ./docker/prod/Dockerfile . 20 | 21 | if [ -n "${PUSH_IMAGE}" ]; then 22 | echo "pushing ${TARGET_IMAGE} images..." 23 | docker push ${TARGET_IMAGE} 24 | fi -------------------------------------------------------------------------------- /hack/scripts/integration-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go test `go list ./... | grep -v vendor` -v -tags='integration' -------------------------------------------------------------------------------- /hack/scripts/k8scodegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | CODE_GENERATOR_IMAGE=quay.io/slok/kube-code-generator:v1.13.5 4 | DIRECTORY=${PWD} 5 | CODE_GENERATOR_PACKAGE=github.com/spotahome/service-level-operator 6 | 7 | docker run --rm -it \ 8 | -v ${DIRECTORY}:/go/src/${CODE_GENERATOR_PACKAGE} \ 9 | -e PROJECT_PACKAGE=${CODE_GENERATOR_PACKAGE} \ 10 | -e CLIENT_GENERATOR_OUT=${CODE_GENERATOR_PACKAGE}/pkg/k8sautogen/client \ 11 | -e APIS_ROOT=${CODE_GENERATOR_PACKAGE}/pkg/apis \ 12 | -e GROUPS_VERSION="monitoring:v1alpha1" \ 13 | -e GENERATION_TARGETS="deepcopy,client" \ 14 | ${CODE_GENERATOR_IMAGE} -------------------------------------------------------------------------------- /hack/scripts/mockgen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go generate ./mocks -------------------------------------------------------------------------------- /hack/scripts/openapicodegen.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | DIR="$( cd "$( dirname "${0}" )" && pwd )" 4 | ROOT_DIR=${DIR}/../.. 5 | 6 | PROJECT_PACKAGE=github.com/spotahome/service-level-operator 7 | IMAGE=quay.io/slok/kube-code-generator:v1.11.3 8 | 9 | # Execute once per package because we want independent output specs per kind/version. 10 | docker run -it --rm \ 11 | -v ${ROOT_DIR}:/go/src/${PROJECT_PACKAGE} \ 12 | -e CRD_PACKAGES=${PROJECT_PACKAGE}/pkg/apis/monitoring/v1alpha1 \ 13 | -e OPENAPI_OUTPUT_PACKAGE=${PROJECT_PACKAGE}/pkg/apis/monitoring/v1alpha1 \ 14 | ${IMAGE} ./update-openapi.sh -------------------------------------------------------------------------------- /hack/scripts/test-alerts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$(dirname "$(readlink -f $0)")" 4 | ROOT_DIR="${DIR}/../.." 5 | ALERTS=${ROOT_DIR}/alerts 6 | 7 | promtool test rules ${ALERTS}/*_test.yaml -------------------------------------------------------------------------------- /hack/scripts/unit-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -o errexit 4 | set -o nounset 5 | 6 | go test `go list ./... | grep -v vendor` -v -------------------------------------------------------------------------------- /img/grafana_graphs1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/slok/service-level-operator/cc67192dac7556d87af78966f789cdba751696a9/img/grafana_graphs1.png -------------------------------------------------------------------------------- /mocks/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package mocks will have all the mocks of the application, we'll try to use mocking using blackbox 3 | testing and integration tests whenever is possible. 4 | */ 5 | package mocks // import "github.com/spotahome/service-level-operator/mocks" 6 | 7 | // Service mocks. 8 | //go:generate mockery -output ./service/sli -outpkg sli -dir ../pkg/service/sli -name Retriever 9 | //go:generate mockery -output ./service/output -outpkg slo -dir ../pkg/service/output -name Output 10 | 11 | // Third party 12 | //go:generate mockery -output ./github.com/prometheus/client_golang/api/prometheus/v1 -outpkg v1 -dir . -name API 13 | -------------------------------------------------------------------------------- /mocks/github.com/prometheus/client_golang/api/prometheus/v1/API.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package v1 4 | 5 | import api "github.com/prometheus/client_golang/api" 6 | import context "context" 7 | import mock "github.com/stretchr/testify/mock" 8 | 9 | import model "github.com/prometheus/common/model" 10 | import time "time" 11 | import v1 "github.com/prometheus/client_golang/api/prometheus/v1" 12 | 13 | // API is an autogenerated mock type for the API type 14 | type API struct { 15 | mock.Mock 16 | } 17 | 18 | // AlertManagers provides a mock function with given fields: ctx 19 | func (_m *API) AlertManagers(ctx context.Context) (v1.AlertManagersResult, error) { 20 | ret := _m.Called(ctx) 21 | 22 | var r0 v1.AlertManagersResult 23 | if rf, ok := ret.Get(0).(func(context.Context) v1.AlertManagersResult); ok { 24 | r0 = rf(ctx) 25 | } else { 26 | r0 = ret.Get(0).(v1.AlertManagersResult) 27 | } 28 | 29 | var r1 error 30 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 31 | r1 = rf(ctx) 32 | } else { 33 | r1 = ret.Error(1) 34 | } 35 | 36 | return r0, r1 37 | } 38 | 39 | // Alerts provides a mock function with given fields: ctx 40 | func (_m *API) Alerts(ctx context.Context) (v1.AlertsResult, error) { 41 | ret := _m.Called(ctx) 42 | 43 | var r0 v1.AlertsResult 44 | if rf, ok := ret.Get(0).(func(context.Context) v1.AlertsResult); ok { 45 | r0 = rf(ctx) 46 | } else { 47 | r0 = ret.Get(0).(v1.AlertsResult) 48 | } 49 | 50 | var r1 error 51 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 52 | r1 = rf(ctx) 53 | } else { 54 | r1 = ret.Error(1) 55 | } 56 | 57 | return r0, r1 58 | } 59 | 60 | // CleanTombstones provides a mock function with given fields: ctx 61 | func (_m *API) CleanTombstones(ctx context.Context) error { 62 | ret := _m.Called(ctx) 63 | 64 | var r0 error 65 | if rf, ok := ret.Get(0).(func(context.Context) error); ok { 66 | r0 = rf(ctx) 67 | } else { 68 | r0 = ret.Error(0) 69 | } 70 | 71 | return r0 72 | } 73 | 74 | // Config provides a mock function with given fields: ctx 75 | func (_m *API) Config(ctx context.Context) (v1.ConfigResult, error) { 76 | ret := _m.Called(ctx) 77 | 78 | var r0 v1.ConfigResult 79 | if rf, ok := ret.Get(0).(func(context.Context) v1.ConfigResult); ok { 80 | r0 = rf(ctx) 81 | } else { 82 | r0 = ret.Get(0).(v1.ConfigResult) 83 | } 84 | 85 | var r1 error 86 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 87 | r1 = rf(ctx) 88 | } else { 89 | r1 = ret.Error(1) 90 | } 91 | 92 | return r0, r1 93 | } 94 | 95 | // DeleteSeries provides a mock function with given fields: ctx, matches, startTime, endTime 96 | func (_m *API) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { 97 | ret := _m.Called(ctx, matches, startTime, endTime) 98 | 99 | var r0 error 100 | if rf, ok := ret.Get(0).(func(context.Context, []string, time.Time, time.Time) error); ok { 101 | r0 = rf(ctx, matches, startTime, endTime) 102 | } else { 103 | r0 = ret.Error(0) 104 | } 105 | 106 | return r0 107 | } 108 | 109 | // Flags provides a mock function with given fields: ctx 110 | func (_m *API) Flags(ctx context.Context) (v1.FlagsResult, error) { 111 | ret := _m.Called(ctx) 112 | 113 | var r0 v1.FlagsResult 114 | if rf, ok := ret.Get(0).(func(context.Context) v1.FlagsResult); ok { 115 | r0 = rf(ctx) 116 | } else { 117 | if ret.Get(0) != nil { 118 | r0 = ret.Get(0).(v1.FlagsResult) 119 | } 120 | } 121 | 122 | var r1 error 123 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 124 | r1 = rf(ctx) 125 | } else { 126 | r1 = ret.Error(1) 127 | } 128 | 129 | return r0, r1 130 | } 131 | 132 | // LabelNames provides a mock function with given fields: ctx 133 | func (_m *API) LabelNames(ctx context.Context) ([]string, api.Warnings, error) { 134 | ret := _m.Called(ctx) 135 | 136 | var r0 []string 137 | if rf, ok := ret.Get(0).(func(context.Context) []string); ok { 138 | r0 = rf(ctx) 139 | } else { 140 | if ret.Get(0) != nil { 141 | r0 = ret.Get(0).([]string) 142 | } 143 | } 144 | 145 | var r1 api.Warnings 146 | if rf, ok := ret.Get(1).(func(context.Context) api.Warnings); ok { 147 | r1 = rf(ctx) 148 | } else { 149 | if ret.Get(1) != nil { 150 | r1 = ret.Get(1).(api.Warnings) 151 | } 152 | } 153 | 154 | var r2 error 155 | if rf, ok := ret.Get(2).(func(context.Context) error); ok { 156 | r2 = rf(ctx) 157 | } else { 158 | r2 = ret.Error(2) 159 | } 160 | 161 | return r0, r1, r2 162 | } 163 | 164 | // LabelValues provides a mock function with given fields: ctx, label 165 | func (_m *API) LabelValues(ctx context.Context, label string) (model.LabelValues, api.Warnings, error) { 166 | ret := _m.Called(ctx, label) 167 | 168 | var r0 model.LabelValues 169 | if rf, ok := ret.Get(0).(func(context.Context, string) model.LabelValues); ok { 170 | r0 = rf(ctx, label) 171 | } else { 172 | if ret.Get(0) != nil { 173 | r0 = ret.Get(0).(model.LabelValues) 174 | } 175 | } 176 | 177 | var r1 api.Warnings 178 | if rf, ok := ret.Get(1).(func(context.Context, string) api.Warnings); ok { 179 | r1 = rf(ctx, label) 180 | } else { 181 | if ret.Get(1) != nil { 182 | r1 = ret.Get(1).(api.Warnings) 183 | } 184 | } 185 | 186 | var r2 error 187 | if rf, ok := ret.Get(2).(func(context.Context, string) error); ok { 188 | r2 = rf(ctx, label) 189 | } else { 190 | r2 = ret.Error(2) 191 | } 192 | 193 | return r0, r1, r2 194 | } 195 | 196 | // Query provides a mock function with given fields: ctx, query, ts 197 | func (_m *API) Query(ctx context.Context, query string, ts time.Time) (model.Value, api.Warnings, error) { 198 | ret := _m.Called(ctx, query, ts) 199 | 200 | var r0 model.Value 201 | if rf, ok := ret.Get(0).(func(context.Context, string, time.Time) model.Value); ok { 202 | r0 = rf(ctx, query, ts) 203 | } else { 204 | if ret.Get(0) != nil { 205 | r0 = ret.Get(0).(model.Value) 206 | } 207 | } 208 | 209 | var r1 api.Warnings 210 | if rf, ok := ret.Get(1).(func(context.Context, string, time.Time) api.Warnings); ok { 211 | r1 = rf(ctx, query, ts) 212 | } else { 213 | if ret.Get(1) != nil { 214 | r1 = ret.Get(1).(api.Warnings) 215 | } 216 | } 217 | 218 | var r2 error 219 | if rf, ok := ret.Get(2).(func(context.Context, string, time.Time) error); ok { 220 | r2 = rf(ctx, query, ts) 221 | } else { 222 | r2 = ret.Error(2) 223 | } 224 | 225 | return r0, r1, r2 226 | } 227 | 228 | // QueryRange provides a mock function with given fields: ctx, query, r 229 | func (_m *API) QueryRange(ctx context.Context, query string, r v1.Range) (model.Value, api.Warnings, error) { 230 | ret := _m.Called(ctx, query, r) 231 | 232 | var r0 model.Value 233 | if rf, ok := ret.Get(0).(func(context.Context, string, v1.Range) model.Value); ok { 234 | r0 = rf(ctx, query, r) 235 | } else { 236 | if ret.Get(0) != nil { 237 | r0 = ret.Get(0).(model.Value) 238 | } 239 | } 240 | 241 | var r1 api.Warnings 242 | if rf, ok := ret.Get(1).(func(context.Context, string, v1.Range) api.Warnings); ok { 243 | r1 = rf(ctx, query, r) 244 | } else { 245 | if ret.Get(1) != nil { 246 | r1 = ret.Get(1).(api.Warnings) 247 | } 248 | } 249 | 250 | var r2 error 251 | if rf, ok := ret.Get(2).(func(context.Context, string, v1.Range) error); ok { 252 | r2 = rf(ctx, query, r) 253 | } else { 254 | r2 = ret.Error(2) 255 | } 256 | 257 | return r0, r1, r2 258 | } 259 | 260 | // Rules provides a mock function with given fields: ctx 261 | func (_m *API) Rules(ctx context.Context) (v1.RulesResult, error) { 262 | ret := _m.Called(ctx) 263 | 264 | var r0 v1.RulesResult 265 | if rf, ok := ret.Get(0).(func(context.Context) v1.RulesResult); ok { 266 | r0 = rf(ctx) 267 | } else { 268 | r0 = ret.Get(0).(v1.RulesResult) 269 | } 270 | 271 | var r1 error 272 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 273 | r1 = rf(ctx) 274 | } else { 275 | r1 = ret.Error(1) 276 | } 277 | 278 | return r0, r1 279 | } 280 | 281 | // Series provides a mock function with given fields: ctx, matches, startTime, endTime 282 | func (_m *API) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, api.Warnings, error) { 283 | ret := _m.Called(ctx, matches, startTime, endTime) 284 | 285 | var r0 []model.LabelSet 286 | if rf, ok := ret.Get(0).(func(context.Context, []string, time.Time, time.Time) []model.LabelSet); ok { 287 | r0 = rf(ctx, matches, startTime, endTime) 288 | } else { 289 | if ret.Get(0) != nil { 290 | r0 = ret.Get(0).([]model.LabelSet) 291 | } 292 | } 293 | 294 | var r1 api.Warnings 295 | if rf, ok := ret.Get(1).(func(context.Context, []string, time.Time, time.Time) api.Warnings); ok { 296 | r1 = rf(ctx, matches, startTime, endTime) 297 | } else { 298 | if ret.Get(1) != nil { 299 | r1 = ret.Get(1).(api.Warnings) 300 | } 301 | } 302 | 303 | var r2 error 304 | if rf, ok := ret.Get(2).(func(context.Context, []string, time.Time, time.Time) error); ok { 305 | r2 = rf(ctx, matches, startTime, endTime) 306 | } else { 307 | r2 = ret.Error(2) 308 | } 309 | 310 | return r0, r1, r2 311 | } 312 | 313 | // Snapshot provides a mock function with given fields: ctx, skipHead 314 | func (_m *API) Snapshot(ctx context.Context, skipHead bool) (v1.SnapshotResult, error) { 315 | ret := _m.Called(ctx, skipHead) 316 | 317 | var r0 v1.SnapshotResult 318 | if rf, ok := ret.Get(0).(func(context.Context, bool) v1.SnapshotResult); ok { 319 | r0 = rf(ctx, skipHead) 320 | } else { 321 | r0 = ret.Get(0).(v1.SnapshotResult) 322 | } 323 | 324 | var r1 error 325 | if rf, ok := ret.Get(1).(func(context.Context, bool) error); ok { 326 | r1 = rf(ctx, skipHead) 327 | } else { 328 | r1 = ret.Error(1) 329 | } 330 | 331 | return r0, r1 332 | } 333 | 334 | // Targets provides a mock function with given fields: ctx 335 | func (_m *API) Targets(ctx context.Context) (v1.TargetsResult, error) { 336 | ret := _m.Called(ctx) 337 | 338 | var r0 v1.TargetsResult 339 | if rf, ok := ret.Get(0).(func(context.Context) v1.TargetsResult); ok { 340 | r0 = rf(ctx) 341 | } else { 342 | r0 = ret.Get(0).(v1.TargetsResult) 343 | } 344 | 345 | var r1 error 346 | if rf, ok := ret.Get(1).(func(context.Context) error); ok { 347 | r1 = rf(ctx) 348 | } else { 349 | r1 = ret.Error(1) 350 | } 351 | 352 | return r0, r1 353 | } 354 | 355 | // TargetsMetadata provides a mock function with given fields: ctx, matchTarget, metric, limit 356 | func (_m *API) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]v1.MetricMetadata, error) { 357 | ret := _m.Called(ctx, matchTarget, metric, limit) 358 | 359 | var r0 []v1.MetricMetadata 360 | if rf, ok := ret.Get(0).(func(context.Context, string, string, string) []v1.MetricMetadata); ok { 361 | r0 = rf(ctx, matchTarget, metric, limit) 362 | } else { 363 | if ret.Get(0) != nil { 364 | r0 = ret.Get(0).([]v1.MetricMetadata) 365 | } 366 | } 367 | 368 | var r1 error 369 | if rf, ok := ret.Get(1).(func(context.Context, string, string, string) error); ok { 370 | r1 = rf(ctx, matchTarget, metric, limit) 371 | } else { 372 | r1 = ret.Error(1) 373 | } 374 | 375 | return r0, r1 376 | } 377 | -------------------------------------------------------------------------------- /mocks/service/output/Output.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package slo 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | 7 | import sli "github.com/spotahome/service-level-operator/pkg/service/sli" 8 | import v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 9 | 10 | // Output is an autogenerated mock type for the Output type 11 | type Output struct { 12 | mock.Mock 13 | } 14 | 15 | // Create provides a mock function with given fields: serviceLevel, _a1, result 16 | func (_m *Output) Create(serviceLevel *v1alpha1.ServiceLevel, _a1 *v1alpha1.SLO, result *sli.Result) error { 17 | ret := _m.Called(serviceLevel, _a1, result) 18 | 19 | var r0 error 20 | if rf, ok := ret.Get(0).(func(*v1alpha1.ServiceLevel, *v1alpha1.SLO, *sli.Result) error); ok { 21 | r0 = rf(serviceLevel, _a1, result) 22 | } else { 23 | r0 = ret.Error(0) 24 | } 25 | 26 | return r0 27 | } 28 | -------------------------------------------------------------------------------- /mocks/service/sli/Retriever.go: -------------------------------------------------------------------------------- 1 | // Code generated by mockery v1.0.0. DO NOT EDIT. 2 | 3 | package sli 4 | 5 | import mock "github.com/stretchr/testify/mock" 6 | import sli "github.com/spotahome/service-level-operator/pkg/service/sli" 7 | import v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 8 | 9 | // Retriever is an autogenerated mock type for the Retriever type 10 | type Retriever struct { 11 | mock.Mock 12 | } 13 | 14 | // Retrieve provides a mock function with given fields: _a0 15 | func (_m *Retriever) Retrieve(_a0 *v1alpha1.SLI) (sli.Result, error) { 16 | ret := _m.Called(_a0) 17 | 18 | var r0 sli.Result 19 | if rf, ok := ret.Get(0).(func(*v1alpha1.SLI) sli.Result); ok { 20 | r0 = rf(_a0) 21 | } else { 22 | r0 = ret.Get(0).(sli.Result) 23 | } 24 | 25 | var r1 error 26 | if rf, ok := ret.Get(1).(func(*v1alpha1.SLI) error); ok { 27 | r1 = rf(_a0) 28 | } else { 29 | r1 = ret.Error(1) 30 | } 31 | 32 | return r0, r1 33 | } 34 | -------------------------------------------------------------------------------- /mocks/thirdparty.go: -------------------------------------------------------------------------------- 1 | package mocks 2 | 3 | import ( 4 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 5 | ) 6 | 7 | // Third party resources to create mocks from their interfaces. 8 | 9 | // API is the interface promv1.API. 10 | type API interface{ promv1.API } 11 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/register.go: -------------------------------------------------------------------------------- 1 | package monitoring 2 | 3 | const ( 4 | // GroupName is the name of the API group. 5 | GroupName = "monitoring.spotahome.com" 6 | ) 7 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | // +k8s:deepcopy-gen=package 2 | // +k8s:openapi-gen=true 3 | 4 | // Package v1alpha1 is the v1alpha1 version of the API. 5 | // +groupName=monitoring.spotahome.com 6 | package v1alpha1 7 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/helpers.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import "fmt" 4 | 5 | // Validate validates and sets defaults on the ServiceLevel 6 | // Kubernetes resource object. 7 | func (s *ServiceLevel) Validate() error { 8 | 9 | if len(s.Spec.ServiceLevelObjectives) == 0 { 10 | return fmt.Errorf("the number of SLOs on a service level must be more than 0") 11 | } 12 | 13 | // Check if there is an input. 14 | for _, slo := range s.Spec.ServiceLevelObjectives { 15 | err := s.validateSLO(&slo) 16 | if err != nil { 17 | return err 18 | } 19 | } 20 | 21 | return nil 22 | } 23 | 24 | func (s *ServiceLevel) validateSLO(slo *SLO) error { 25 | if slo.Name == "" { 26 | return fmt.Errorf("a SLO must have a name") 27 | } 28 | 29 | if slo.AvailabilityObjectivePercent == 0 { 30 | return fmt.Errorf("the %s SLO must have a availability objective percent", slo.Name) 31 | } 32 | 33 | // Check inputs. 34 | if slo.ServiceLevelIndicator.Prometheus == nil { 35 | return fmt.Errorf("the %s SLO must have at least one input source", slo.Name) 36 | } 37 | 38 | // Check outputs. 39 | if slo.Output.Prometheus == nil { 40 | return fmt.Errorf("the %s SLO must have at least one output source", slo.Name) 41 | } 42 | 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/helpers_test.go: -------------------------------------------------------------------------------- 1 | package v1alpha1_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 8 | 9 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 10 | ) 11 | 12 | func TestServiceLevelValidation(t *testing.T) { 13 | // Setup the different combinations of service level to validate. 14 | goodSL := &monitoringv1alpha1.ServiceLevel{ 15 | ObjectMeta: metav1.ObjectMeta{ 16 | Name: "fake-service0", 17 | }, 18 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 19 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 20 | { 21 | Name: "fake_slo0", 22 | Description: "fake slo 0.", 23 | AvailabilityObjectivePercent: 99.99, 24 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 25 | SLISource: monitoringv1alpha1.SLISource{ 26 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 27 | Address: "http://fake:9090", 28 | TotalQuery: `slo0_total`, 29 | ErrorQuery: `slo0_error`, 30 | }, 31 | }, 32 | }, 33 | Output: monitoringv1alpha1.Output{ 34 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 35 | }, 36 | }, 37 | }, 38 | }, 39 | } 40 | slWithoutSLO := goodSL.DeepCopy() 41 | slWithoutSLO.Spec.ServiceLevelObjectives = []monitoringv1alpha1.SLO{} 42 | slSLOWithoutName := goodSL.DeepCopy() 43 | slSLOWithoutName.Spec.ServiceLevelObjectives[0].Name = "" 44 | slSLOWithoutObjective := goodSL.DeepCopy() 45 | slSLOWithoutObjective.Spec.ServiceLevelObjectives[0].AvailabilityObjectivePercent = 0 46 | slSLOWithoutSLI := goodSL.DeepCopy() 47 | slSLOWithoutSLI.Spec.ServiceLevelObjectives[0].ServiceLevelIndicator.Prometheus = nil 48 | slSLOWithoutOutput := goodSL.DeepCopy() 49 | slSLOWithoutOutput.Spec.ServiceLevelObjectives[0].Output.Prometheus = nil 50 | 51 | tests := []struct { 52 | name string 53 | serviceLevel *monitoringv1alpha1.ServiceLevel 54 | expErr bool 55 | }{ 56 | { 57 | name: "A valid ServiceLevel should be valid.", 58 | serviceLevel: goodSL, 59 | expErr: false, 60 | }, 61 | { 62 | name: "A ServiceLevel without SLOs houldn't be valid.", 63 | serviceLevel: slWithoutSLO, 64 | expErr: true, 65 | }, 66 | { 67 | name: "A ServiceLevel with an SLO without name shouldn't be valid.", 68 | serviceLevel: slSLOWithoutName, 69 | expErr: true, 70 | }, 71 | { 72 | name: "A ServiceLevel with an SLO without objective shouldn't be valid.", 73 | serviceLevel: slSLOWithoutObjective, 74 | expErr: true, 75 | }, 76 | { 77 | name: "A ServiceLevel with an SLO without SLI shouldn't be valid.", 78 | serviceLevel: slSLOWithoutSLI, 79 | expErr: true, 80 | }, 81 | { 82 | name: "A ServiceLevel with an SLO without output shouldn't be valid.", 83 | serviceLevel: slSLOWithoutOutput, 84 | expErr: true, 85 | }, 86 | } 87 | 88 | for _, test := range tests { 89 | t.Run(test.name, func(t *testing.T) { 90 | assert := assert.New(t) 91 | 92 | err := test.serviceLevel.Validate() 93 | 94 | if test.expErr { 95 | assert.Error(err) 96 | } else { 97 | assert.NoError(err) 98 | } 99 | }) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/register.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" 5 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 6 | "k8s.io/apimachinery/pkg/runtime" 7 | "k8s.io/apimachinery/pkg/runtime/schema" 8 | 9 | "github.com/spotahome/service-level-operator/pkg/apis/monitoring" 10 | ) 11 | 12 | const ( 13 | version = "v1alpha1" 14 | ) 15 | 16 | // ServiceLevel constants 17 | const ( 18 | ServiceLevelKind = "ServiceLevel" 19 | ServiceLevelName = "servicelevel" 20 | ServiceLevelNamePlural = "servicelevels" 21 | ServiceLevelScope = apiextensionsv1beta1.NamespaceScoped 22 | ) 23 | 24 | // SchemeGroupVersion is group version used to register these objects 25 | var SchemeGroupVersion = schema.GroupVersion{Group: monitoring.GroupName, Version: version} 26 | 27 | // Kind takes an unqualified kind and returns back a Group qualified GroupKind 28 | func Kind(kind string) schema.GroupKind { 29 | return VersionKind(kind).GroupKind() 30 | } 31 | 32 | // VersionKind takes an unqualified kind and returns back a Group qualified GroupVersionKind 33 | func VersionKind(kind string) schema.GroupVersionKind { 34 | return SchemeGroupVersion.WithKind(kind) 35 | } 36 | 37 | // Resource takes an unqualified resource and returns a Group qualified GroupResource 38 | func Resource(resource string) schema.GroupResource { 39 | return SchemeGroupVersion.WithResource(resource).GroupResource() 40 | } 41 | 42 | var ( 43 | SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) 44 | AddToScheme = SchemeBuilder.AddToScheme 45 | ) 46 | 47 | // Adds the list of known types to Scheme. 48 | func addKnownTypes(scheme *runtime.Scheme) error { 49 | scheme.AddKnownTypes(SchemeGroupVersion, 50 | &ServiceLevel{}, 51 | &ServiceLevelList{}, 52 | ) 53 | metav1.AddToGroupVersion(scheme, SchemeGroupVersion) 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/types.go: -------------------------------------------------------------------------------- 1 | package v1alpha1 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | ) 6 | 7 | // +genclient 8 | // +k8s:openapi-gen=true 9 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 10 | 11 | // ServiceLevel represents a service level policy to measure the service level 12 | // of an application. 13 | type ServiceLevel struct { 14 | metav1.TypeMeta `json:",inline"` 15 | // Standard object's metadata. 16 | // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata 17 | // +optional 18 | metav1.ObjectMeta `json:"metadata,omitempty"` 19 | 20 | // Specification of the ddesired behaviour of the pod terminator. 21 | // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status 22 | // +optional 23 | Spec ServiceLevelSpec `json:"spec,omitempty"` 24 | } 25 | 26 | // ServiceLevelSpec is the spec for a ServiceLevel resource. 27 | type ServiceLevelSpec struct { 28 | // ServiceLevelObjectives is the list of SLOs of a service/app. 29 | // +optional 30 | ServiceLevelObjectives []SLO `json:"serviceLevelObjectives,omitempty"` 31 | } 32 | 33 | // SLO represents a SLO. 34 | type SLO struct { 35 | // Name of the SLO, must be made of [a-zA-z0-9] and '_'(underscore) characters. 36 | Name string `json:"name"` 37 | // Description is a description of the SLO. 38 | // +optional 39 | Description string `json:"description,omitempty"` 40 | // Disable will disable the SLO. 41 | Disable bool `json:"disable,omitempty"` 42 | // AvailabilityObjectivePercent is the percentage of availability target for the SLO. 43 | AvailabilityObjectivePercent float64 `json:"availabilityObjectivePercent"` 44 | // ServiceLevelIndicator is the SLI associated with the SLO. 45 | ServiceLevelIndicator SLI `json:"serviceLevelIndicator"` 46 | // Output is the output backedn of the SLO. 47 | Output Output `json:"output"` 48 | } 49 | 50 | // SLI is the SLI to get for the SLO. 51 | type SLI struct { 52 | SLISource `json:",inline"` 53 | } 54 | 55 | // SLISource is where the SLI will get from. 56 | type SLISource struct { 57 | // Prometheus is the prometheus SLI source. 58 | // +optional 59 | Prometheus *PrometheusSLISource `json:"prometheus,omitempty"` 60 | } 61 | 62 | // PrometheusSLISource is the source to get SLIs from a Prometheus backend. 63 | type PrometheusSLISource struct { 64 | // Address is the address of the Prometheus. 65 | Address string `json:"address"` 66 | // TotalQuery is the query that gets the total that will be the base to get the unavailability 67 | // of the SLO based on the errorQuery (errorQuery / totalQuery). 68 | TotalQuery string `json:"totalQuery"` 69 | // ErrorQuery is the query that gets the total errors that then will be divided against the total. 70 | ErrorQuery string `json:"errorQuery"` 71 | } 72 | 73 | // Output is how the SLO will expose the generated SLO. 74 | type Output struct { 75 | //Prometheus is the prometheus format for the SLO output. 76 | // +optional 77 | Prometheus *PrometheusOutputSource `json:"prometheus,omitempty"` 78 | } 79 | 80 | // PrometheusOutputSource is the source of the output in prometheus format. 81 | type PrometheusOutputSource struct { 82 | // Labels are the labels that will be set to the output metrics of this SLO. 83 | // +optional 84 | Labels map[string]string `json:"labels,omitempty"` 85 | } 86 | 87 | // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object 88 | 89 | // ServiceLevelList is a list of ServiceLevel resources 90 | type ServiceLevelList struct { 91 | metav1.TypeMeta `json:",inline"` 92 | metav1.ListMeta `json:"metadata"` 93 | 94 | Items []ServiceLevel `json:"items"` 95 | } 96 | -------------------------------------------------------------------------------- /pkg/apis/monitoring/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /* 4 | Copyright The Kubernetes Authors. 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); 7 | you may not use this file except in compliance with the License. 8 | You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | */ 18 | 19 | // Code generated by deepcopy-gen. DO NOT EDIT. 20 | 21 | package v1alpha1 22 | 23 | import ( 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | ) 26 | 27 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 28 | func (in *Output) DeepCopyInto(out *Output) { 29 | *out = *in 30 | if in.Prometheus != nil { 31 | in, out := &in.Prometheus, &out.Prometheus 32 | *out = new(PrometheusOutputSource) 33 | (*in).DeepCopyInto(*out) 34 | } 35 | return 36 | } 37 | 38 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Output. 39 | func (in *Output) DeepCopy() *Output { 40 | if in == nil { 41 | return nil 42 | } 43 | out := new(Output) 44 | in.DeepCopyInto(out) 45 | return out 46 | } 47 | 48 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 49 | func (in *PrometheusOutputSource) DeepCopyInto(out *PrometheusOutputSource) { 50 | *out = *in 51 | if in.Labels != nil { 52 | in, out := &in.Labels, &out.Labels 53 | *out = make(map[string]string, len(*in)) 54 | for key, val := range *in { 55 | (*out)[key] = val 56 | } 57 | } 58 | return 59 | } 60 | 61 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusOutputSource. 62 | func (in *PrometheusOutputSource) DeepCopy() *PrometheusOutputSource { 63 | if in == nil { 64 | return nil 65 | } 66 | out := new(PrometheusOutputSource) 67 | in.DeepCopyInto(out) 68 | return out 69 | } 70 | 71 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 72 | func (in *PrometheusSLISource) DeepCopyInto(out *PrometheusSLISource) { 73 | *out = *in 74 | return 75 | } 76 | 77 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusSLISource. 78 | func (in *PrometheusSLISource) DeepCopy() *PrometheusSLISource { 79 | if in == nil { 80 | return nil 81 | } 82 | out := new(PrometheusSLISource) 83 | in.DeepCopyInto(out) 84 | return out 85 | } 86 | 87 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 88 | func (in *SLI) DeepCopyInto(out *SLI) { 89 | *out = *in 90 | in.SLISource.DeepCopyInto(&out.SLISource) 91 | return 92 | } 93 | 94 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SLI. 95 | func (in *SLI) DeepCopy() *SLI { 96 | if in == nil { 97 | return nil 98 | } 99 | out := new(SLI) 100 | in.DeepCopyInto(out) 101 | return out 102 | } 103 | 104 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 105 | func (in *SLISource) DeepCopyInto(out *SLISource) { 106 | *out = *in 107 | if in.Prometheus != nil { 108 | in, out := &in.Prometheus, &out.Prometheus 109 | *out = new(PrometheusSLISource) 110 | **out = **in 111 | } 112 | return 113 | } 114 | 115 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SLISource. 116 | func (in *SLISource) DeepCopy() *SLISource { 117 | if in == nil { 118 | return nil 119 | } 120 | out := new(SLISource) 121 | in.DeepCopyInto(out) 122 | return out 123 | } 124 | 125 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 126 | func (in *SLO) DeepCopyInto(out *SLO) { 127 | *out = *in 128 | in.ServiceLevelIndicator.DeepCopyInto(&out.ServiceLevelIndicator) 129 | in.Output.DeepCopyInto(&out.Output) 130 | return 131 | } 132 | 133 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SLO. 134 | func (in *SLO) DeepCopy() *SLO { 135 | if in == nil { 136 | return nil 137 | } 138 | out := new(SLO) 139 | in.DeepCopyInto(out) 140 | return out 141 | } 142 | 143 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 144 | func (in *ServiceLevel) DeepCopyInto(out *ServiceLevel) { 145 | *out = *in 146 | out.TypeMeta = in.TypeMeta 147 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 148 | in.Spec.DeepCopyInto(&out.Spec) 149 | return 150 | } 151 | 152 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceLevel. 153 | func (in *ServiceLevel) DeepCopy() *ServiceLevel { 154 | if in == nil { 155 | return nil 156 | } 157 | out := new(ServiceLevel) 158 | in.DeepCopyInto(out) 159 | return out 160 | } 161 | 162 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 163 | func (in *ServiceLevel) DeepCopyObject() runtime.Object { 164 | if c := in.DeepCopy(); c != nil { 165 | return c 166 | } 167 | return nil 168 | } 169 | 170 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 171 | func (in *ServiceLevelList) DeepCopyInto(out *ServiceLevelList) { 172 | *out = *in 173 | out.TypeMeta = in.TypeMeta 174 | out.ListMeta = in.ListMeta 175 | if in.Items != nil { 176 | in, out := &in.Items, &out.Items 177 | *out = make([]ServiceLevel, len(*in)) 178 | for i := range *in { 179 | (*in)[i].DeepCopyInto(&(*out)[i]) 180 | } 181 | } 182 | return 183 | } 184 | 185 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceLevelList. 186 | func (in *ServiceLevelList) DeepCopy() *ServiceLevelList { 187 | if in == nil { 188 | return nil 189 | } 190 | out := new(ServiceLevelList) 191 | in.DeepCopyInto(out) 192 | return out 193 | } 194 | 195 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 196 | func (in *ServiceLevelList) DeepCopyObject() runtime.Object { 197 | if c := in.DeepCopy(); c != nil { 198 | return c 199 | } 200 | return nil 201 | } 202 | 203 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 204 | func (in *ServiceLevelSpec) DeepCopyInto(out *ServiceLevelSpec) { 205 | *out = *in 206 | if in.ServiceLevelObjectives != nil { 207 | in, out := &in.ServiceLevelObjectives, &out.ServiceLevelObjectives 208 | *out = make([]SLO, len(*in)) 209 | for i := range *in { 210 | (*in)[i].DeepCopyInto(&(*out)[i]) 211 | } 212 | } 213 | return 214 | } 215 | 216 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceLevelSpec. 217 | func (in *ServiceLevelSpec) DeepCopy() *ServiceLevelSpec { 218 | if in == nil { 219 | return nil 220 | } 221 | out := new(ServiceLevelSpec) 222 | in.DeepCopyInto(out) 223 | return out 224 | } 225 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/clientset.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package versioned 20 | 21 | import ( 22 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1" 23 | discovery "k8s.io/client-go/discovery" 24 | rest "k8s.io/client-go/rest" 25 | flowcontrol "k8s.io/client-go/util/flowcontrol" 26 | ) 27 | 28 | type Interface interface { 29 | Discovery() discovery.DiscoveryInterface 30 | MonitoringV1alpha1() monitoringv1alpha1.MonitoringV1alpha1Interface 31 | // Deprecated: please explicitly pick a version if possible. 32 | Monitoring() monitoringv1alpha1.MonitoringV1alpha1Interface 33 | } 34 | 35 | // Clientset contains the clients for groups. Each group has exactly one 36 | // version included in a Clientset. 37 | type Clientset struct { 38 | *discovery.DiscoveryClient 39 | monitoringV1alpha1 *monitoringv1alpha1.MonitoringV1alpha1Client 40 | } 41 | 42 | // MonitoringV1alpha1 retrieves the MonitoringV1alpha1Client 43 | func (c *Clientset) MonitoringV1alpha1() monitoringv1alpha1.MonitoringV1alpha1Interface { 44 | return c.monitoringV1alpha1 45 | } 46 | 47 | // Deprecated: Monitoring retrieves the default version of MonitoringClient. 48 | // Please explicitly pick a version. 49 | func (c *Clientset) Monitoring() monitoringv1alpha1.MonitoringV1alpha1Interface { 50 | return c.monitoringV1alpha1 51 | } 52 | 53 | // Discovery retrieves the DiscoveryClient 54 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 55 | if c == nil { 56 | return nil 57 | } 58 | return c.DiscoveryClient 59 | } 60 | 61 | // NewForConfig creates a new Clientset for the given config. 62 | func NewForConfig(c *rest.Config) (*Clientset, error) { 63 | configShallowCopy := *c 64 | if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { 65 | configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) 66 | } 67 | var cs Clientset 68 | var err error 69 | cs.monitoringV1alpha1, err = monitoringv1alpha1.NewForConfig(&configShallowCopy) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(&configShallowCopy) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return &cs, nil 79 | } 80 | 81 | // NewForConfigOrDie creates a new Clientset for the given config and 82 | // panics if there is an error in the config. 83 | func NewForConfigOrDie(c *rest.Config) *Clientset { 84 | var cs Clientset 85 | cs.monitoringV1alpha1 = monitoringv1alpha1.NewForConfigOrDie(c) 86 | 87 | cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) 88 | return &cs 89 | } 90 | 91 | // New creates a new Clientset for the given RESTClient. 92 | func New(c rest.Interface) *Clientset { 93 | var cs Clientset 94 | cs.monitoringV1alpha1 = monitoringv1alpha1.New(c) 95 | 96 | cs.DiscoveryClient = discovery.NewDiscoveryClient(c) 97 | return &cs 98 | } 99 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated clientset. 20 | package versioned 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/fake/clientset_generated.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | clientset "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 23 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1" 24 | fakemonitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/watch" 27 | "k8s.io/client-go/discovery" 28 | fakediscovery "k8s.io/client-go/discovery/fake" 29 | "k8s.io/client-go/testing" 30 | ) 31 | 32 | // NewSimpleClientset returns a clientset that will respond with the provided objects. 33 | // It's backed by a very simple object tracker that processes creates, updates and deletions as-is, 34 | // without applying any validations and/or defaults. It shouldn't be considered a replacement 35 | // for a real clientset and is mostly useful in simple unit tests. 36 | func NewSimpleClientset(objects ...runtime.Object) *Clientset { 37 | o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) 38 | for _, obj := range objects { 39 | if err := o.Add(obj); err != nil { 40 | panic(err) 41 | } 42 | } 43 | 44 | cs := &Clientset{} 45 | cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} 46 | cs.AddReactor("*", "*", testing.ObjectReaction(o)) 47 | cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { 48 | gvr := action.GetResource() 49 | ns := action.GetNamespace() 50 | watch, err := o.Watch(gvr, ns) 51 | if err != nil { 52 | return false, nil, err 53 | } 54 | return true, watch, nil 55 | }) 56 | 57 | return cs 58 | } 59 | 60 | // Clientset implements clientset.Interface. Meant to be embedded into a 61 | // struct to get a default implementation. This makes faking out just the method 62 | // you want to test easier. 63 | type Clientset struct { 64 | testing.Fake 65 | discovery *fakediscovery.FakeDiscovery 66 | } 67 | 68 | func (c *Clientset) Discovery() discovery.DiscoveryInterface { 69 | return c.discovery 70 | } 71 | 72 | var _ clientset.Interface = &Clientset{} 73 | 74 | // MonitoringV1alpha1 retrieves the MonitoringV1alpha1Client 75 | func (c *Clientset) MonitoringV1alpha1() monitoringv1alpha1.MonitoringV1alpha1Interface { 76 | return &fakemonitoringv1alpha1.FakeMonitoringV1alpha1{Fake: &c.Fake} 77 | } 78 | 79 | // Monitoring retrieves the MonitoringV1alpha1Client 80 | func (c *Clientset) Monitoring() monitoringv1alpha1.MonitoringV1alpha1Interface { 81 | return &fakemonitoringv1alpha1.FakeMonitoringV1alpha1{Fake: &c.Fake} 82 | } 83 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated fake clientset. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/fake/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var scheme = runtime.NewScheme() 31 | var codecs = serializer.NewCodecFactory(scheme) 32 | var parameterCodec = runtime.NewParameterCodec(scheme) 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | monitoringv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/scheme/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package contains the scheme of the automatically generated clientset. 20 | package scheme 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/scheme/register.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package scheme 20 | 21 | import ( 22 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | runtime "k8s.io/apimachinery/pkg/runtime" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 27 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 28 | ) 29 | 30 | var Scheme = runtime.NewScheme() 31 | var Codecs = serializer.NewCodecFactory(Scheme) 32 | var ParameterCodec = runtime.NewParameterCodec(Scheme) 33 | var localSchemeBuilder = runtime.SchemeBuilder{ 34 | monitoringv1alpha1.AddToScheme, 35 | } 36 | 37 | // AddToScheme adds all types of this clientset into the given scheme. This allows composition 38 | // of clientsets, like in: 39 | // 40 | // import ( 41 | // "k8s.io/client-go/kubernetes" 42 | // clientsetscheme "k8s.io/client-go/kubernetes/scheme" 43 | // aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" 44 | // ) 45 | // 46 | // kclientset, _ := kubernetes.NewForConfig(c) 47 | // _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) 48 | // 49 | // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types 50 | // correctly. 51 | var AddToScheme = localSchemeBuilder.AddToScheme 52 | 53 | func init() { 54 | v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 55 | utilruntime.Must(AddToScheme(Scheme)) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // This package has the automatically generated typed clients. 20 | package v1alpha1 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | // Package fake has the automatically generated clients. 20 | package fake 21 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake/fake_monitoring_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1" 23 | rest "k8s.io/client-go/rest" 24 | testing "k8s.io/client-go/testing" 25 | ) 26 | 27 | type FakeMonitoringV1alpha1 struct { 28 | *testing.Fake 29 | } 30 | 31 | func (c *FakeMonitoringV1alpha1) ServiceLevels(namespace string) v1alpha1.ServiceLevelInterface { 32 | return &FakeServiceLevels{c, namespace} 33 | } 34 | 35 | // RESTClient returns a RESTClient that is used to communicate 36 | // with API server by this client implementation. 37 | func (c *FakeMonitoringV1alpha1) RESTClient() rest.Interface { 38 | var ret *rest.RESTClient 39 | return ret 40 | } 41 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/fake/fake_servicelevel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package fake 20 | 21 | import ( 22 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 24 | labels "k8s.io/apimachinery/pkg/labels" 25 | schema "k8s.io/apimachinery/pkg/runtime/schema" 26 | types "k8s.io/apimachinery/pkg/types" 27 | watch "k8s.io/apimachinery/pkg/watch" 28 | testing "k8s.io/client-go/testing" 29 | ) 30 | 31 | // FakeServiceLevels implements ServiceLevelInterface 32 | type FakeServiceLevels struct { 33 | Fake *FakeMonitoringV1alpha1 34 | ns string 35 | } 36 | 37 | var servicelevelsResource = schema.GroupVersionResource{Group: "monitoring.spotahome.com", Version: "v1alpha1", Resource: "servicelevels"} 38 | 39 | var servicelevelsKind = schema.GroupVersionKind{Group: "monitoring.spotahome.com", Version: "v1alpha1", Kind: "ServiceLevel"} 40 | 41 | // Get takes name of the serviceLevel, and returns the corresponding serviceLevel object, and an error if there is any. 42 | func (c *FakeServiceLevels) Get(name string, options v1.GetOptions) (result *v1alpha1.ServiceLevel, err error) { 43 | obj, err := c.Fake. 44 | Invokes(testing.NewGetAction(servicelevelsResource, c.ns, name), &v1alpha1.ServiceLevel{}) 45 | 46 | if obj == nil { 47 | return nil, err 48 | } 49 | return obj.(*v1alpha1.ServiceLevel), err 50 | } 51 | 52 | // List takes label and field selectors, and returns the list of ServiceLevels that match those selectors. 53 | func (c *FakeServiceLevels) List(opts v1.ListOptions) (result *v1alpha1.ServiceLevelList, err error) { 54 | obj, err := c.Fake. 55 | Invokes(testing.NewListAction(servicelevelsResource, servicelevelsKind, c.ns, opts), &v1alpha1.ServiceLevelList{}) 56 | 57 | if obj == nil { 58 | return nil, err 59 | } 60 | 61 | label, _, _ := testing.ExtractFromListOptions(opts) 62 | if label == nil { 63 | label = labels.Everything() 64 | } 65 | list := &v1alpha1.ServiceLevelList{ListMeta: obj.(*v1alpha1.ServiceLevelList).ListMeta} 66 | for _, item := range obj.(*v1alpha1.ServiceLevelList).Items { 67 | if label.Matches(labels.Set(item.Labels)) { 68 | list.Items = append(list.Items, item) 69 | } 70 | } 71 | return list, err 72 | } 73 | 74 | // Watch returns a watch.Interface that watches the requested serviceLevels. 75 | func (c *FakeServiceLevels) Watch(opts v1.ListOptions) (watch.Interface, error) { 76 | return c.Fake. 77 | InvokesWatch(testing.NewWatchAction(servicelevelsResource, c.ns, opts)) 78 | 79 | } 80 | 81 | // Create takes the representation of a serviceLevel and creates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 82 | func (c *FakeServiceLevels) Create(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 83 | obj, err := c.Fake. 84 | Invokes(testing.NewCreateAction(servicelevelsResource, c.ns, serviceLevel), &v1alpha1.ServiceLevel{}) 85 | 86 | if obj == nil { 87 | return nil, err 88 | } 89 | return obj.(*v1alpha1.ServiceLevel), err 90 | } 91 | 92 | // Update takes the representation of a serviceLevel and updates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 93 | func (c *FakeServiceLevels) Update(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 94 | obj, err := c.Fake. 95 | Invokes(testing.NewUpdateAction(servicelevelsResource, c.ns, serviceLevel), &v1alpha1.ServiceLevel{}) 96 | 97 | if obj == nil { 98 | return nil, err 99 | } 100 | return obj.(*v1alpha1.ServiceLevel), err 101 | } 102 | 103 | // Delete takes name of the serviceLevel and deletes it. Returns an error if one occurs. 104 | func (c *FakeServiceLevels) Delete(name string, options *v1.DeleteOptions) error { 105 | _, err := c.Fake. 106 | Invokes(testing.NewDeleteAction(servicelevelsResource, c.ns, name), &v1alpha1.ServiceLevel{}) 107 | 108 | return err 109 | } 110 | 111 | // DeleteCollection deletes a collection of objects. 112 | func (c *FakeServiceLevels) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { 113 | action := testing.NewDeleteCollectionAction(servicelevelsResource, c.ns, listOptions) 114 | 115 | _, err := c.Fake.Invokes(action, &v1alpha1.ServiceLevelList{}) 116 | return err 117 | } 118 | 119 | // Patch applies the patch and returns the patched serviceLevel. 120 | func (c *FakeServiceLevels) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ServiceLevel, err error) { 121 | obj, err := c.Fake. 122 | Invokes(testing.NewPatchSubresourceAction(servicelevelsResource, c.ns, name, pt, data, subresources...), &v1alpha1.ServiceLevel{}) 123 | 124 | if obj == nil { 125 | return nil, err 126 | } 127 | return obj.(*v1alpha1.ServiceLevel), err 128 | } 129 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/generated_expansion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | type ServiceLevelExpansion interface{} 22 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/monitoring_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 23 | "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/scheme" 24 | serializer "k8s.io/apimachinery/pkg/runtime/serializer" 25 | rest "k8s.io/client-go/rest" 26 | ) 27 | 28 | type MonitoringV1alpha1Interface interface { 29 | RESTClient() rest.Interface 30 | ServiceLevelsGetter 31 | } 32 | 33 | // MonitoringV1alpha1Client is used to interact with features provided by the monitoring.spotahome.com group. 34 | type MonitoringV1alpha1Client struct { 35 | restClient rest.Interface 36 | } 37 | 38 | func (c *MonitoringV1alpha1Client) ServiceLevels(namespace string) ServiceLevelInterface { 39 | return newServiceLevels(c, namespace) 40 | } 41 | 42 | // NewForConfig creates a new MonitoringV1alpha1Client for the given config. 43 | func NewForConfig(c *rest.Config) (*MonitoringV1alpha1Client, error) { 44 | config := *c 45 | if err := setConfigDefaults(&config); err != nil { 46 | return nil, err 47 | } 48 | client, err := rest.RESTClientFor(&config) 49 | if err != nil { 50 | return nil, err 51 | } 52 | return &MonitoringV1alpha1Client{client}, nil 53 | } 54 | 55 | // NewForConfigOrDie creates a new MonitoringV1alpha1Client for the given config and 56 | // panics if there is an error in the config. 57 | func NewForConfigOrDie(c *rest.Config) *MonitoringV1alpha1Client { 58 | client, err := NewForConfig(c) 59 | if err != nil { 60 | panic(err) 61 | } 62 | return client 63 | } 64 | 65 | // New creates a new MonitoringV1alpha1Client for the given RESTClient. 66 | func New(c rest.Interface) *MonitoringV1alpha1Client { 67 | return &MonitoringV1alpha1Client{c} 68 | } 69 | 70 | func setConfigDefaults(config *rest.Config) error { 71 | gv := v1alpha1.SchemeGroupVersion 72 | config.GroupVersion = &gv 73 | config.APIPath = "/apis" 74 | config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} 75 | 76 | if config.UserAgent == "" { 77 | config.UserAgent = rest.DefaultKubernetesUserAgent() 78 | } 79 | 80 | return nil 81 | } 82 | 83 | // RESTClient returns a RESTClient that is used to communicate 84 | // with API server by this client implementation. 85 | func (c *MonitoringV1alpha1Client) RESTClient() rest.Interface { 86 | if c == nil { 87 | return nil 88 | } 89 | return c.restClient 90 | } 91 | -------------------------------------------------------------------------------- /pkg/k8sautogen/client/clientset/versioned/typed/monitoring/v1alpha1/servicelevel.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Code generated by client-gen. DO NOT EDIT. 18 | 19 | package v1alpha1 20 | 21 | import ( 22 | "time" 23 | 24 | v1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 25 | scheme "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/scheme" 26 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | types "k8s.io/apimachinery/pkg/types" 28 | watch "k8s.io/apimachinery/pkg/watch" 29 | rest "k8s.io/client-go/rest" 30 | ) 31 | 32 | // ServiceLevelsGetter has a method to return a ServiceLevelInterface. 33 | // A group's client should implement this interface. 34 | type ServiceLevelsGetter interface { 35 | ServiceLevels(namespace string) ServiceLevelInterface 36 | } 37 | 38 | // ServiceLevelInterface has methods to work with ServiceLevel resources. 39 | type ServiceLevelInterface interface { 40 | Create(*v1alpha1.ServiceLevel) (*v1alpha1.ServiceLevel, error) 41 | Update(*v1alpha1.ServiceLevel) (*v1alpha1.ServiceLevel, error) 42 | Delete(name string, options *v1.DeleteOptions) error 43 | DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error 44 | Get(name string, options v1.GetOptions) (*v1alpha1.ServiceLevel, error) 45 | List(opts v1.ListOptions) (*v1alpha1.ServiceLevelList, error) 46 | Watch(opts v1.ListOptions) (watch.Interface, error) 47 | Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ServiceLevel, err error) 48 | ServiceLevelExpansion 49 | } 50 | 51 | // serviceLevels implements ServiceLevelInterface 52 | type serviceLevels struct { 53 | client rest.Interface 54 | ns string 55 | } 56 | 57 | // newServiceLevels returns a ServiceLevels 58 | func newServiceLevels(c *MonitoringV1alpha1Client, namespace string) *serviceLevels { 59 | return &serviceLevels{ 60 | client: c.RESTClient(), 61 | ns: namespace, 62 | } 63 | } 64 | 65 | // Get takes name of the serviceLevel, and returns the corresponding serviceLevel object, and an error if there is any. 66 | func (c *serviceLevels) Get(name string, options v1.GetOptions) (result *v1alpha1.ServiceLevel, err error) { 67 | result = &v1alpha1.ServiceLevel{} 68 | err = c.client.Get(). 69 | Namespace(c.ns). 70 | Resource("servicelevels"). 71 | Name(name). 72 | VersionedParams(&options, scheme.ParameterCodec). 73 | Do(). 74 | Into(result) 75 | return 76 | } 77 | 78 | // List takes label and field selectors, and returns the list of ServiceLevels that match those selectors. 79 | func (c *serviceLevels) List(opts v1.ListOptions) (result *v1alpha1.ServiceLevelList, err error) { 80 | var timeout time.Duration 81 | if opts.TimeoutSeconds != nil { 82 | timeout = time.Duration(*opts.TimeoutSeconds) * time.Second 83 | } 84 | result = &v1alpha1.ServiceLevelList{} 85 | err = c.client.Get(). 86 | Namespace(c.ns). 87 | Resource("servicelevels"). 88 | VersionedParams(&opts, scheme.ParameterCodec). 89 | Timeout(timeout). 90 | Do(). 91 | Into(result) 92 | return 93 | } 94 | 95 | // Watch returns a watch.Interface that watches the requested serviceLevels. 96 | func (c *serviceLevels) Watch(opts v1.ListOptions) (watch.Interface, error) { 97 | var timeout time.Duration 98 | if opts.TimeoutSeconds != nil { 99 | timeout = time.Duration(*opts.TimeoutSeconds) * time.Second 100 | } 101 | opts.Watch = true 102 | return c.client.Get(). 103 | Namespace(c.ns). 104 | Resource("servicelevels"). 105 | VersionedParams(&opts, scheme.ParameterCodec). 106 | Timeout(timeout). 107 | Watch() 108 | } 109 | 110 | // Create takes the representation of a serviceLevel and creates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 111 | func (c *serviceLevels) Create(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 112 | result = &v1alpha1.ServiceLevel{} 113 | err = c.client.Post(). 114 | Namespace(c.ns). 115 | Resource("servicelevels"). 116 | Body(serviceLevel). 117 | Do(). 118 | Into(result) 119 | return 120 | } 121 | 122 | // Update takes the representation of a serviceLevel and updates it. Returns the server's representation of the serviceLevel, and an error, if there is any. 123 | func (c *serviceLevels) Update(serviceLevel *v1alpha1.ServiceLevel) (result *v1alpha1.ServiceLevel, err error) { 124 | result = &v1alpha1.ServiceLevel{} 125 | err = c.client.Put(). 126 | Namespace(c.ns). 127 | Resource("servicelevels"). 128 | Name(serviceLevel.Name). 129 | Body(serviceLevel). 130 | Do(). 131 | Into(result) 132 | return 133 | } 134 | 135 | // Delete takes name of the serviceLevel and deletes it. Returns an error if one occurs. 136 | func (c *serviceLevels) Delete(name string, options *v1.DeleteOptions) error { 137 | return c.client.Delete(). 138 | Namespace(c.ns). 139 | Resource("servicelevels"). 140 | Name(name). 141 | Body(options). 142 | Do(). 143 | Error() 144 | } 145 | 146 | // DeleteCollection deletes a collection of objects. 147 | func (c *serviceLevels) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { 148 | var timeout time.Duration 149 | if listOptions.TimeoutSeconds != nil { 150 | timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second 151 | } 152 | return c.client.Delete(). 153 | Namespace(c.ns). 154 | Resource("servicelevels"). 155 | VersionedParams(&listOptions, scheme.ParameterCodec). 156 | Timeout(timeout). 157 | Body(options). 158 | Do(). 159 | Error() 160 | } 161 | 162 | // Patch applies the patch and returns the patched serviceLevel. 163 | func (c *serviceLevels) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ServiceLevel, err error) { 164 | result = &v1alpha1.ServiceLevel{} 165 | err = c.client.Patch(pt). 166 | Namespace(c.ns). 167 | Resource("servicelevels"). 168 | SubResource(subresources...). 169 | Name(name). 170 | Body(data). 171 | Do(). 172 | Into(result) 173 | return 174 | } 175 | -------------------------------------------------------------------------------- /pkg/log/dummy.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | // Dummy is a dummy logger 4 | var Dummy = dummyLogger{} 5 | 6 | type dummyLogger struct{} 7 | 8 | func (l dummyLogger) Debug(...interface{}) {} 9 | func (l dummyLogger) Debugln(...interface{}) {} 10 | func (l dummyLogger) Debugf(string, ...interface{}) {} 11 | func (l dummyLogger) Info(...interface{}) {} 12 | func (l dummyLogger) Infoln(...interface{}) {} 13 | func (l dummyLogger) Infof(string, ...interface{}) {} 14 | func (l dummyLogger) Warn(...interface{}) {} 15 | func (l dummyLogger) Warnln(...interface{}) {} 16 | func (l dummyLogger) Warnf(string, ...interface{}) {} 17 | func (l dummyLogger) Warningf(format string, args ...interface{}) {} 18 | func (l dummyLogger) Error(...interface{}) {} 19 | func (l dummyLogger) Errorln(...interface{}) {} 20 | func (l dummyLogger) Errorf(string, ...interface{}) {} 21 | func (l dummyLogger) Fatal(...interface{}) {} 22 | func (l dummyLogger) Fatalln(...interface{}) {} 23 | func (l dummyLogger) Fatalf(string, ...interface{}) {} 24 | func (l dummyLogger) Panic(...interface{}) {} 25 | func (l dummyLogger) Panicln(...interface{}) {} 26 | func (l dummyLogger) Panicf(string, ...interface{}) {} 27 | func (l dummyLogger) With(key string, value interface{}) Logger { return l } 28 | func (l dummyLogger) WithField(key string, value interface{}) Logger { return l } 29 | func (l dummyLogger) Set(level Level) error { return nil } 30 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | "strings" 7 | 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | // Level refers to the level of logging 12 | type Level string 13 | 14 | // Logger is an interface that needs to be implemented in order to log. 15 | type Logger interface { 16 | Debug(...interface{}) 17 | Debugln(...interface{}) 18 | Debugf(string, ...interface{}) 19 | 20 | Info(...interface{}) 21 | Infoln(...interface{}) 22 | Infof(string, ...interface{}) 23 | 24 | Warn(...interface{}) 25 | Warnln(...interface{}) 26 | Warnf(string, ...interface{}) 27 | Warningf(string, ...interface{}) 28 | 29 | Error(...interface{}) 30 | Errorln(...interface{}) 31 | Errorf(string, ...interface{}) 32 | 33 | Fatal(...interface{}) 34 | Fatalln(...interface{}) 35 | Fatalf(string, ...interface{}) 36 | 37 | Panic(...interface{}) 38 | Panicln(...interface{}) 39 | Panicf(string, ...interface{}) 40 | 41 | With(key string, value interface{}) Logger 42 | WithField(key string, value interface{}) Logger 43 | Set(level Level) error 44 | } 45 | 46 | type logger struct { 47 | entry *logrus.Entry 48 | } 49 | 50 | func (l logger) Debug(args ...interface{}) { 51 | l.sourced().Debug(args...) 52 | } 53 | 54 | func (l logger) Debugln(args ...interface{}) { 55 | l.sourced().Debugln(args...) 56 | } 57 | 58 | func (l logger) Debugf(format string, args ...interface{}) { 59 | l.sourced().Debugf(format, args...) 60 | } 61 | 62 | func (l logger) Info(args ...interface{}) { 63 | l.sourced().Info(args...) 64 | } 65 | 66 | func (l logger) Infoln(args ...interface{}) { 67 | l.sourced().Infoln(args...) 68 | } 69 | 70 | func (l logger) Infof(format string, args ...interface{}) { 71 | l.sourced().Infof(format, args...) 72 | } 73 | 74 | func (l logger) Warn(args ...interface{}) { 75 | l.sourced().Warn(args...) 76 | } 77 | 78 | func (l logger) Warnln(args ...interface{}) { 79 | l.sourced().Warnln(args...) 80 | } 81 | 82 | func (l logger) Warnf(format string, args ...interface{}) { 83 | l.sourced().Warnf(format, args...) 84 | } 85 | 86 | func (l logger) Warningf(format string, args ...interface{}) { 87 | l.sourced().Warnf(format, args...) 88 | } 89 | 90 | func (l logger) Error(args ...interface{}) { 91 | l.sourced().Error(args...) 92 | } 93 | 94 | func (l logger) Errorln(args ...interface{}) { 95 | l.sourced().Errorln(args...) 96 | } 97 | 98 | func (l logger) Errorf(format string, args ...interface{}) { 99 | l.sourced().Errorf(format, args...) 100 | } 101 | 102 | func (l logger) Fatal(args ...interface{}) { 103 | l.sourced().Fatal(args...) 104 | } 105 | 106 | func (l logger) Fatalln(args ...interface{}) { 107 | l.sourced().Fatalln(args...) 108 | } 109 | 110 | func (l logger) Fatalf(format string, args ...interface{}) { 111 | l.sourced().Fatalf(format, args...) 112 | } 113 | func (l logger) Panic(args ...interface{}) { 114 | l.sourced().Panic(args...) 115 | } 116 | func (l logger) Panicln(args ...interface{}) { 117 | l.sourced().Panicln(args...) 118 | } 119 | func (l logger) Panicf(format string, args ...interface{}) { 120 | l.sourced().Panicf(format, args...) 121 | } 122 | 123 | func (l logger) With(key string, value interface{}) Logger { 124 | return &logger{l.entry.WithField(key, value)} 125 | } 126 | 127 | func (l logger) WithField(key string, value interface{}) Logger { 128 | return &logger{l.entry.WithField(key, value)} 129 | } 130 | 131 | func (l *logger) Set(level Level) error { 132 | leLev, err := logrus.ParseLevel(string(level)) 133 | if err != nil { 134 | return err 135 | } 136 | l.entry.Logger.Level = leLev 137 | return nil 138 | } 139 | 140 | func (l logger) sourced() *logrus.Entry { 141 | _, file, line, ok := runtime.Caller(3) 142 | if !ok { 143 | file = "" 144 | line = 1 145 | } else { 146 | slash := strings.LastIndex(file, "/") 147 | file = file[slash+1:] 148 | } 149 | return l.entry.WithField("src", fmt.Sprintf("%s:%d", file, line)) 150 | } 151 | 152 | var jsonBaseLogger = func() Logger { 153 | l := logrus.New() 154 | l.Formatter = &logrus.JSONFormatter{} 155 | return &logger{ 156 | entry: &logrus.Entry{ 157 | Logger: l, 158 | }, 159 | } 160 | }() 161 | 162 | var baseLogger = &logger{ 163 | entry: &logrus.Entry{ 164 | Logger: logrus.New(), 165 | }, 166 | } 167 | 168 | // Base returns the base logger 169 | func Base(json bool) Logger { 170 | if json { 171 | return jsonBaseLogger 172 | } 173 | return baseLogger 174 | } 175 | 176 | // Debug logs debug message 177 | func Debug(args ...interface{}) { 178 | baseLogger.sourced().Debug(args...) 179 | } 180 | 181 | // Debugln logs debug message 182 | func Debugln(args ...interface{}) { 183 | baseLogger.sourced().Debugln(args...) 184 | } 185 | 186 | // Debugf logs debug message 187 | func Debugf(format string, args ...interface{}) { 188 | baseLogger.sourced().Debugf(format, args...) 189 | } 190 | 191 | // Info logs info message 192 | func Info(args ...interface{}) { 193 | baseLogger.sourced().Info(args...) 194 | } 195 | 196 | // Infoln logs info message 197 | func Infoln(args ...interface{}) { 198 | baseLogger.sourced().Infoln(args...) 199 | } 200 | 201 | // Infof logs info message 202 | func Infof(format string, args ...interface{}) { 203 | baseLogger.sourced().Infof(format, args...) 204 | } 205 | 206 | // Warn logs warn message 207 | func Warn(args ...interface{}) { 208 | baseLogger.sourced().Warn(args...) 209 | } 210 | 211 | // Warnln logs warn message 212 | func Warnln(args ...interface{}) { 213 | baseLogger.sourced().Warnln(args...) 214 | } 215 | 216 | // Warnf logs warn message 217 | func Warnf(format string, args ...interface{}) { 218 | baseLogger.sourced().Warnf(format, args...) 219 | } 220 | 221 | // Error logs error message 222 | func Error(args ...interface{}) { 223 | baseLogger.sourced().Error(args...) 224 | } 225 | 226 | // Errorln logs error message 227 | func Errorln(args ...interface{}) { 228 | baseLogger.sourced().Errorln(args...) 229 | } 230 | 231 | // Errorf logs error message 232 | func Errorf(format string, args ...interface{}) { 233 | baseLogger.sourced().Errorf(format, args...) 234 | } 235 | 236 | // Fatal logs fatal message 237 | func Fatal(args ...interface{}) { 238 | baseLogger.sourced().Fatal(args...) 239 | } 240 | 241 | // Fatalln logs fatal message 242 | func Fatalln(args ...interface{}) { 243 | baseLogger.sourced().Fatalln(args...) 244 | } 245 | 246 | // Fatalf logs fatal message 247 | func Fatalf(format string, args ...interface{}) { 248 | baseLogger.sourced().Fatalf(format, args...) 249 | } 250 | 251 | // With adds a key:value to the logger 252 | func With(key string, value interface{}) Logger { 253 | return baseLogger.With(key, value) 254 | } 255 | 256 | // WithField adds a key:value to the logger 257 | func WithField(key string, value interface{}) Logger { 258 | return baseLogger.WithField(key, value) 259 | } 260 | 261 | // Set will set the logger level 262 | func Set(level Level) error { 263 | return baseLogger.Set(level) 264 | } 265 | 266 | // Panic logs panic message 267 | func Panic(args ...interface{}) { 268 | baseLogger.Panic(args...) 269 | } 270 | 271 | // Panicln logs panicln message 272 | func Panicln(args ...interface{}) { 273 | baseLogger.Panicln(args...) 274 | } 275 | 276 | // Panicf logs panicln message 277 | func Panicf(format string, args ...interface{}) { 278 | baseLogger.Panicf(format, args...) 279 | } 280 | -------------------------------------------------------------------------------- /pkg/operator/crd.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/runtime" 6 | "k8s.io/apimachinery/pkg/watch" 7 | "k8s.io/client-go/tools/cache" 8 | 9 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 10 | "github.com/spotahome/service-level-operator/pkg/log" 11 | "github.com/spotahome/service-level-operator/pkg/service/kubernetes" 12 | ) 13 | 14 | // serviceLevelCRD is the crd release. 15 | type serviceLevelCRD struct { 16 | cfg Config 17 | service kubernetes.Service 18 | logger log.Logger 19 | } 20 | 21 | func newServiceLevelCRD(cfg Config, service kubernetes.Service, logger log.Logger) *serviceLevelCRD { 22 | logger = logger.With("crd", "servicelevel") 23 | return &serviceLevelCRD{ 24 | cfg: cfg, 25 | service: service, 26 | logger: logger, 27 | } 28 | } 29 | 30 | // Initialize satisfies resource.crd interface. 31 | func (s *serviceLevelCRD) Initialize() error { 32 | crd := kubernetes.CRDConf{ 33 | Kind: monitoringv1alpha1.ServiceLevelKind, 34 | NamePlural: monitoringv1alpha1.ServiceLevelNamePlural, 35 | Group: monitoringv1alpha1.SchemeGroupVersion.Group, 36 | Version: monitoringv1alpha1.SchemeGroupVersion.Version, 37 | Scope: monitoringv1alpha1.ServiceLevelScope, 38 | Categories: []string{"monitoring", "slo"}, 39 | EnableStatusSubresource: true, 40 | } 41 | 42 | return s.service.EnsurePresentCRD(crd) 43 | } 44 | 45 | // GetListerWatcher satisfies resource.crd interface (and retrieve.Retriever). 46 | func (s *serviceLevelCRD) GetListerWatcher() cache.ListerWatcher { 47 | return &cache.ListWatch{ 48 | ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { 49 | options.LabelSelector = s.cfg.LabelSelector 50 | return s.service.ListServiceLevels(s.cfg.Namespace, options) 51 | }, 52 | WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { 53 | options.LabelSelector = s.cfg.LabelSelector 54 | return s.service.WatchServiceLevels(s.cfg.Namespace, options) 55 | }, 56 | } 57 | } 58 | 59 | // GetObject satisfies resource.crd interface (and retrieve.Retriever). 60 | func (s *serviceLevelCRD) GetObject() runtime.Object { 61 | return &monitoringv1alpha1.ServiceLevel{} 62 | } 63 | -------------------------------------------------------------------------------- /pkg/operator/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | operator handles handles the measurement of the different SLIs and SLOs based on 3 | the metrics described in the CRDs and creating new metrics that will expose the 4 | current SLO measurements of the services. 5 | */ 6 | 7 | package operator 8 | -------------------------------------------------------------------------------- /pkg/operator/factory.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | kmetrics "github.com/spotahome/kooper/monitoring/metrics" 8 | "github.com/spotahome/kooper/operator" 9 | "github.com/spotahome/kooper/operator/controller" 10 | 11 | "github.com/spotahome/service-level-operator/pkg/log" 12 | promcli "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 13 | "github.com/spotahome/service-level-operator/pkg/service/kubernetes" 14 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 15 | "github.com/spotahome/service-level-operator/pkg/service/output" 16 | "github.com/spotahome/service-level-operator/pkg/service/sli" 17 | ) 18 | 19 | const ( 20 | operatorName = "service-level-operator" 21 | jobRetries = 3 22 | ) 23 | 24 | // Config is the configuration for the ci operator. 25 | type Config struct { 26 | // ResyncPeriod is the resync period of the controllers. 27 | ResyncPeriod time.Duration 28 | // ConcurretWorkers are number of workers to handle the events. 29 | ConcurretWorkers int 30 | // LabelSelector is the label selector to filter Kubernetes resources by labels. 31 | LabelSelector string 32 | // Namespace is the namespace to filter Kubernetes resources by a single namespace. 33 | Namespace string 34 | } 35 | 36 | // New returns pod terminator operator. 37 | func New(cfg Config, promreg *prometheus.Registry, promCliFactory promcli.ClientFactory, k8ssvc kubernetes.Service, metricssvc metrics.Service, logger log.Logger) (operator.Operator, error) { 38 | 39 | // Create crd. 40 | ptCRD := newServiceLevelCRD(cfg, k8ssvc, logger) 41 | 42 | // Create services. 43 | promRetriever := sli.NewPrometheus(promCliFactory, logger.WithField("sli-retriever", "prometheus")) 44 | retrieverFact := sli.NewRetrieverFactory( 45 | sli.NewMetricsMiddleware(metricssvc, "prometheus", promRetriever), 46 | ) 47 | 48 | promOutput := output.NewPrometheus(output.PrometheusCfg{}, promreg, logger.WithField("slo-output", "prometheus")) 49 | outputFact := output.NewFactory( 50 | output.NewMetricsMiddleware(metricssvc, "prometheus", promOutput), 51 | ) 52 | 53 | // Create handler. 54 | handler := NewHandler(outputFact, retrieverFact, logger) 55 | 56 | // Create controller. 57 | ctrlCfg := &controller.Config{ 58 | Name: operatorName, 59 | ConcurrentWorkers: cfg.ConcurretWorkers, 60 | ResyncInterval: cfg.ResyncPeriod, 61 | ProcessingJobRetries: jobRetries, 62 | } 63 | 64 | ctrl := controller.New( 65 | ctrlCfg, 66 | handler, 67 | ptCRD, 68 | nil, 69 | nil, 70 | kmetrics.NewPrometheus(promreg), 71 | logger) 72 | 73 | // Assemble CRD and controller to create the operator. 74 | return operator.NewOperator(ptCRD, ctrl, logger), nil 75 | } 76 | -------------------------------------------------------------------------------- /pkg/operator/handler.go: -------------------------------------------------------------------------------- 1 | package operator 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "k8s.io/apimachinery/pkg/runtime" 9 | 10 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 11 | "github.com/spotahome/service-level-operator/pkg/log" 12 | "github.com/spotahome/service-level-operator/pkg/service/output" 13 | "github.com/spotahome/service-level-operator/pkg/service/sli" 14 | ) 15 | 16 | // Handler is the Operator handler. 17 | type Handler struct { 18 | outputerFact output.Factory 19 | retrieverFact sli.RetrieverFactory 20 | logger log.Logger 21 | } 22 | 23 | // NewHandler returns a new project handler 24 | func NewHandler(outputerFact output.Factory, retrieverFact sli.RetrieverFactory, logger log.Logger) *Handler { 25 | return &Handler{ 26 | outputerFact: outputerFact, 27 | retrieverFact: retrieverFact, 28 | logger: logger, 29 | } 30 | } 31 | 32 | // Add will ensure the the ci builds and jobs are persisted. 33 | func (h *Handler) Add(_ context.Context, obj runtime.Object) error { 34 | sl, ok := obj.(*monitoringv1alpha1.ServiceLevel) 35 | if !ok { 36 | return fmt.Errorf("can't handle received object, it's not a service level object") 37 | } 38 | 39 | slc := sl.DeepCopy() 40 | 41 | err := slc.Validate() 42 | if err != nil { 43 | return err 44 | } 45 | 46 | var wg sync.WaitGroup 47 | wg.Add(len(slc.Spec.ServiceLevelObjectives)) 48 | 49 | // Retrieve the SLIs. 50 | for _, slo := range slc.Spec.ServiceLevelObjectives { 51 | slo := slo 52 | 53 | go func() { 54 | defer wg.Done() 55 | err := h.processSLO(slc, &slo) 56 | // Don't stop if one of the SLOs errors, the rest should 57 | // be processed independently. 58 | if err != nil { 59 | h.logger.With("sl", sl.Name).With("slo", slo.Name).Errorf("error processing SLO: %s", err) 60 | } 61 | }() 62 | } 63 | 64 | wg.Wait() 65 | return nil 66 | } 67 | 68 | func (h *Handler) processSLO(sl *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO) error { 69 | if slo.Disable { 70 | h.logger.Debugf("ignoring SLO %s", slo.Name) 71 | return nil 72 | } 73 | 74 | retriever, err := h.retrieverFact.GetStrategy(&slo.ServiceLevelIndicator) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | res, err := retriever.Retrieve(&slo.ServiceLevelIndicator) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | outputer, err := h.outputerFact.GetStrategy(slo) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | err = outputer.Create(sl, slo, &res) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | return nil 95 | } 96 | 97 | // Delete handles the deletion of a release. 98 | func (h *Handler) Delete(_ context.Context, name string) error { 99 | h.logger.Debugf("delete received") 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /pkg/operator/handler_test.go: -------------------------------------------------------------------------------- 1 | package operator_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/mock" 9 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 10 | 11 | moutput "github.com/spotahome/service-level-operator/mocks/service/output" 12 | msli "github.com/spotahome/service-level-operator/mocks/service/sli" 13 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 14 | "github.com/spotahome/service-level-operator/pkg/log" 15 | "github.com/spotahome/service-level-operator/pkg/operator" 16 | "github.com/spotahome/service-level-operator/pkg/service/output" 17 | "github.com/spotahome/service-level-operator/pkg/service/sli" 18 | ) 19 | 20 | var ( 21 | sl0 = &monitoringv1alpha1.ServiceLevel{ 22 | ObjectMeta: metav1.ObjectMeta{ 23 | Name: "fake-service0", 24 | Namespace: "fake", 25 | }, 26 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 27 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 28 | { 29 | Name: "slo0", 30 | AvailabilityObjectivePercent: 99.99, 31 | Disable: true, 32 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 33 | SLISource: monitoringv1alpha1.SLISource{ 34 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 35 | Address: "http://127.0.0.1:9090", 36 | TotalQuery: `sum(increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[5m]))`, 37 | ErrorQuery: `sum(increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com", code=~"5.."}[5m]))`, 38 | }, 39 | }, 40 | }, 41 | Output: monitoringv1alpha1.Output{ 42 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 43 | }, 44 | }, 45 | }, 46 | }, 47 | } 48 | 49 | sl1 = &monitoringv1alpha1.ServiceLevel{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Name: "fake-service0", 52 | Namespace: "fake", 53 | }, 54 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 55 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 56 | { 57 | Name: "slo0", 58 | AvailabilityObjectivePercent: 99.95, 59 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 60 | SLISource: monitoringv1alpha1.SLISource{ 61 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 62 | }, 63 | }, 64 | Output: monitoringv1alpha1.Output{ 65 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 66 | }, 67 | }, 68 | { 69 | Name: "slo1", 70 | AvailabilityObjectivePercent: 99.99, 71 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 72 | SLISource: monitoringv1alpha1.SLISource{ 73 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 74 | }, 75 | }, 76 | Output: monitoringv1alpha1.Output{ 77 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 78 | }, 79 | }, 80 | { 81 | Name: "slo2", 82 | AvailabilityObjectivePercent: 99.9, 83 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 84 | SLISource: monitoringv1alpha1.SLISource{ 85 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 86 | }, 87 | }, 88 | Output: monitoringv1alpha1.Output{ 89 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 90 | }, 91 | }, 92 | { 93 | Name: "slo3", 94 | AvailabilityObjectivePercent: 99.9999, 95 | Disable: true, 96 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 97 | SLISource: monitoringv1alpha1.SLISource{ 98 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{}, 99 | }, 100 | }, 101 | Output: monitoringv1alpha1.Output{ 102 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 103 | }, 104 | }, 105 | }, 106 | }, 107 | } 108 | ) 109 | 110 | func TestHandler(t *testing.T) { 111 | tests := []struct { 112 | name string 113 | serviceLevel *monitoringv1alpha1.ServiceLevel 114 | processTimes int 115 | expErr bool 116 | }{ 117 | { 118 | name: "With disabled SLO should not process anything.", 119 | serviceLevel: sl0, 120 | processTimes: 0, 121 | expErr: false, 122 | }, 123 | { 124 | name: "A service level with multiple slos should process all slos.", 125 | serviceLevel: sl1, 126 | processTimes: 3, 127 | expErr: false, 128 | }, 129 | } 130 | 131 | for _, test := range tests { 132 | t.Run(test.name, func(t *testing.T) { 133 | assert := assert.New(t) 134 | 135 | // Mocks. 136 | mout := &moutput.Output{} 137 | moutf := output.MockFactory{Mock: mout} 138 | mret := &msli.Retriever{} 139 | mretf := sli.MockRetrieverFactory{Mock: mret} 140 | 141 | if test.processTimes > 0 { 142 | mout.On("Create", mock.Anything, mock.Anything, mock.Anything).Times(test.processTimes).Return(nil) 143 | mret.On("Retrieve", mock.Anything).Times(test.processTimes).Return(sli.Result{}, nil) 144 | } 145 | 146 | h := operator.NewHandler(moutf, mretf, log.Dummy) 147 | err := h.Add(context.Background(), test.serviceLevel) 148 | 149 | if test.expErr { 150 | assert.Error(err) 151 | } else if assert.NoError(err) { 152 | mout.AssertExpectations(t) 153 | mret.AssertExpectations(t) 154 | } 155 | }) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/service/client/kubernetes/factory.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 5 | "k8s.io/client-go/kubernetes" 6 | "k8s.io/client-go/rest" 7 | 8 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 9 | ) 10 | 11 | // ClientFactory knows how to get Kubernetes clients. 12 | type ClientFactory interface { 13 | // GetSTDClient gets the Kubernetes standard client (pods, services...). 14 | GetSTDClient() (kubernetes.Interface, error) 15 | // GetCRDClient gets the Kubernetes client for the CRDs described in this application. 16 | GetCRDClient() (crdcli.Interface, error) 17 | // GetAPIExtensionClient gets the Kubernetes api extensions client (crds...). 18 | GetAPIExtensionClient() (apiextensionscli.Interface, error) 19 | } 20 | 21 | type factory struct { 22 | restCfg *rest.Config 23 | 24 | stdcli kubernetes.Interface 25 | crdcli crdcli.Interface 26 | aexcli apiextensionscli.Interface 27 | } 28 | 29 | // NewFactory returns a new kubernetes client factory. 30 | func NewFactory(config *rest.Config) ClientFactory { 31 | return &factory{ 32 | restCfg: config, 33 | } 34 | } 35 | 36 | func (f *factory) GetSTDClient() (kubernetes.Interface, error) { 37 | if f.stdcli == nil { 38 | cli, err := kubernetes.NewForConfig(f.restCfg) 39 | if err != nil { 40 | return nil, err 41 | } 42 | f.stdcli = cli 43 | } 44 | return f.stdcli, nil 45 | } 46 | func (f *factory) GetCRDClient() (crdcli.Interface, error) { 47 | if f.crdcli == nil { 48 | cli, err := crdcli.NewForConfig(f.restCfg) 49 | if err != nil { 50 | return nil, err 51 | } 52 | f.crdcli = cli 53 | } 54 | return f.crdcli, nil 55 | } 56 | func (f *factory) GetAPIExtensionClient() (apiextensionscli.Interface, error) { 57 | if f.aexcli == nil { 58 | cli, err := apiextensionscli.NewForConfig(f.restCfg) 59 | if err != nil { 60 | return nil, err 61 | } 62 | f.aexcli = cli 63 | } 64 | return f.aexcli, nil 65 | } 66 | -------------------------------------------------------------------------------- /pkg/service/client/kubernetes/fake.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 5 | apiextensionsclifake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" 6 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 7 | "k8s.io/apimachinery/pkg/runtime" 8 | "k8s.io/apimachinery/pkg/version" 9 | fakediscovery "k8s.io/client-go/discovery/fake" 10 | "k8s.io/client-go/kubernetes" 11 | kubernetesfake "k8s.io/client-go/kubernetes/fake" 12 | 13 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 14 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 15 | crdclifake "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned/fake" 16 | ) 17 | 18 | // fakeFactory is a fake factory that has already loaded faked objects on the Kubernetes clients. 19 | type fakeFactory struct{} 20 | 21 | // NewFake returns the faked Kubernetes clients factory. 22 | func NewFake() ClientFactory { 23 | return &fakeFactory{} 24 | } 25 | 26 | func (f *fakeFactory) GetSTDClient() (kubernetes.Interface, error) { 27 | return kubernetesfake.NewSimpleClientset(stdObjs...), nil 28 | } 29 | func (f *fakeFactory) GetCRDClient() (crdcli.Interface, error) { 30 | return crdclifake.NewSimpleClientset(crdObjs...), nil 31 | } 32 | func (f *fakeFactory) GetAPIExtensionClient() (apiextensionscli.Interface, error) { 33 | cli := apiextensionsclifake.NewSimpleClientset(aexObjs...) 34 | 35 | // Fake cluster version (Required for CRD version checks). 36 | fakeDiscovery, _ := cli.Discovery().(*fakediscovery.FakeDiscovery) 37 | fakeDiscovery.FakedServerVersion = &version.Info{ 38 | GitVersion: "v1.10.5", 39 | } 40 | 41 | return cli, nil 42 | } 43 | 44 | var ( 45 | stdObjs = []runtime.Object{} 46 | 47 | // The field selector doesn't work with a fake K8s client: https://github.com/kubernetes/client-go/issues/326 48 | crdObjs = []runtime.Object{ 49 | &monitoringv1alpha1.ServiceLevel{ 50 | ObjectMeta: metav1.ObjectMeta{ 51 | Name: "fake-service0", 52 | Namespace: "ns0", 53 | Labels: map[string]string{ 54 | "wrong": "false", 55 | }, 56 | }, 57 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 58 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 59 | { 60 | Name: "fake_slo0", 61 | Description: "fake slo 0.", 62 | AvailabilityObjectivePercent: 99.99, 63 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 64 | SLISource: monitoringv1alpha1.SLISource{ 65 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 66 | Address: "http://fake:9090", 67 | TotalQuery: `slo0_total`, 68 | ErrorQuery: `slo0_error`, 69 | }, 70 | }, 71 | }, 72 | Output: monitoringv1alpha1.Output{ 73 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 74 | Labels: map[string]string{ 75 | "fake": "true", 76 | "team": "fake-team0", 77 | }, 78 | }, 79 | }, 80 | }, 81 | { 82 | Name: "fake_slo1", 83 | Description: "fake slo 1.", 84 | AvailabilityObjectivePercent: 99.9, 85 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 86 | SLISource: monitoringv1alpha1.SLISource{ 87 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 88 | Address: "http://fake:9090", 89 | TotalQuery: `slo1_total`, 90 | ErrorQuery: `slo1_error`, 91 | }, 92 | }, 93 | }, 94 | Output: monitoringv1alpha1.Output{ 95 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 96 | Labels: map[string]string{ 97 | "fake": "true", 98 | "team": "fake-team1", 99 | }, 100 | }, 101 | }, 102 | }, 103 | { 104 | Name: "fake_slo2", 105 | Description: "fake slo 2.", 106 | AvailabilityObjectivePercent: 99.998, 107 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 108 | SLISource: monitoringv1alpha1.SLISource{ 109 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 110 | Address: "http://fake:9090", 111 | TotalQuery: `slo2_total`, 112 | ErrorQuery: `slo2_error`, 113 | }, 114 | }, 115 | }, 116 | Output: monitoringv1alpha1.Output{ 117 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 118 | Labels: map[string]string{ 119 | "fake": "true", 120 | "team": "fake-team2", 121 | }, 122 | }, 123 | }, 124 | }, 125 | }, 126 | }, 127 | }, 128 | &monitoringv1alpha1.ServiceLevel{ 129 | ObjectMeta: metav1.ObjectMeta{ 130 | Name: "fake-service1", 131 | Namespace: "ns1", 132 | Labels: map[string]string{ 133 | "wrong": "false", 134 | }, 135 | }, 136 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 137 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 138 | { 139 | Name: "fake_slo3", 140 | Description: "fake slo 3.", 141 | AvailabilityObjectivePercent: 99, 142 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 143 | SLISource: monitoringv1alpha1.SLISource{ 144 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 145 | Address: "http://fake:9090", 146 | TotalQuery: `slo3_total`, 147 | ErrorQuery: `slo3_error`, 148 | }, 149 | }, 150 | }, 151 | Output: monitoringv1alpha1.Output{ 152 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 153 | Labels: map[string]string{ 154 | "fake": "true", 155 | "team": "fake-team3", 156 | }, 157 | }, 158 | }, 159 | }, 160 | }, 161 | }, 162 | }, 163 | &monitoringv1alpha1.ServiceLevel{ 164 | ObjectMeta: metav1.ObjectMeta{ 165 | Name: "fake-service2-no-output", 166 | Namespace: "ns0", 167 | Labels: map[string]string{ 168 | "wrong": "true", 169 | }, 170 | }, 171 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 172 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 173 | { 174 | Name: "fake_slo4", 175 | Description: "fake slo 4.", 176 | AvailabilityObjectivePercent: 99, 177 | ServiceLevelIndicator: monitoringv1alpha1.SLI{ 178 | SLISource: monitoringv1alpha1.SLISource{ 179 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 180 | Address: "http://fake:9090", 181 | TotalQuery: `slo3_total`, 182 | ErrorQuery: `slo3_error`, 183 | }, 184 | }, 185 | }, 186 | Output: monitoringv1alpha1.Output{}, 187 | }, 188 | }, 189 | }, 190 | }, 191 | 192 | &monitoringv1alpha1.ServiceLevel{ 193 | ObjectMeta: metav1.ObjectMeta{ 194 | Name: "fake-service3-no-input", 195 | Namespace: "ns1", 196 | Labels: map[string]string{ 197 | "wrong": "true", 198 | }, 199 | }, 200 | Spec: monitoringv1alpha1.ServiceLevelSpec{ 201 | ServiceLevelObjectives: []monitoringv1alpha1.SLO{ 202 | { 203 | Name: "fake_slo5", 204 | Description: "fake slo 5.", 205 | AvailabilityObjectivePercent: 99, 206 | ServiceLevelIndicator: monitoringv1alpha1.SLI{}, 207 | Output: monitoringv1alpha1.Output{ 208 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 209 | Labels: map[string]string{ 210 | "wrong": "true", 211 | }, 212 | }, 213 | }, 214 | }, 215 | }, 216 | }, 217 | }, 218 | } 219 | 220 | aexObjs = []runtime.Object{} 221 | ) 222 | -------------------------------------------------------------------------------- /pkg/service/client/prometheus/factory.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/prometheus/client_golang/api" 8 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 9 | ) 10 | 11 | // ClientFactory knows how to get prometheus API clients. 12 | type ClientFactory interface { 13 | // GetV1APIClient returns a new prometheus v1 API client. 14 | // address is the address of the prometheus. 15 | GetV1APIClient(address string) (promv1.API, error) 16 | } 17 | 18 | // BaseFactory returns Prometheus clients based on the address. 19 | // This factory implements a way of returning default Prometheus 20 | // clients in case it was set. 21 | type BaseFactory struct { 22 | v1Clis map[string]api.Client 23 | climu sync.Mutex 24 | } 25 | 26 | // NewBaseFactory returns a new client Basefactory. 27 | func NewBaseFactory() *BaseFactory { 28 | return &BaseFactory{ 29 | v1Clis: map[string]api.Client{}, 30 | } 31 | } 32 | 33 | // GetV1APIClient satisfies ClientFactory interface. 34 | func (f *BaseFactory) GetV1APIClient(address string) (promv1.API, error) { 35 | f.climu.Lock() 36 | defer f.climu.Unlock() 37 | 38 | var err error 39 | cli, ok := f.v1Clis[address] 40 | if !ok { 41 | cli, err = newClient(address) 42 | if err != nil { 43 | return nil, fmt.Errorf("error creating prometheus client: %s", err) 44 | } 45 | f.v1Clis[address] = cli 46 | } 47 | return promv1.NewAPI(cli), nil 48 | } 49 | 50 | // WithDefaultV1APIClient sets a default client for V1 api client. 51 | func (f *BaseFactory) WithDefaultV1APIClient(address string) error { 52 | const defAddressKey = "" 53 | f.climu.Lock() 54 | defer f.climu.Unlock() 55 | 56 | dc, err := newClient(address) 57 | if err != nil { 58 | return fmt.Errorf("error creating prometheus client: %s", err) 59 | } 60 | f.v1Clis[defAddressKey] = dc 61 | 62 | return nil 63 | } 64 | 65 | func newClient(address string) (api.Client, error) { 66 | if address == "" { 67 | return nil, fmt.Errorf("address can't be empty") 68 | } 69 | 70 | return api.NewClient(api.Config{Address: address}) 71 | } 72 | 73 | // MockFactory returns a predefined prometheus v1 API client. 74 | type MockFactory struct { 75 | Cli promv1.API 76 | } 77 | 78 | // GetV1APIClient satisfies ClientFactory interface. 79 | func (m *MockFactory) GetV1APIClient(_ string) (promv1.API, error) { 80 | return m.Cli, nil 81 | } 82 | -------------------------------------------------------------------------------- /pkg/service/client/prometheus/factory_test.go: -------------------------------------------------------------------------------- 1 | package prometheus_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 9 | ) 10 | 11 | func TestBaseFactoryV1Client(t *testing.T) { 12 | tests := map[string]struct { 13 | cli func() *prometheus.BaseFactory 14 | address string 15 | expErr bool 16 | }{ 17 | "A regular client address should be returned without error.": { 18 | cli: func() *prometheus.BaseFactory { 19 | return prometheus.NewBaseFactory() 20 | }, 21 | address: "http://127.0.0.1:9090", 22 | }, 23 | 24 | "Getting a missing address client should error.": { 25 | cli: func() *prometheus.BaseFactory { 26 | return prometheus.NewBaseFactory() 27 | }, 28 | address: "", 29 | expErr: true, 30 | }, 31 | 32 | "Getting a missing address client with a default client it should not error.": { 33 | cli: func() *prometheus.BaseFactory { 34 | f := prometheus.NewBaseFactory() 35 | f.WithDefaultV1APIClient("http://127.0.0.1:9090") 36 | return f 37 | }, 38 | address: "", 39 | expErr: false, 40 | }, 41 | } 42 | 43 | for name, test := range tests { 44 | t.Run(name, func(t *testing.T) { 45 | assert := assert.New(t) 46 | 47 | f := test.cli() 48 | _, err := f.GetV1APIClient(test.address) 49 | 50 | if test.expErr { 51 | assert.Error(err) 52 | } else { 53 | assert.NoError(err) 54 | } 55 | }) 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /pkg/service/client/prometheus/fake.go: -------------------------------------------------------------------------------- 1 | package prometheus 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/api" 9 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 10 | "github.com/prometheus/common/model" 11 | ) 12 | 13 | var ( 14 | slo3CallCount int 15 | ) 16 | 17 | type fakeFactory struct { 18 | } 19 | 20 | // NewFakeFactory returns a new fake factory. 21 | func NewFakeFactory() ClientFactory { 22 | return &fakeFactory{} 23 | } 24 | 25 | // GetV1APIClient satisfies ClientFactory interface. 26 | func (f *fakeFactory) GetV1APIClient(_ string) (promv1.API, error) { 27 | return &fakeAPICli{ 28 | queryFuncs: map[string]func() float64{ 29 | "slo0_total": func() float64 { return 100 }, 30 | "slo0_error": func() float64 { return 1 }, 31 | "slo1_total": func() float64 { return 1000 }, 32 | "slo1_error": func() float64 { return 1 }, 33 | "slo2_total": func() float64 { return 100000 }, 34 | "slo2_error": func() float64 { return 12 }, 35 | "slo3_total": func() float64 { return 10000 }, 36 | "slo3_error": func() float64 { 37 | // Every 2 calls return error. 38 | slo3CallCount++ 39 | if slo3CallCount%2 == 0 { 40 | return 1 41 | } 42 | return 0 43 | }, 44 | }, 45 | }, nil 46 | } 47 | 48 | // fakeAPICli is a faked http client. 49 | type fakeAPICli struct { 50 | queryFuncs map[string]func() float64 51 | } 52 | 53 | func (f *fakeAPICli) Query(_ context.Context, query string, ts time.Time) (model.Value, api.Warnings, error) { 54 | 55 | fn, ok := f.queryFuncs[query] 56 | if !ok { 57 | return nil, nil, fmt.Errorf("not faked result") 58 | } 59 | 60 | return model.Vector{ 61 | &model.Sample{ 62 | Metric: model.Metric{}, 63 | Timestamp: model.Time(time.Now().UTC().Nanosecond()), 64 | Value: model.SampleValue(fn()), 65 | }, 66 | }, nil, nil 67 | } 68 | 69 | func (f *fakeAPICli) Alerts(ctx context.Context) (promv1.AlertsResult, error) { 70 | return promv1.AlertsResult{}, nil 71 | } 72 | func (f *fakeAPICli) AlertManagers(_ context.Context) (promv1.AlertManagersResult, error) { 73 | return promv1.AlertManagersResult{}, nil 74 | } 75 | func (f *fakeAPICli) CleanTombstones(_ context.Context) error { 76 | return nil 77 | } 78 | func (f *fakeAPICli) Config(_ context.Context) (promv1.ConfigResult, error) { 79 | return promv1.ConfigResult{}, nil 80 | } 81 | func (f *fakeAPICli) DeleteSeries(_ context.Context, matches []string, startTime time.Time, endTime time.Time) error { 82 | return nil 83 | } 84 | func (f *fakeAPICli) Flags(_ context.Context) (promv1.FlagsResult, error) { 85 | return promv1.FlagsResult{}, nil 86 | } 87 | func (f *fakeAPICli) LabelNames(ctx context.Context) ([]string, api.Warnings, error) { 88 | return nil, nil, nil 89 | } 90 | func (f *fakeAPICli) LabelValues(_ context.Context, label string) (model.LabelValues, api.Warnings, error) { 91 | return model.LabelValues{}, nil, nil 92 | } 93 | func (f *fakeAPICli) QueryRange(_ context.Context, query string, r promv1.Range) (model.Value, api.Warnings, error) { 94 | return nil, nil, nil 95 | } 96 | func (f *fakeAPICli) Series(_ context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, api.Warnings, error) { 97 | return []model.LabelSet{}, nil, nil 98 | } 99 | func (f *fakeAPICli) Snapshot(_ context.Context, skipHead bool) (promv1.SnapshotResult, error) { 100 | return promv1.SnapshotResult{}, nil 101 | } 102 | func (f *fakeAPICli) Rules(ctx context.Context) (promv1.RulesResult, error) { 103 | return promv1.RulesResult{}, nil 104 | } 105 | func (f *fakeAPICli) Targets(_ context.Context) (promv1.TargetsResult, error) { 106 | return promv1.TargetsResult{}, nil 107 | } 108 | func (f *fakeAPICli) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]promv1.MetricMetadata, error) { 109 | return nil, nil 110 | } 111 | -------------------------------------------------------------------------------- /pkg/service/configuration/configuration.go: -------------------------------------------------------------------------------- 1 | package configuration 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "io/ioutil" 8 | ) 9 | 10 | // DefaultSLISource is a configuration object with the default 11 | // endpoints. 12 | type DefaultSLISource struct { 13 | Prometheus PrometheusSLISource `json:"prometheus,omitempty"` 14 | } 15 | 16 | // PrometheusSLISource is the default prometheus source. 17 | type PrometheusSLISource struct { 18 | Address string `json:"address,omitempty"` 19 | } 20 | 21 | 22 | // Loader knows how to load configuration based on different formats. 23 | // At this moment configuration is not versioned, the configuration 24 | // is so simple that if it grows we could refactor and add version, 25 | // in this case not versioned configuration could be loaded as v1. 26 | type Loader interface { 27 | // LoadDefaultSLISource will load the default sli source configuration . 28 | LoadDefaultSLISource(ctx context.Context, r io.Reader) (*DefaultSLISource, error) 29 | } 30 | 31 | // JSONLoader knows how to load application configuration. 32 | type JSONLoader struct{} 33 | 34 | // LoadDefaultSLISource satisfies Loader interface by loading in JSON format. 35 | func (j JSONLoader) LoadDefaultSLISource(_ context.Context, r io.Reader) (*DefaultSLISource, error) { 36 | bs, err := ioutil.ReadAll(r) 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | cfg := &DefaultSLISource{} 42 | err = json.Unmarshal(bs, cfg) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | return cfg, nil 48 | } 49 | -------------------------------------------------------------------------------- /pkg/service/configuration/configuration_test.go: -------------------------------------------------------------------------------- 1 | package configuration_test 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/spotahome/service-level-operator/pkg/service/configuration" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestJSONLoaderLoadDefaultSLISource(t *testing.T) { 13 | tests := map[string]struct { 14 | jsonConfig string 15 | expConfig *configuration.DefaultSLISource 16 | expErr bool 17 | }{ 18 | "Correct JSON configuration should be loaded without error.": { 19 | jsonConfig: `{"prometheus": {"address": "http://test:9090"}}`, 20 | expConfig: &configuration.DefaultSLISource{ 21 | Prometheus: configuration.PrometheusSLISource{ 22 | Address: "http://test:9090", 23 | }, 24 | }, 25 | }, 26 | 27 | "A malformed JSON should error.": { 28 | jsonConfig: `{"prometheus":`, 29 | expErr: true, 30 | }, 31 | } 32 | 33 | for name, test := range tests { 34 | t.Run(name, func(t *testing.T) { 35 | assert := assert.New(t) 36 | 37 | r := strings.NewReader(test.jsonConfig) 38 | gotConfig, err := configuration.JSONLoader{}.LoadDefaultSLISource(context.TODO(), r) 39 | if test.expErr { 40 | assert.Error(err) 41 | } else if assert.NoError(err) { 42 | assert.Equal(test.expConfig, gotConfig) 43 | } 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/service/kubernetes/crd.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | koopercrd "github.com/spotahome/kooper/client/crd" 5 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 6 | 7 | "github.com/spotahome/service-level-operator/pkg/log" 8 | ) 9 | 10 | // CRDConf is the configuration of the crd. 11 | type CRDConf = koopercrd.Conf 12 | 13 | // CRD is the CRD service that knows how to interact with k8s to manage them. 14 | type CRD interface { 15 | // EnsurePresentCRD will create the custom resource and wait to be ready 16 | // if there is not already present. 17 | EnsurePresentCRD(conf CRDConf) error 18 | } 19 | 20 | // crdService is the CRD service implementation using API calls to kubernetes. 21 | type crd struct { 22 | crdCli koopercrd.Interface 23 | logger log.Logger 24 | } 25 | 26 | // NewCRD returns a new CRD KubeService. 27 | func NewCRD(aeClient apiextensionscli.Interface, logger log.Logger) CRD { 28 | logger = logger.With("service", "k8s.crd") 29 | crdCli := koopercrd.NewClient(aeClient, logger) 30 | 31 | return &crd{ 32 | crdCli: crdCli, 33 | logger: logger, 34 | } 35 | } 36 | 37 | // EnsurePresentCRD satisfies workspace.Service interface. 38 | func (c *crd) EnsurePresentCRD(conf CRDConf) error { 39 | return c.crdCli.EnsurePresent(conf) 40 | } 41 | -------------------------------------------------------------------------------- /pkg/service/kubernetes/kubernetes.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | apiextensionscli "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" 5 | "k8s.io/client-go/kubernetes" 6 | 7 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 8 | "github.com/spotahome/service-level-operator/pkg/log" 9 | ) 10 | 11 | // Service is the service used to interact with the Kubernetes 12 | // objects. 13 | type Service interface { 14 | ServiceLevel 15 | CRD 16 | } 17 | 18 | type service struct { 19 | ServiceLevel 20 | CRD 21 | } 22 | 23 | // New returns a new Kubernetes service. 24 | func New(stdcli kubernetes.Interface, crdcli crdcli.Interface, apiextcli apiextensionscli.Interface, logger log.Logger) Service { 25 | return &service{ 26 | ServiceLevel: NewServiceLevel(crdcli, logger), 27 | CRD: NewCRD(apiextcli, logger), 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pkg/service/kubernetes/servicelevel.go: -------------------------------------------------------------------------------- 1 | package kubernetes 2 | 3 | import ( 4 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 5 | "k8s.io/apimachinery/pkg/watch" 6 | 7 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 8 | crdcli "github.com/spotahome/service-level-operator/pkg/k8sautogen/client/clientset/versioned" 9 | "github.com/spotahome/service-level-operator/pkg/log" 10 | ) 11 | 12 | // ServiceLevel knows how to interact with Kubernetes on the 13 | // ServiceLevel CRs 14 | type ServiceLevel interface { 15 | // ListServiceLevels will list the service levels. 16 | ListServiceLevels(namespace string, opts metav1.ListOptions) (*monitoringv1alpha1.ServiceLevelList, error) 17 | // ListServiceLevels will list the service levels. 18 | WatchServiceLevels(namespace string, opt metav1.ListOptions) (watch.Interface, error) 19 | } 20 | 21 | type serviceLevel struct { 22 | cli crdcli.Interface 23 | logger log.Logger 24 | } 25 | 26 | // NewServiceLevel returns a new service level service. 27 | func NewServiceLevel(crdcli crdcli.Interface, logger log.Logger) ServiceLevel { 28 | return &serviceLevel{ 29 | cli: crdcli, 30 | logger: logger, 31 | } 32 | } 33 | 34 | func (s *serviceLevel) ListServiceLevels(namespace string, opts metav1.ListOptions) (*monitoringv1alpha1.ServiceLevelList, error) { 35 | return s.cli.MonitoringV1alpha1().ServiceLevels(namespace).List(opts) 36 | } 37 | func (s *serviceLevel) WatchServiceLevels(namespace string, opts metav1.ListOptions) (watch.Interface, error) { 38 | return s.cli.MonitoringV1alpha1().ServiceLevels(namespace).Watch(opts) 39 | } 40 | -------------------------------------------------------------------------------- /pkg/service/metrics/dummy.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Dummy is a Dummy implementation of the metrics service. 10 | var Dummy = &dummy{} 11 | 12 | type dummy struct{} 13 | 14 | func (dummy) ObserveSLIRetrieveDuration(_ *monitoringv1alpha1.SLI, _ string, startTime time.Time) {} 15 | func (dummy) IncSLIRetrieveError(_ *monitoringv1alpha1.SLI, _ string) {} 16 | func (dummy) ObserveOuputCreateDuration(_ *monitoringv1alpha1.SLO, _ string, startTime time.Time) {} 17 | func (dummy) IncOuputCreateError(_ *monitoringv1alpha1.SLO, _ string) {} 18 | -------------------------------------------------------------------------------- /pkg/service/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Service knows how to monitoring the different parts, flows and processes 10 | // of the application to give more insights and improve the observability 11 | // of the application. 12 | type Service interface { 13 | // ObserveSLIRetrieveDuration will monitoring the duration of the process of gathering the group of 14 | // SLIs for a SLO. 15 | ObserveSLIRetrieveDuration(sli *monitoringv1alpha1.SLI, kind string, startTime time.Time) 16 | // IncSLIRetrieveError will increment the number of errors on the retrieval of the SLIs. 17 | IncSLIRetrieveError(sli *monitoringv1alpha1.SLI, kind string) 18 | // ObserveOuputCreateDuration monitorings the duration of the process of creating the output for the SLO 19 | ObserveOuputCreateDuration(slo *monitoringv1alpha1.SLO, kind string, startTime time.Time) 20 | // IncOuputCreateError will increment the number of errors on the SLO output creation. 21 | IncOuputCreateError(slo *monitoringv1alpha1.SLO, kind string) 22 | } 23 | -------------------------------------------------------------------------------- /pkg/service/metrics/prometheus.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 8 | ) 9 | 10 | const ( 11 | promNamespace = "service_level" 12 | promSubsystem = "processing" 13 | ) 14 | 15 | var ( 16 | buckets = prometheus.DefBuckets 17 | ) 18 | 19 | type prometheusService struct { 20 | sliRetrieveHistogram *prometheus.HistogramVec 21 | sliRetrieveErrCounter *prometheus.CounterVec 22 | outputCreateHistogram *prometheus.HistogramVec 23 | outputCreateErrCounter *prometheus.CounterVec 24 | 25 | reg prometheus.Registerer 26 | } 27 | 28 | // NewPrometheus returns a new metrics.Service implementation that 29 | // knows how to monitor gusing Prometheus as backend. 30 | func NewPrometheus(reg prometheus.Registerer) Service { 31 | p := &prometheusService{ 32 | sliRetrieveHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 33 | Namespace: promNamespace, 34 | Subsystem: promSubsystem, 35 | Name: "sli_retrieve_duration_seconds", 36 | Help: "The duration seconds to retrieve the SLIs.", 37 | Buckets: buckets, 38 | }, []string{"kind"}), 39 | 40 | sliRetrieveErrCounter: prometheus.NewCounterVec(prometheus.CounterOpts{ 41 | Namespace: promNamespace, 42 | Subsystem: promSubsystem, 43 | Name: "sli_retrieve_failures_total", 44 | Help: "Total number sli retrieval failures.", 45 | }, []string{"kind"}), 46 | 47 | outputCreateHistogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ 48 | Namespace: promNamespace, 49 | Subsystem: promSubsystem, 50 | Name: "output_create_duration_seconds", 51 | Help: "The duration seconds to create the output of the SLI and SLO results.", 52 | Buckets: buckets, 53 | }, []string{"kind"}), 54 | 55 | outputCreateErrCounter: prometheus.NewCounterVec(prometheus.CounterOpts{ 56 | Namespace: promNamespace, 57 | Subsystem: promSubsystem, 58 | Name: "output_create_failures_total", 59 | Help: "Total number SLI and SLO output creation failures.", 60 | }, []string{"kind"}), 61 | 62 | reg: reg, 63 | } 64 | 65 | p.registerMetrics() 66 | 67 | return p 68 | } 69 | 70 | func (p prometheusService) registerMetrics() { 71 | p.reg.MustRegister( 72 | p.sliRetrieveHistogram, 73 | p.sliRetrieveErrCounter, 74 | p.outputCreateHistogram, 75 | p.outputCreateErrCounter, 76 | ) 77 | } 78 | 79 | // ObserveSLIRetrieveDuration satisfies metrics.Service interface. 80 | func (p prometheusService) ObserveSLIRetrieveDuration(_ *monitoringv1alpha1.SLI, kind string, startTime time.Time) { 81 | p.sliRetrieveHistogram.WithLabelValues(kind).Observe(time.Since(startTime).Seconds()) 82 | } 83 | 84 | // IncSLIRetrieveError satisfies metrics.Service interface. 85 | func (p prometheusService) IncSLIRetrieveError(_ *monitoringv1alpha1.SLI, kind string) { 86 | p.sliRetrieveErrCounter.WithLabelValues(kind).Inc() 87 | } 88 | 89 | // ObserveOuputCreateDuration satisfies metrics.Service interface. 90 | func (p prometheusService) ObserveOuputCreateDuration(_ *monitoringv1alpha1.SLO, kind string, startTime time.Time) { 91 | p.outputCreateHistogram.WithLabelValues(kind).Observe(time.Since(startTime).Seconds()) 92 | } 93 | 94 | // IncOuputCreateError satisfies metrics.Service interface. 95 | func (p prometheusService) IncOuputCreateError(_ *monitoringv1alpha1.SLO, kind string) { 96 | p.outputCreateErrCounter.WithLabelValues(kind).Inc() 97 | } 98 | -------------------------------------------------------------------------------- /pkg/service/metrics/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package metrics_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | "github.com/stretchr/testify/assert" 12 | 13 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 14 | ) 15 | 16 | func TestPrometheusMetrics(t *testing.T) { 17 | kind := "test" 18 | 19 | tests := []struct { 20 | name string 21 | addMetrics func(metrics.Service) 22 | expMetrics []string 23 | expCode int 24 | }{ 25 | { 26 | name: "Measuring SLO realted metrics should expose SLO processing metrics on the prometheus endpoint.", 27 | addMetrics: func(s metrics.Service) { 28 | now := time.Now() 29 | s.IncOuputCreateError(nil, kind) 30 | s.IncOuputCreateError(nil, kind) 31 | s.ObserveOuputCreateDuration(nil, kind, now.Add(-6*time.Second)) 32 | s.ObserveOuputCreateDuration(nil, kind, now.Add(-27*time.Millisecond)) 33 | }, 34 | expMetrics: []string{ 35 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.005"} 0`, 36 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.01"} 0`, 37 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.025"} 0`, 38 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.05"} 1`, 39 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.1"} 1`, 40 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.25"} 1`, 41 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="0.5"} 1`, 42 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="1"} 1`, 43 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="2.5"} 1`, 44 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="5"} 1`, 45 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="10"} 2`, 46 | `service_level_processing_output_create_duration_seconds_bucket{kind="test",le="+Inf"} 2`, 47 | `service_level_processing_output_create_duration_seconds_count{kind="test"} 2`, 48 | 49 | `service_level_processing_output_create_failures_total{kind="test"} 2`, 50 | }, 51 | expCode: 200, 52 | }, 53 | { 54 | name: "Measuring SLI realted metrics should expose SLI processing metrics on the prometheus endpoint.", 55 | addMetrics: func(s metrics.Service) { 56 | now := time.Now() 57 | s.IncSLIRetrieveError(nil, kind) 58 | s.IncSLIRetrieveError(nil, kind) 59 | s.IncSLIRetrieveError(nil, kind) 60 | s.ObserveSLIRetrieveDuration(nil, kind, now.Add(-3*time.Second)) 61 | s.ObserveSLIRetrieveDuration(nil, kind, now.Add(-15*time.Millisecond)) 62 | s.ObserveSLIRetrieveDuration(nil, kind, now.Add(-567*time.Millisecond)) 63 | }, 64 | expMetrics: []string{ 65 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.005"} 0`, 66 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.01"} 0`, 67 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.025"} 1`, 68 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.05"} 1`, 69 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.1"} 1`, 70 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.25"} 1`, 71 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="0.5"} 1`, 72 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="1"} 2`, 73 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="2.5"} 2`, 74 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="5"} 3`, 75 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="10"} 3`, 76 | `service_level_processing_sli_retrieve_duration_seconds_bucket{kind="test",le="+Inf"} 3`, 77 | `service_level_processing_sli_retrieve_duration_seconds_count{kind="test"} 3`, 78 | 79 | `service_level_processing_sli_retrieve_failures_total{kind="test"} 3`, 80 | }, 81 | expCode: 200, 82 | }, 83 | } 84 | 85 | for _, test := range tests { 86 | t.Run(test.name, func(t *testing.T) { 87 | assert := assert.New(t) 88 | 89 | reg := prometheus.NewRegistry() 90 | m := metrics.NewPrometheus(reg) 91 | 92 | // Add desired metrics 93 | test.addMetrics(m) 94 | 95 | // Ask prometheus for the metrics 96 | h := promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) 97 | r := httptest.NewRequest("GET", "/metrics", nil) 98 | w := httptest.NewRecorder() 99 | h.ServeHTTP(w, r) 100 | resp := w.Result() 101 | 102 | // Check all metrics are present. 103 | if assert.Equal(test.expCode, resp.StatusCode) { 104 | body, _ := ioutil.ReadAll(resp.Body) 105 | for _, expMetric := range test.expMetrics { 106 | assert.Contains(string(body), expMetric, "metric not present on the result of metrics service") 107 | } 108 | } 109 | }) 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /pkg/service/output/factory.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Factory is a factory that knows how to get the correct 10 | // Output strategy based on the SLO output source. 11 | type Factory interface { 12 | // GetGetStrategy returns a output based on the SLO source. 13 | GetStrategy(*monitoringv1alpha1.SLO) (Output, error) 14 | } 15 | 16 | // factory doesn't create objects per se, it only knows 17 | // what strategy to return based on the passed SLI. 18 | type factory struct { 19 | promOutput Output 20 | } 21 | 22 | // NewFactory returns a new output factory. 23 | func NewFactory(promOutput Output) Factory { 24 | return &factory{ 25 | promOutput: promOutput, 26 | } 27 | } 28 | 29 | // GetStrategy satsifies OutputFactory interface. 30 | func (f factory) GetStrategy(s *monitoringv1alpha1.SLO) (Output, error) { 31 | if s.Output.Prometheus != nil { 32 | return f.promOutput, nil 33 | } 34 | 35 | return nil, fmt.Errorf("%s unsupported output kind", s.Name) 36 | } 37 | 38 | // MockFactory returns the mocked output strategy. 39 | type MockFactory struct { 40 | Mock Output 41 | } 42 | 43 | // GetStrategy satisfies Factory interface. 44 | func (m MockFactory) GetStrategy(_ *monitoringv1alpha1.SLO) (Output, error) { 45 | return m.Mock, nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/service/output/middleware.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 8 | "github.com/spotahome/service-level-operator/pkg/service/sli" 9 | ) 10 | 11 | // metricsMiddleware will monitoring the calls to the SLO output. 12 | type metricsMiddleware struct { 13 | kind string 14 | metricssvc metrics.Service 15 | next Output 16 | } 17 | 18 | // NewMetricsMiddleware returns a new metrics middleware that wraps a Output SLO 19 | // service and monitorings with metrics. 20 | func NewMetricsMiddleware(metricssvc metrics.Service, kind string, next Output) Output { 21 | return metricsMiddleware{ 22 | kind: kind, 23 | metricssvc: metricssvc, 24 | next: next, 25 | } 26 | } 27 | 28 | // Create satisfies slo.Output interface. 29 | func (m metricsMiddleware) Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) (err error) { 30 | defer func(t time.Time) { 31 | m.metricssvc.ObserveOuputCreateDuration(slo, m.kind, t) 32 | if err != nil { 33 | m.metricssvc.IncOuputCreateError(slo, m.kind) 34 | } 35 | }(time.Now()) 36 | return m.next.Create(serviceLevel, slo, result) 37 | } 38 | -------------------------------------------------------------------------------- /pkg/service/output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 5 | "github.com/spotahome/service-level-operator/pkg/log" 6 | "github.com/spotahome/service-level-operator/pkg/service/sli" 7 | ) 8 | 9 | // Output knows how expose/send/create the output of a SLO and SLI result. 10 | type Output interface { 11 | // Create will create the SLI result and the SLO on the specific format. 12 | // It receives the SLI's SLO and it's result. 13 | Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) error 14 | } 15 | 16 | type logger struct { 17 | logger log.Logger 18 | } 19 | 20 | // NewLogger returns a new output logger service that will output the SLOs on 21 | // the specified logger. 22 | func NewLogger(l log.Logger) Output { 23 | return &logger{ 24 | logger: l, 25 | } 26 | } 27 | 28 | // Create will log the result on the console. 29 | func (l *logger) Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) error { 30 | errorRat, err := result.ErrorRatio() 31 | if err != nil { 32 | return err 33 | } 34 | l.logger.With("id", serviceLevel.Name). 35 | With("slo", slo.Name). 36 | With("availability-target", slo.AvailabilityObjectivePercent). 37 | Infof("SLI error ratio: %f", errorRat) 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /pkg/service/output/prometheus.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | 10 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 11 | "github.com/spotahome/service-level-operator/pkg/log" 12 | "github.com/spotahome/service-level-operator/pkg/service/sli" 13 | ) 14 | 15 | const ( 16 | promNS = "service_level" 17 | promSLOSubsystem = "slo" 18 | promSLISubsystem = "sli" 19 | defExpireDuration = 90 * time.Second 20 | ) 21 | 22 | // metricValue is an internal type to store the counters 23 | // of the metrics so when the collector is called it creates 24 | // the metrics based on this values. 25 | type metricValue struct { 26 | serviceLevel *monitoringv1alpha1.ServiceLevel 27 | slo *monitoringv1alpha1.SLO 28 | errorSum float64 29 | countSum float64 30 | objective float64 31 | expire time.Time // expire is the time where this metric will expire unless it's refreshed. 32 | } 33 | 34 | // PrometheusCfg is the configuration of the Prometheus Output. 35 | type PrometheusCfg struct { 36 | // ExpireDuration is the time a metric will expire if is not refreshed. 37 | ExpireDuration time.Duration 38 | } 39 | 40 | // Validate will validate the cfg setting safe defaults. 41 | func (p *PrometheusCfg) Validate() { 42 | if p.ExpireDuration == 0 { 43 | p.ExpireDuration = defExpireDuration 44 | } 45 | } 46 | 47 | // Prometheus knows how to set the output of the SLO on a Prometheus backend. 48 | // The way it works this output is creating two main counters, one that increments 49 | // the error and other that increments the full ratio. 50 | // Example: 51 | // error ratio: 0 + 0 + 0.001 + 0.1 + 0.01 = 0.111 52 | // full ratio: 1 + 1 + 1 + 1 + 1 = 5 53 | // 54 | // You could get the total availability ratio with 1-(0.111/5) = 0.9778 55 | // In other words the availability of all this time is: 97.78% 56 | // 57 | // Under the hood this service is a prometheus collector, it will send to 58 | // prometheus dynamic metrics (because of dynamic labels) when the collect 59 | // process is called. This is made by storing the internal counters and 60 | // generating the metrics when the collect process is callend on each scrape. 61 | type prometheusOutput struct { 62 | cfg PrometheusCfg 63 | metricValuesMu sync.Mutex 64 | metricValues map[string]*metricValue 65 | reg prometheus.Registerer 66 | logger log.Logger 67 | } 68 | 69 | // NewPrometheus returns a new Prometheus output. 70 | func NewPrometheus(cfg PrometheusCfg, reg prometheus.Registerer, logger log.Logger) Output { 71 | cfg.Validate() 72 | 73 | p := &prometheusOutput{ 74 | cfg: cfg, 75 | metricValues: map[string]*metricValue{}, 76 | reg: reg, 77 | logger: logger, 78 | } 79 | 80 | // Autoregister as collector of SLO metrics for prometheus. 81 | p.reg.MustRegister(p) 82 | 83 | return p 84 | } 85 | 86 | // Create satisfies output interface. By setting the correct values on the different 87 | // metrics of the SLO. 88 | func (p *prometheusOutput) Create(serviceLevel *monitoringv1alpha1.ServiceLevel, slo *monitoringv1alpha1.SLO, result *sli.Result) error { 89 | p.metricValuesMu.Lock() 90 | defer p.metricValuesMu.Unlock() 91 | 92 | // Get the current metrics for the SLO. 93 | sloID := fmt.Sprintf("%s-%s-%s", serviceLevel.Namespace, serviceLevel.Name, slo.Name) 94 | if _, ok := p.metricValues[sloID]; !ok { 95 | p.metricValues[sloID] = &metricValue{} 96 | } 97 | 98 | // Add metric values. 99 | errRat, err := result.ErrorRatio() 100 | if err != nil { 101 | return err 102 | } 103 | 104 | // Check it's a possitive number, this shouldn't be necessary but for 105 | // safety we do it. 106 | if errRat < 0 { 107 | errRat = 0 108 | } 109 | 110 | metric := p.metricValues[sloID] 111 | metric.serviceLevel = serviceLevel 112 | metric.slo = slo 113 | metric.errorSum += errRat 114 | metric.countSum++ 115 | // Objective is in % so we convert to ratio (0-1). 116 | metric.objective = slo.AvailabilityObjectivePercent / 100 117 | // Refresh the metric expiration. 118 | metric.expire = time.Now().Add(p.cfg.ExpireDuration) 119 | 120 | return nil 121 | } 122 | 123 | // Describe satisfies prometheus.Collector interface. 124 | func (p *prometheusOutput) Describe(chan<- *prometheus.Desc) {} 125 | 126 | // Collect satisfies prometheus.Collector interface. 127 | func (p *prometheusOutput) Collect(ch chan<- prometheus.Metric) { 128 | p.metricValuesMu.Lock() 129 | defer p.metricValuesMu.Unlock() 130 | p.logger.Debugf("start collecting all service level metrics") 131 | 132 | for id, metric := range p.metricValues { 133 | metric := metric 134 | 135 | // If metric has expired then remove from the map. 136 | if time.Now().After(metric.expire) { 137 | p.logger.With("slo", metric.slo.Name).With("service-level", metric.serviceLevel.Name).Infof("metric expired, removing") 138 | delete(p.metricValues, id) 139 | continue 140 | } 141 | 142 | ns := metric.serviceLevel.Namespace 143 | slName := metric.serviceLevel.Name 144 | sloName := metric.slo.Name 145 | var labels map[string]string 146 | // Check just in case. 147 | if metric.slo.Output.Prometheus != nil && metric.slo.Output.Prometheus.Labels != nil { 148 | labels = metric.slo.Output.Prometheus.Labels 149 | } 150 | 151 | ch <- p.getSLIErrorMetric(ns, slName, sloName, labels, metric.errorSum) 152 | ch <- p.getSLICountMetric(ns, slName, sloName, labels, metric.countSum) 153 | ch <- p.getSLOObjectiveMetric(ns, slName, sloName, labels, metric.objective) 154 | } 155 | 156 | // Collect all SLOs metric. 157 | p.logger.Debugf("finished collecting all the service level metrics") 158 | } 159 | 160 | func (p *prometheusOutput) getSLIErrorMetric(ns, serviceLevel, slo string, constLabels prometheus.Labels, value float64) prometheus.Metric { 161 | return prometheus.MustNewConstMetric( 162 | prometheus.NewDesc( 163 | prometheus.BuildFQName(promNS, promSLISubsystem, "result_error_ratio_total"), 164 | "Is the error or failure ratio of an SLI result.", 165 | []string{"namespace", "service_level", "slo"}, 166 | constLabels, 167 | ), 168 | prometheus.CounterValue, 169 | value, 170 | ns, serviceLevel, slo, 171 | ) 172 | } 173 | 174 | func (p *prometheusOutput) getSLICountMetric(ns, serviceLevel, slo string, constLabels prometheus.Labels, value float64) prometheus.Metric { 175 | return prometheus.MustNewConstMetric( 176 | prometheus.NewDesc( 177 | prometheus.BuildFQName(promNS, promSLISubsystem, "result_count_total"), 178 | "Is the number of times an SLI result has been processed.", 179 | []string{"namespace", "service_level", "slo"}, 180 | constLabels, 181 | ), 182 | prometheus.CounterValue, 183 | value, 184 | ns, serviceLevel, slo, 185 | ) 186 | } 187 | 188 | func (p *prometheusOutput) getSLOObjectiveMetric(ns, serviceLevel, slo string, constLabels prometheus.Labels, value float64) prometheus.Metric { 189 | return prometheus.MustNewConstMetric( 190 | prometheus.NewDesc( 191 | prometheus.BuildFQName(promNS, promSLOSubsystem, "objective_ratio"), 192 | "Is the objective of the SLO in ratio unit.", 193 | []string{"namespace", "service_level", "slo"}, 194 | constLabels, 195 | ), 196 | prometheus.GaugeValue, 197 | value, 198 | ns, serviceLevel, slo, 199 | ) 200 | } 201 | -------------------------------------------------------------------------------- /pkg/service/output/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package output_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | "github.com/stretchr/testify/assert" 12 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 13 | 14 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 15 | "github.com/spotahome/service-level-operator/pkg/log" 16 | "github.com/spotahome/service-level-operator/pkg/service/output" 17 | "github.com/spotahome/service-level-operator/pkg/service/sli" 18 | ) 19 | 20 | var ( 21 | sl0 = &monitoringv1alpha1.ServiceLevel{ 22 | ObjectMeta: metav1.ObjectMeta{ 23 | Name: "sl0-test", 24 | Namespace: "ns0", 25 | }, 26 | } 27 | sl1 = &monitoringv1alpha1.ServiceLevel{ 28 | ObjectMeta: metav1.ObjectMeta{ 29 | Name: "sl1-test", 30 | Namespace: "ns1", 31 | }, 32 | } 33 | slo00 = &monitoringv1alpha1.SLO{ 34 | Name: "slo00-test", 35 | AvailabilityObjectivePercent: 99.999, 36 | Output: monitoringv1alpha1.Output{ 37 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 38 | }, 39 | } 40 | slo01 = &monitoringv1alpha1.SLO{ 41 | Name: "slo01-test", 42 | AvailabilityObjectivePercent: 99.98, 43 | Output: monitoringv1alpha1.Output{ 44 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 45 | }, 46 | } 47 | slo10 = &monitoringv1alpha1.SLO{ 48 | Name: "slo10-test", 49 | AvailabilityObjectivePercent: 99.99978, 50 | Output: monitoringv1alpha1.Output{ 51 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{}, 52 | }, 53 | } 54 | slo11 = &monitoringv1alpha1.SLO{ 55 | Name: "slo11-test", 56 | AvailabilityObjectivePercent: 95.9981, 57 | Output: monitoringv1alpha1.Output{ 58 | Prometheus: &monitoringv1alpha1.PrometheusOutputSource{ 59 | Labels: map[string]string{ 60 | "env": "test", 61 | "team": "team1", 62 | }, 63 | }, 64 | }, 65 | } 66 | ) 67 | 68 | func TestPrometheusOutput(t *testing.T) { 69 | tests := []struct { 70 | name string 71 | cfg output.PrometheusCfg 72 | createResults func(output output.Output) 73 | expMetrics []string 74 | expMissingMetrics []string 75 | }{ 76 | { 77 | name: "Creating a output result should expose all the required metrics", 78 | createResults: func(output output.Output) { 79 | output.Create(sl0, slo00, &sli.Result{ 80 | TotalQ: 1000000, 81 | ErrorQ: 122, 82 | }) 83 | }, 84 | expMetrics: []string{ 85 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.000122`, 86 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 1`, 87 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 88 | }, 89 | }, 90 | { 91 | name: "Expired metrics shouldn't be exposed", 92 | cfg: output.PrometheusCfg{ 93 | ExpireDuration: 500 * time.Microsecond, 94 | }, 95 | createResults: func(output output.Output) { 96 | output.Create(sl0, slo00, &sli.Result{ 97 | TotalQ: 1000000, 98 | ErrorQ: 122, 99 | }) 100 | time.Sleep(1 * time.Millisecond) 101 | }, 102 | expMissingMetrics: []string{ 103 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.000122`, 104 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 1`, 105 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 106 | }, 107 | }, 108 | { 109 | name: "Creating a output result should expose all the required metrics (multiple adds on same SLO).", 110 | createResults: func(output output.Output) { 111 | slis := []*sli.Result{ 112 | &sli.Result{TotalQ: 1000000, ErrorQ: 122}, 113 | &sli.Result{TotalQ: 999, ErrorQ: 1}, 114 | &sli.Result{TotalQ: 812392, ErrorQ: 94}, 115 | &sli.Result{TotalQ: 83, ErrorQ: 83}, 116 | &sli.Result{TotalQ: 11223, ErrorQ: 11222}, 117 | &sli.Result{TotalQ: 9999999999, ErrorQ: 2}, 118 | &sli.Result{TotalQ: 1245, ErrorQ: 0}, 119 | &sli.Result{TotalQ: 9019, ErrorQ: 1001}, 120 | } 121 | for _, sli := range slis { 122 | output.Create(sl0, slo00, sli) 123 | } 124 | }, 125 | expMetrics: []string{ 126 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 2.112137520556389`, 127 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 8`, 128 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 129 | }, 130 | }, 131 | { 132 | name: "Creating a output result should expose all the required metrics (multiple SLOs).", 133 | createResults: func(output output.Output) { 134 | output.Create(sl0, slo00, &sli.Result{ 135 | TotalQ: 1000000, 136 | ErrorQ: 122, 137 | }) 138 | output.Create(sl0, slo01, &sli.Result{ 139 | TotalQ: 1011, 140 | ErrorQ: 340, 141 | }) 142 | output.Create(sl1, slo10, &sli.Result{ 143 | TotalQ: 9212, 144 | ErrorQ: 1, 145 | }) 146 | output.Create(sl1, slo10, &sli.Result{ 147 | TotalQ: 3456, 148 | ErrorQ: 3, 149 | }) 150 | output.Create(sl1, slo11, &sli.Result{ 151 | TotalQ: 998, 152 | ErrorQ: 7, 153 | }) 154 | }, 155 | expMetrics: []string{ 156 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.000122`, 157 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 1`, 158 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo00-test"} 0.9999899999999999`, 159 | 160 | `service_level_sli_result_error_ratio_total{namespace="ns0",service_level="sl0-test",slo="slo01-test"} 0.3363006923837784`, 161 | `service_level_sli_result_count_total{namespace="ns0",service_level="sl0-test",slo="slo01-test"} 1`, 162 | `service_level_slo_objective_ratio{namespace="ns0",service_level="sl0-test",slo="slo01-test"} 0.9998`, 163 | 164 | `service_level_sli_result_error_ratio_total{namespace="ns1",service_level="sl1-test",slo="slo10-test"} 0.0009766096154773965`, 165 | `service_level_sli_result_count_total{namespace="ns1",service_level="sl1-test",slo="slo10-test"} 2`, 166 | `service_level_slo_objective_ratio{namespace="ns1",service_level="sl1-test",slo="slo10-test"} 0.9999978`, 167 | 168 | `service_level_sli_result_error_ratio_total{env="test",namespace="ns1",service_level="sl1-test",slo="slo11-test",team="team1"} 0.0070140280561122245`, 169 | `service_level_sli_result_count_total{env="test",namespace="ns1",service_level="sl1-test",slo="slo11-test",team="team1"} 1`, 170 | `service_level_slo_objective_ratio{env="test",namespace="ns1",service_level="sl1-test",slo="slo11-test",team="team1"} 0.959981`, 171 | }, 172 | }, 173 | } 174 | 175 | for _, test := range tests { 176 | t.Run(test.name, func(t *testing.T) { 177 | assert := assert.New(t) 178 | promReg := prometheus.NewRegistry() 179 | 180 | output := output.NewPrometheus(test.cfg, promReg, log.Dummy) 181 | test.createResults(output) 182 | 183 | // Check metrics 184 | h := promhttp.HandlerFor(promReg, promhttp.HandlerOpts{}) 185 | w := httptest.NewRecorder() 186 | req := httptest.NewRequest("GET", "/metrics", nil) 187 | h.ServeHTTP(w, req) 188 | 189 | metrics, _ := ioutil.ReadAll(w.Result().Body) 190 | for _, expMetric := range test.expMetrics { 191 | assert.Contains(string(metrics), expMetric) 192 | } 193 | for _, expMissingMetric := range test.expMissingMetrics { 194 | assert.NotContains(string(metrics), expMissingMetric) 195 | } 196 | }) 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /pkg/service/sli/factory.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "errors" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // RetrieverFactory is a factory that knows how to get the correct 10 | // Retriever based on the SLI source. 11 | type RetrieverFactory interface { 12 | // GetRetriever returns a retriever based on the SLI source. 13 | GetStrategy(*monitoringv1alpha1.SLI) (Retriever, error) 14 | } 15 | 16 | // retrieverFactory doesn't create objects per se, it only knows 17 | // what strategy to return based on the passed SLI. 18 | type retrieverFactory struct { 19 | promRetriever Retriever 20 | } 21 | 22 | // NewRetrieverFactory returns a new retriever factory. 23 | func NewRetrieverFactory(promRetriever Retriever) RetrieverFactory { 24 | return &retrieverFactory{ 25 | promRetriever: promRetriever, 26 | } 27 | } 28 | 29 | // GetRetriever satsifies RetrieverFactory interface. 30 | func (r retrieverFactory) GetStrategy(s *monitoringv1alpha1.SLI) (Retriever, error) { 31 | if s.Prometheus != nil { 32 | return r.promRetriever, nil 33 | } 34 | 35 | return nil, errors.New("unsupported retriever kind") 36 | } 37 | 38 | // MockRetrieverFactory returns the mocked retriever strategy. 39 | type MockRetrieverFactory struct { 40 | Mock Retriever 41 | } 42 | 43 | // GetStrategy satisfies RetrieverFactory interface. 44 | func (m MockRetrieverFactory) GetStrategy(_ *monitoringv1alpha1.SLI) (Retriever, error) { 45 | return m.Mock, nil 46 | } 47 | -------------------------------------------------------------------------------- /pkg/service/sli/middleware.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "time" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | "github.com/spotahome/service-level-operator/pkg/service/metrics" 8 | ) 9 | 10 | // metricsMiddleware will monitoring the calls to the SLI Retriever. 11 | type metricsMiddleware struct { 12 | kind string 13 | metricssvc metrics.Service 14 | next Retriever 15 | } 16 | 17 | // NewMetricsMiddleware returns a new metrics middleware that wraps a Retriever SLI 18 | // service and monitorings with metrics. 19 | func NewMetricsMiddleware(metricssvc metrics.Service, kind string, next Retriever) Retriever { 20 | return metricsMiddleware{ 21 | kind: kind, 22 | metricssvc: metricssvc, 23 | next: next, 24 | } 25 | } 26 | 27 | // Retrieve satisfies sli.Retriever interface. 28 | func (m metricsMiddleware) Retrieve(sli *monitoringv1alpha1.SLI) (result Result, err error) { 29 | defer func(t time.Time) { 30 | m.metricssvc.ObserveSLIRetrieveDuration(sli, m.kind, t) 31 | if err != nil { 32 | m.metricssvc.IncSLIRetrieveError(sli, m.kind) 33 | } 34 | }(time.Now()) 35 | return m.next.Retrieve(sli) 36 | } 37 | -------------------------------------------------------------------------------- /pkg/service/sli/prometheus.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "golang.org/x/sync/errgroup" 9 | 10 | promv1 "github.com/prometheus/client_golang/api/prometheus/v1" 11 | "github.com/prometheus/common/model" 12 | 13 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 14 | "github.com/spotahome/service-level-operator/pkg/log" 15 | promcli "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 16 | ) 17 | 18 | const promCliTimeout = 2 * time.Second 19 | 20 | // prometheus knows how to get SLIs from a prometheus backend. 21 | type prometheus struct { 22 | cliFactory promcli.ClientFactory 23 | logger log.Logger 24 | } 25 | 26 | // NewPrometheus returns a new prometheus SLI service. 27 | func NewPrometheus(promCliFactory promcli.ClientFactory, logger log.Logger) Retriever { 28 | return &prometheus{ 29 | cliFactory: promCliFactory, 30 | logger: logger, 31 | } 32 | } 33 | 34 | // Retrieve satisfies Service interface.. 35 | func (p *prometheus) Retrieve(sli *monitoringv1alpha1.SLI) (Result, error) { 36 | cli, err := p.cliFactory.GetV1APIClient(sli.Prometheus.Address) 37 | if err != nil { 38 | return Result{}, err 39 | } 40 | 41 | // Get both metrics. 42 | res := Result{} 43 | 44 | promclictx, cancel := context.WithTimeout(context.Background(), promCliTimeout) 45 | defer cancel() 46 | 47 | // Make queries concurrently. 48 | g, ctx := errgroup.WithContext(promclictx) 49 | g.Go(func() error { 50 | res.TotalQ, err = p.getVectorMetric(ctx, cli, sli.Prometheus.TotalQuery) 51 | return err 52 | }) 53 | g.Go(func() error { 54 | res.ErrorQ, err = p.getVectorMetric(ctx, cli, sli.Prometheus.ErrorQuery) 55 | return err 56 | }) 57 | 58 | // Wait for the first error or until all of them have finished. 59 | err = g.Wait() 60 | if err != nil { 61 | return Result{}, err 62 | } 63 | 64 | return res, nil 65 | } 66 | 67 | func (p *prometheus) getVectorMetric(ctx context.Context, cli promv1.API, query string) (float64, error) { 68 | // Make the query. 69 | val, _, err := cli.Query(ctx, query, time.Now()) 70 | if err != nil { 71 | return 0, err 72 | } 73 | 74 | if val == nil { 75 | return 0, fmt.Errorf("nil value received from prometheus") 76 | } 77 | 78 | // Only vectors are valid metrics. 79 | if val.Type() != model.ValVector { 80 | return 0, fmt.Errorf("received metric needs to be a vector, received: %s", val.Type()) 81 | } 82 | mtr := val.(model.Vector) 83 | 84 | // If we obtain no metric then for us is 0. 85 | if len(mtr) == 0 { 86 | return 0, nil 87 | } 88 | 89 | // More than one metric should be an error. 90 | if len(mtr) != 1 { 91 | return 0, fmt.Errorf("wrong samples length, should not be more than 1, got: %d", len(mtr)) 92 | } 93 | 94 | return float64(mtr[0].Value), nil 95 | } 96 | -------------------------------------------------------------------------------- /pkg/service/sli/prometheus_test.go: -------------------------------------------------------------------------------- 1 | package sli_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/prometheus/common/model" 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/mock" 10 | 11 | mpromv1 "github.com/spotahome/service-level-operator/mocks/github.com/prometheus/client_golang/api/prometheus/v1" 12 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 13 | "github.com/spotahome/service-level-operator/pkg/log" 14 | prometheusvc "github.com/spotahome/service-level-operator/pkg/service/client/prometheus" 15 | "github.com/spotahome/service-level-operator/pkg/service/sli" 16 | ) 17 | 18 | func TestPrometheusRetrieve(t *testing.T) { 19 | sli0 := &monitoringv1alpha1.SLI{ 20 | SLISource: monitoringv1alpha1.SLISource{ 21 | Prometheus: &monitoringv1alpha1.PrometheusSLISource{ 22 | TotalQuery: "test_total_query", 23 | ErrorQuery: "test_error_query", 24 | }, 25 | }, 26 | } 27 | vector2 := model.Vector{ 28 | &model.Sample{ 29 | Metric: model.Metric{}, 30 | Value: model.SampleValue(2), 31 | }, 32 | } 33 | vector100 := model.Vector{ 34 | &model.Sample{ 35 | Metric: model.Metric{}, 36 | Value: model.SampleValue(100), 37 | }, 38 | } 39 | 40 | tests := []struct { 41 | name string 42 | sli *monitoringv1alpha1.SLI 43 | 44 | totalQueryResult model.Value 45 | totalQueryErr error 46 | errorQueryResult model.Value 47 | errorQueryErr error 48 | 49 | expResult sli.Result 50 | expErr bool 51 | }{ 52 | { 53 | name: "If no result from prometheus it should fail.", 54 | sli: sli0, 55 | expErr: true, 56 | }, 57 | { 58 | name: "Failing total query should make the retrieval fail.", 59 | sli: sli0, 60 | totalQueryResult: vector100, 61 | totalQueryErr: errors.New("wanted error"), 62 | errorQueryResult: vector2, 63 | expErr: true, 64 | }, 65 | { 66 | name: "Failing error query should make the retrieval fail.", 67 | sli: sli0, 68 | totalQueryResult: vector100, 69 | errorQueryResult: vector2, 70 | errorQueryErr: errors.New("wanted error"), 71 | expErr: true, 72 | }, 73 | { 74 | name: "If the query doesn't return a vector it should fail.", 75 | sli: sli0, 76 | totalQueryResult: &model.Scalar{ 77 | Value: model.SampleValue(2), 78 | }, 79 | errorQueryResult: vector2, 80 | expErr: true, 81 | }, 82 | { 83 | name: "If the query returns more than one metric it should fail.", 84 | sli: sli0, 85 | totalQueryResult: model.Vector{ 86 | &model.Sample{ 87 | Value: model.SampleValue(1), 88 | }, 89 | &model.Sample{ 90 | Value: model.SampleValue(2), 91 | }, 92 | }, 93 | errorQueryResult: vector2, 94 | expErr: true, 95 | }, 96 | { 97 | name: "If the query returns 0 metrics it should treat as a 0 value.", 98 | sli: sli0, 99 | totalQueryResult: vector2, 100 | errorQueryResult: model.Vector{}, 101 | expErr: false, 102 | expResult: sli.Result{ 103 | TotalQ: 2, 104 | ErrorQ: 0, 105 | }, 106 | }, 107 | { 108 | name: "Quering prometheus for total and error metrics should return a correct result", 109 | sli: sli0, 110 | totalQueryResult: vector100, 111 | errorQueryResult: vector2, 112 | expResult: sli.Result{ 113 | TotalQ: 100, 114 | ErrorQ: 2, 115 | }, 116 | }, 117 | } 118 | 119 | for _, test := range tests { 120 | t.Run(test.name, func(t *testing.T) { 121 | assert := assert.New(t) 122 | 123 | // Mocks. 124 | mapi := &mpromv1.API{} 125 | mpromfactory := &prometheusvc.MockFactory{Cli: mapi} 126 | mapi.On("Query", mock.Anything, test.sli.Prometheus.TotalQuery, mock.Anything).Return(test.totalQueryResult, nil, test.errorQueryErr) 127 | mapi.On("Query", mock.Anything, test.sli.Prometheus.ErrorQuery, mock.Anything).Return(test.errorQueryResult, nil, test.totalQueryErr) 128 | 129 | retriever := sli.NewPrometheus(mpromfactory, log.Dummy) 130 | res, err := retriever.Retrieve(test.sli) 131 | 132 | if test.expErr { 133 | assert.Error(err) 134 | } else if assert.NoError(err) { 135 | assert.Equal(test.expResult, res) 136 | } 137 | }) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /pkg/service/sli/sli.go: -------------------------------------------------------------------------------- 1 | package sli 2 | 3 | import ( 4 | "fmt" 5 | 6 | monitoringv1alpha1 "github.com/spotahome/service-level-operator/pkg/apis/monitoring/v1alpha1" 7 | ) 8 | 9 | // Result is the result of getting a SLI from a backend. 10 | type Result struct { 11 | // TotalQ is the result of applying the total query. 12 | TotalQ float64 13 | // ErrorQ is the result of applying the error query. 14 | ErrorQ float64 15 | } 16 | 17 | // AvailabilityRatio returns the availability of an SLI result in 18 | // ratio unit (0-1). 19 | func (r *Result) AvailabilityRatio() (float64, error) { 20 | if r.TotalQ < r.ErrorQ { 21 | return 0, fmt.Errorf("%f can't be higher than %f", r.ErrorQ, r.TotalQ) 22 | } 23 | 24 | // If no total then everything ok. 25 | if r.TotalQ <= 0 { 26 | return 1, nil 27 | } 28 | 29 | eRat, err := r.ErrorRatio() 30 | if err != nil { 31 | return 0, err 32 | } 33 | 34 | return 1 - eRat, nil 35 | } 36 | 37 | // ErrorRatio returns the error of an SLI result in. 38 | // ratio unit (0-1). 39 | func (r *Result) ErrorRatio() (float64, error) { 40 | if r.TotalQ < r.ErrorQ { 41 | return 0, fmt.Errorf("%f can't be higher than %f", r.ErrorQ, r.TotalQ) 42 | } 43 | 44 | // If no total then everything ok. 45 | if r.TotalQ <= 0 { 46 | return 0, nil 47 | } 48 | 49 | return r.ErrorQ / r.TotalQ, nil 50 | } 51 | 52 | // Retriever knows how to get SLIs from different backends. 53 | type Retriever interface { 54 | // Retrieve returns the result of a SLI retrieved from the implemented backend. 55 | Retrieve(*monitoringv1alpha1.SLI) (Result, error) 56 | } 57 | -------------------------------------------------------------------------------- /pkg/service/sli/sli_test.go: -------------------------------------------------------------------------------- 1 | package sli_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/spotahome/service-level-operator/pkg/service/sli" 9 | ) 10 | 11 | func TestSLIResult(t *testing.T) { 12 | tests := []struct { 13 | name string 14 | errorQ float64 15 | totalQ float64 16 | expAvailability float64 17 | expAvailabiityErr bool 18 | expError float64 19 | expErrorErr bool 20 | }{ 21 | { 22 | name: "Not having a total quantity should return everything ok.", 23 | expAvailability: 1, 24 | expError: 0, 25 | }, 26 | { 27 | name: "Having more errors than total should be impossible.", 28 | errorQ: 600, 29 | totalQ: 300, 30 | expErrorErr: true, 31 | expAvailabiityErr: true, 32 | }, 33 | { 34 | name: "If half of the total are errors then the ratio of availability and error should be 0.5.", 35 | errorQ: 300, 36 | totalQ: 600, 37 | expAvailability: 0.5, 38 | expError: 0.5, 39 | }, 40 | { 41 | name: "If a 33% of errors then the ratios should be 0.33 and 0.66.", 42 | errorQ: 33, 43 | totalQ: 100, 44 | expAvailability: 0.6699999999999999, 45 | expError: 0.33, 46 | }, 47 | { 48 | name: "In small quantities the ratios should be correctly calculated.", 49 | errorQ: 4, 50 | totalQ: 10, 51 | expAvailability: 0.6, 52 | expError: 0.4, 53 | }, 54 | { 55 | name: "In big quantities the ratios should be correctly calculated.", 56 | errorQ: 240, 57 | totalQ: 10000000, 58 | expAvailability: 0.999976, 59 | expError: 0.000024, 60 | }, 61 | } 62 | 63 | for _, test := range tests { 64 | t.Run(test.name, func(t *testing.T) { 65 | assert := assert.New(t) 66 | 67 | res := sli.Result{ 68 | TotalQ: test.totalQ, 69 | ErrorQ: test.errorQ, 70 | } 71 | 72 | av, err := res.AvailabilityRatio() 73 | if test.expAvailabiityErr { 74 | assert.Error(err) 75 | } else if assert.NoError(err) { 76 | assert.Equal(test.expAvailability, av) 77 | } 78 | 79 | dw, err := res.ErrorRatio() 80 | if test.expErrorErr { 81 | assert.Error(err) 82 | } else if assert.NoError(err) { 83 | assert.Equal(test.expError, dw) 84 | } 85 | }) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/manual/slos.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: monitoring.spotahome.com/v1alpha1 2 | kind: ServiceLevel 3 | metadata: 4 | name: awesome-service 5 | spec: 6 | serviceLevelObjectives: 7 | # A typical 5xx request SLO. 8 | - name: "9999_http_request_lt_500" 9 | description: 99.99% of requests must be served with <500 status code. 10 | disable: false 11 | availabilityObjectivePercent: 99.99 12 | serviceLevelIndicator: 13 | prometheus: 14 | address: http://127.0.0.1:9091 15 | totalQuery: | 16 | sum( 17 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 18 | errorQuery: | 19 | sum( 20 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com", code=~"5.."}[2m])) 21 | output: 22 | prometheus: {} 23 | 24 | # Latency example. 25 | - name: "90_http_request_latency_lt_250ms" 26 | description: 90% of requests must be served with in less than 250ms. 27 | disable: false 28 | availabilityObjectivePercent: 90 29 | serviceLevelIndicator: 30 | prometheus: 31 | address: http://127.0.0.1:9091 32 | totalQuery: | 33 | sum( 34 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 35 | errorQuery: | 36 | sum( 37 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 38 | - 39 | sum( 40 | increase(skipper_serve_host_duration_seconds_bucket{host="www_spotahome_com", le="0.25"}[2m])) 41 | output: 42 | prometheus: {} 43 | 44 | # Same latency example as previous one but with different latency bucket. Is normal to have 45 | # multiple SLOs on different latency limits. 46 | - name: "99_http_request_latency_lt_500ms" 47 | description: 99% of requests must be served with in less than 500ms. 48 | disable: false 49 | availabilityObjectivePercent: 99 50 | serviceLevelIndicator: 51 | prometheus: 52 | address: http://127.0.0.1:9091 53 | totalQuery: | 54 | sum( 55 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 56 | errorQuery: | 57 | sum( 58 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 59 | - 60 | sum( 61 | increase(skipper_serve_host_duration_seconds_bucket{host="www_spotahome_com", le="0.5"}[2m])) 62 | output: 63 | prometheus: {} 64 | 65 | # Same as previous one except the availability objective. This way we can see that 66 | # with the same SLIs our SLO changes drastically. 67 | - name: "90_http_request_latency_lt_500ms" 68 | description: 90% of requests must be served with in less than 500ms. 69 | disable: false 70 | availabilityObjectivePercent: 90 71 | serviceLevelIndicator: 72 | prometheus: 73 | address: http://127.0.0.1:9091 74 | totalQuery: | 75 | sum( 76 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 77 | errorQuery: | 78 | sum( 79 | increase(skipper_serve_host_duration_seconds_count{host="www_spotahome_com"}[2m])) 80 | - 81 | sum( 82 | increase(skipper_serve_host_duration_seconds_bucket{host="www_spotahome_com", le="0.5"}[2m])) 83 | output: 84 | prometheus: 85 | labels: 86 | env: production 87 | team: a-team 88 | --------------------------------------------------------------------------------