├── .buildkite └── pipeline.yml ├── .dockerignore ├── .gitignore ├── Dockerfile ├── Dockerfile.ci ├── LICENSE ├── Makefile ├── PROJECT ├── README.md ├── api └── v1alpha1 │ ├── cortex_types.go │ ├── cortex_webhook.go │ ├── cortex_webhook_test.go │ ├── groupversion_info.go │ ├── webhook_suite_test.go │ └── zz_generated.deepcopy.go ├── config ├── certmanager │ ├── certificate.yaml │ ├── kustomization.yaml │ └── kustomizeconfig.yaml ├── crd │ ├── bases │ │ └── cortex.opstrace.io_cortices.yaml │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ └── patches │ │ ├── cainjection_in_cortices.yaml │ │ └── webhook_in_cortices.yaml ├── default │ ├── kustomization.yaml │ ├── manager_auth_proxy_patch.yaml │ ├── manager_config_patch.yaml │ ├── manager_webhook_patch.yaml │ └── webhookcainjection_patch.yaml ├── manager │ ├── controller_manager_config.yaml │ ├── kustomization.yaml │ └── manager.yaml ├── prometheus │ ├── kustomization.yaml │ └── monitor.yaml ├── rbac │ ├── auth_proxy_client_clusterrole.yaml │ ├── auth_proxy_role.yaml │ ├── auth_proxy_role_binding.yaml │ ├── auth_proxy_service.yaml │ ├── cortex_editor_role.yaml │ ├── cortex_viewer_role.yaml │ ├── kustomization.yaml │ ├── leader_election_role.yaml │ ├── leader_election_role_binding.yaml │ ├── role.yaml │ ├── role_binding.yaml │ └── service_account.yaml ├── samples │ └── cortex_v1alpha1_cortex.yaml └── webhook │ ├── kustomization.yaml │ ├── kustomizeconfig.yaml │ ├── manifests.v1beta1.yaml │ └── service.yaml ├── controllers ├── cortex_alertmanager_controller.go ├── cortex_compactor_controller.go ├── cortex_controller.go ├── cortex_controller_test.go ├── cortex_default_config.go ├── cortex_distributor_controller.go ├── cortex_ingester_controller.go ├── cortex_querier_controller.go ├── cortex_query_frontend_controller.go ├── cortex_ruler_controller.go ├── cortex_runtime_config_controller.go ├── cortex_store_gateway_controller.go ├── helpers.go ├── kubernetes_resource.go ├── memcached_controller.go └── suite_test.go ├── docs ├── guides │ └── terraform │ │ ├── README.md │ │ ├── eks.tf │ │ ├── example.tfvars │ │ ├── iam.tf │ │ ├── outputs.tf │ │ ├── s3.tf │ │ ├── variables.tf │ │ ├── version.tf │ │ └── vpc.tf └── samples │ └── grafana-example-manifest.yaml ├── go.mod ├── go.sum ├── hack └── boilerplate.go.txt └── main.go /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | env: 2 | CHECKOUT_VERSION_STRING: "${BUILDKITE_COMMIT:0:8}-ci" 3 | steps: 4 | - label: "preamble" 5 | key: "preamble" 6 | command: 7 | - make build-ci-image 8 | - label: "lint, build" 9 | key: "build" 10 | depends_on: 11 | - "preamble" 12 | command: 13 | - make ci-build 14 | - label: "test" 15 | key: "test" 16 | depends_on: 17 | - "build" 18 | command: 19 | - make ci-test 20 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file 2 | # Ignore all files which are not go type 3 | !**/*.go 4 | !**/*.mod 5 | !**/*.sum 6 | testbin/ 7 | docs/ 8 | bin/ 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | testbin/* 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Kubernetes Generated files - skip generated files, except for vendored files 18 | 19 | !vendor/**/zz_generated.* 20 | 21 | # editor and IDE paraphernalia 22 | .idea 23 | *.swp 24 | *.swo 25 | *~ 26 | 27 | # terraform files 28 | docs/guides/terraform/.terraform.* 29 | docs/guides/terraform/.terraform/ 30 | docs/guides/terraform/terraform.tfstate 31 | docs/guides/terraform/terraform.tfstate.backup 32 | docs/guides/terraform/kubeconfig 33 | 34 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Build the manager binary 2 | FROM golang:1.15 as builder 3 | 4 | WORKDIR /workspace 5 | # Copy the Go Modules manifests 6 | COPY go.mod go.mod 7 | COPY go.sum go.sum 8 | # cache deps before building and copying source so that we don't need to re-download as much 9 | # and so that source changes don't invalidate our downloaded layer 10 | RUN go mod download 11 | 12 | # Copy the go source 13 | COPY main.go main.go 14 | COPY api/ api/ 15 | COPY controllers/ controllers/ 16 | 17 | # Build 18 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go 19 | 20 | # Use distroless as minimal base image to package the manager binary 21 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 22 | FROM gcr.io/distroless/static:nonroot 23 | WORKDIR / 24 | COPY --from=builder /workspace/manager . 25 | USER 65532:65532 26 | 27 | ENTRYPOINT ["/manager"] 28 | -------------------------------------------------------------------------------- /Dockerfile.ci: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 as builder 2 | 3 | WORKDIR /workspace 4 | # Copy the Go Modules manifests 5 | COPY go.mod go.mod 6 | COPY go.sum go.sum 7 | # cache deps before building and copying source so that we don't need to 8 | # re-download as much and so that source changes don't invalidate our downloaded 9 | # layer 10 | RUN go mod download 11 | 12 | # Install tools required to run tests 13 | COPY hack/ hack/ 14 | COPY Makefile Makefile 15 | 16 | ENV CI=true 17 | 18 | RUN mkdir -p /local/bin && make controller-gen kustomize envtest 19 | 20 | # Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable. 21 | ENV SKIP_FETCH_TOOLS=true 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | # Image URL to use all building/pushing image targets 3 | IMG ?= opstrace/cortex-operator:latest 4 | # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) 5 | CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" 6 | 7 | # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) 8 | ifeq (,$(shell go env GOBIN)) 9 | GOBIN=$(shell go env GOPATH)/bin 10 | else 11 | GOBIN=$(shell go env GOBIN) 12 | endif 13 | 14 | # Setting SHELL to bash allows bash commands to be executed by recipes. 15 | # This is a requirement for 'setup-envtest.sh' in the test target. 16 | # Options are set to exit when a recipe line exits non-zero or a piped command fails. 17 | SHELL = /usr/bin/env bash -o pipefail 18 | .SHELLFLAGS = -ec 19 | 20 | ifeq ($(CI),true) 21 | BASEDIR=/local 22 | else 23 | BASEDIR=$(shell pwd) 24 | endif 25 | 26 | all: build 27 | 28 | ##@ General 29 | 30 | # The help target prints out all targets with their descriptions organized 31 | # beneath their categories. The categories are represented by '##@' and the 32 | # target descriptions by '##'. The awk commands is responsible for reading the 33 | # entire set of makefiles included in this invocation, looking for lines of the 34 | # file as xyz: ## something, and then pretty-format the target and help. Then, 35 | # if there's a line with ##@ something, that gets pretty-printed as a category. 36 | # More info on the usage of ANSI control characters for terminal formatting: 37 | # https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters 38 | # More info on the awk command: 39 | # http://linuxcommand.org/lc3_adv_awk.php 40 | 41 | help: ## Display this help. 42 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 43 | 44 | ##@ Development 45 | 46 | manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. 47 | $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases 48 | 49 | generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. 50 | $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." 51 | 52 | fmt: ## Run go fmt against code. 53 | go fmt ./... 54 | 55 | vet: ## Run go vet against code. 56 | go vet ./... 57 | 58 | ENVTEST_ASSETS_DIR=$(BASEDIR)/testbin 59 | test: envtest manifests generate fmt vet ## Run tests. 60 | source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out 61 | 62 | envtest: 63 | mkdir -p ${ENVTEST_ASSETS_DIR} 64 | test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh 65 | source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); 66 | 67 | ##@ Build 68 | 69 | build: generate fmt vet ## Build manager binary. 70 | go build -o bin/manager main.go 71 | 72 | run: manifests generate fmt vet ## Run a controller from your host. 73 | go run ./main.go 74 | 75 | docker-build: ## Build docker image with the manager. 76 | docker build -t ${IMG} . 77 | 78 | docker-push: ## Push docker image with the manager. 79 | docker push ${IMG} 80 | 81 | ##@ Deployment 82 | 83 | install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. 84 | $(KUSTOMIZE) build config/crd | kubectl apply -f - 85 | 86 | uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. 87 | $(KUSTOMIZE) build config/crd | kubectl delete -f - 88 | 89 | deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. 90 | cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} 91 | $(KUSTOMIZE) build config/default | kubectl apply -f - 92 | 93 | undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. 94 | $(KUSTOMIZE) build config/default | kubectl delete -f - 95 | 96 | cert-manager: ## Install cert-manager into the K8s cluster specified in ~/.kube/config. 97 | kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.3.1/cert-manager.yaml 98 | 99 | CONTROLLER_GEN = $(BASEDIR)/bin/controller-gen 100 | controller-gen: ## Download controller-gen locally if necessary. 101 | $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.1) 102 | 103 | KUSTOMIZE = $(BASEDIR)/bin/kustomize 104 | kustomize: ## Download kustomize locally if necessary. 105 | $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) 106 | 107 | ##@ Kind Deployment 108 | kind: ## Create a kind cluster and install cert-manager. 109 | kind create cluster 110 | $(MAKE) cert-manager 111 | 112 | kind-deploy: manifests kustomize docker-build kind-load-image deploy ## Deploy to kind cluster. 113 | 114 | kind-undeploy: undeploy ## Undeploy from kind cluster. 115 | 116 | kind-load-image: ## Load docker image to kind cluster. 117 | kind load docker-image ${IMG} 118 | 119 | # go-get-tool will 'go get' any package $2 and install it to $1. 120 | define go-get-tool 121 | @[ -f $(1) ] || { \ 122 | set -e ;\ 123 | TMP_DIR=$$(mktemp -d) ;\ 124 | cd $$TMP_DIR ;\ 125 | go mod init tmp ;\ 126 | echo "Downloading $(2)" ;\ 127 | GOBIN=$(BASEDIR)/bin go get $(2) ;\ 128 | rm -rf $$TMP_DIR ;\ 129 | } 130 | endef 131 | 132 | ##@ CI 133 | 134 | OS := $(shell uname -s) 135 | 136 | # 137 | # CI specific targets are only tested to run on Linux 138 | # 139 | ifeq ($(OS),Linux) 140 | 141 | CI_IMG_TAG ?= $(shell sha1sum Dockerfile.ci | awk '{ print $$1 }') 142 | CI_IMG ?= opstrace/cortex-operator-ci:$(CI_IMG_TAG) 143 | 144 | build-ci-image: ## Build docker image to run in CI. 145 | DOCKER_BUILDKIT=1 docker build --ssh default -f Dockerfile.ci --tag $(CI_IMG) . 146 | 147 | pull-ci-image: ## Pull CI docker Image. 148 | docker pull $(CI_IMG) 149 | 150 | push-ci-image: ## Push CI docker image. 151 | docker push $(CI_IMG) 152 | 153 | ci-%: ## `make ci-` will run the Makefile target in the CI docker Image. 154 | docker run --rm -ti \ 155 | -v $(shell pwd):/workspace \ 156 | -w /workspace \ 157 | $(CI_IMG) make $* 158 | 159 | endif 160 | -------------------------------------------------------------------------------- /PROJECT: -------------------------------------------------------------------------------- 1 | domain: opstrace.io 2 | layout: 3 | - go.kubebuilder.io/v3 4 | projectName: cortex-operator 5 | repo: github.com/opstrace/cortex-operator 6 | resources: 7 | - api: 8 | crdVersion: v1 9 | namespaced: true 10 | controller: true 11 | domain: opstrace.io 12 | group: cortex 13 | kind: Cortex 14 | path: github.com/opstrace/cortex-operator/api/v1alpha1 15 | version: v1alpha1 16 | webhooks: 17 | defaulting: true 18 | validation: true 19 | webhookVersion: v1beta1 20 | version: "3" 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cortex-operator 2 | 3 | The cortex-operator is a project to manage the lifecycle of [Cortex](https://cortexmetrics.io/) in Kubernetes. 4 | 5 | **Project status: alpha** Not all planned features are completed. The API, spec, status and other user facing objects will, most likely, change. *Don't use it in production.* 6 | 7 | ## Requirements 8 | 9 | ### Build 10 | 11 | - Docker 12 | - Kubectl 13 | 14 | ### Run 15 | 16 | - [EKS](https://aws.amazon.com/eks/) cluster version 1.18+ 17 | - Two S3 buckets 18 | - AWS IAM policy for nodes in EKS cluster to access S3 buckets 19 | - Kubectl configured to access the EKS cluster 20 | - [cert-manager](https://github.com/jetstack/cert-manager) version 1.3.1+ 21 | 22 | Check this [guide](./docs/guides/terraform/README.md) on how to set up the infrastructure with Terraform. 23 | 24 | ## Installation 25 | 26 | Build and push your Docker image to the location specified by `IMG`: 27 | 28 | ``` 29 | make docker-build docker-push IMG= 30 | ``` 31 | 32 | Install the CRDs in the cluster: 33 | 34 | ``` 35 | make install 36 | ``` 37 | 38 | Deploy the controller to the cluster with the Docker image specified by `IMG`: 39 | 40 | ``` 41 | make deploy IMG= 42 | ``` 43 | 44 | ## Uninstall 45 | 46 | Remove the controller from the cluster: 47 | 48 | ``` 49 | make undeploy 50 | ``` 51 | 52 | Remove the CRDs from the cluster: 53 | 54 | ``` 55 | make uninstall 56 | ``` 57 | 58 | ## Quickstart 59 | 60 | You can use this [guide](./docs/guides/terraform/README.md) to deploy the required infrastructure with Terraform. 61 | 62 | Edit the [sample resource](./config/samples/cortex_v1alpha1_cortex.yaml) and set the bucket name for the blocks storage. If you used our guide, it'd be in the form of `cortex-operator-example-XXXX-data`. Set the bucket name for the alert manager and ruler configuration. If you used our guide, it'd be in the form of `cortex-operator-example-XXXX-config` 63 | 64 | Create a Cortex resource to trigger the cortex-operator to start a deployment: 65 | 66 | ``` 67 | kubectl apply -f config/samples/cortex_v1alpha1_cortex.yaml 68 | ``` 69 | 70 | You should see a flurry of activity in the logs of the `cortex-operator`: 71 | 72 | ``` 73 | kubectl logs -n cortex-operator-system deploy/cortex-operator-controller-manager manager 74 | ``` 75 | 76 | You can confirm all the pods are up and running: 77 | 78 | ``` 79 | kubectl get pods 80 | ``` 81 | 82 | The output should be something like this: 83 | 84 | ``` 85 | NAME READY STATUS RESTARTS AGE 86 | compactor-0 1/1 Running 0 2m59s 87 | compactor-1 1/1 Running 0 2m46s 88 | distributor-db7f645c7-2bzlj 1/1 Running 0 2m59s 89 | distributor-db7f645c7-2n22k 1/1 Running 0 2m59s 90 | ingester-0 1/1 Running 0 2m59s 91 | ingester-1 1/1 Running 0 1m26s 92 | memcached-0 1/1 Running 0 3m 93 | memcached-index-queries-0 1/1 Running 0 3m 94 | memcached-index-writes-0 1/1 Running 0 3m 95 | memcached-metadata-0 1/1 Running 0 3m 96 | memcached-results-0 1/1 Running 0 3m 97 | querier-7dbd4cb465-66q95 1/1 Running 0 2m59s 98 | querier-7dbd4cb465-frfnj 1/1 Running 1 2m59s 99 | query-frontend-b9f7f97b7-g7lsf 1/1 Running 0 2m59s 100 | query-frontend-b9f7f97b7-tsppd 1/1 Running 0 2m59s 101 | store-gateway-0 1/1 Running 0 2m59s 102 | store-gateway-1 1/1 Running 0 2m34s 103 | ``` 104 | 105 | You can now send metrics to Cortex. As an example, let's set up Grafana Agent to collect metrics from the Kubernetes nodes and send them to Cortex. 106 | 107 | Create all the resources with: 108 | 109 | ``` 110 | kubectl apply -f docs/samples/grafana-example-manifest.yaml 111 | ``` 112 | 113 | In the future, we'll set up a Grafana dashboard to check these metrics, but for now, we'll use [cortex-tools](https://github.com/grafana/cortex-tools) to confirm Cortex is receiving metrics from the Grafana Agent. 114 | 115 | Set up port-forward with kubectl to query Cortex: 116 | 117 | ``` 118 | kubectl port-forward svc/query-frontend 8080:80 119 | ``` 120 | 121 | In another terminal run 122 | ``` 123 | cortextool remote-read dump --address=http://localhost:8080 --remote-read-path=/api/v1/read 124 | ``` 125 | 126 | The output should be something like this: 127 | 128 | ``` 129 | INFO[0000] Created remote read client using endpoint 'http://localhost:8080/api/v1/read' 130 | INFO[0000] Querying time from=2021-05-19T12:25:29Z to=2021-05-19T13:25:29Z with selector=up 131 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430648242 132 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430663242 133 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430678242 134 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430693242 135 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430708242 136 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430723242 137 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430635807 138 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430650807 139 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430665807 140 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430680807 141 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430695807 142 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430710807 143 | {__name__="up", instance="ip-10-0-0-42.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430725810 144 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430641820 145 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430656820 146 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430671820 147 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430686820 148 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430701820 149 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/agent", namespace="monitoring"} 1 1621430716820 150 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430645938 151 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430660938 152 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430675938 153 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430690938 154 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430705938 155 | {__name__="up", instance="ip-10-0-1-47.us-west-2.compute.internal", job="monitoring/node-exporter", namespace="monitoring"} 1 1621430720938 156 | ``` 157 | 158 | ## Cortex Runtime Configuration 159 | 160 | Cortex has a concept of [“runtime config” file](https://cortexmetrics.io/docs/configuration/arguments/#runtime-configuration-file) that Cortex components reload while running. It allows the operator to change aspects of Cortex configuration without restarting it. 161 | 162 | The `cortex-operator` supports this feature by configuring setting the `runtime_config.overrides` field in the CRD resource. The operator creates a ConfigMap named `cortex-runtime-config` in the namespace where Cortex is running. The ConfigMap is mounted in the Cortex components as a Kubernetes Volume. 163 | 164 | Example for setting the limits: 165 | 166 | ``` 167 | apiVersion: cortex.opstrace.io/v1alpha1 168 | kind: Cortex 169 | metadata: 170 | name: cortex-sample 171 | spec: 172 | 173 | runtime_config: 174 | overrides: 175 | tenant1: 176 | ingestion_rate: 10000 177 | max_series_per_metric: 100000 178 | max_series_per_query: 100000 179 | tenant2: 180 | max_samples_per_query: 1000000 181 | max_series_per_metric: 100000 182 | max_series_per_query: 100000 183 | ``` 184 | 185 | ## Specifying Kubernetes imagePullSecrets 186 | 187 | To set the [imagePullSecrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) with the `cortex-operator` you can set the `.spec.service_account_spec.image_pull_secrets` field. 188 | 189 | Example: 190 | 191 | ``` 192 | apiVersion: cortex.opstrace.io/v1alpha1 193 | kind: Cortex 194 | metadata: 195 | name: cortex-sample 196 | spec: 197 | service_account_spec: 198 | image_pull_secrets: 199 | - name: secret-name 200 | ``` 201 | 202 | ## Roadmap 203 | 204 | - Deploy Cortex in different topologies 205 | - Automate moving workloads to other instance 206 | - Auto-scaling of services 207 | 208 | ## Contributing 209 | 210 | Pull requests are welcome. For significant changes, please open an issue first to discuss what you would like to change. 211 | 212 | ## License 213 | [Apache License, Version 2](http://www.apache.org/licenses/LICENSE-2.0) 214 | 215 | -------------------------------------------------------------------------------- /api/v1alpha1/cortex_types.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "crypto/sha256" 21 | "encoding/hex" 22 | "fmt" 23 | 24 | corev1 "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/api/resource" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/utils/pointer" 29 | ) 30 | 31 | // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. 32 | 33 | // CortexSpec defines the desired state of Cortex 34 | type CortexSpec struct { 35 | // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster 36 | // Important: Run "make" to regenerate code after modifying this file 37 | 38 | // Image of Cortex to deploy. 39 | Image string `json:"image,omitempty"` 40 | 41 | ServiceAccountSpec *ServiceAccountSpec `json:"service_account_spec,omitempty"` 42 | 43 | IngesterSpec *StatefulSetSpec `json:"ingester_spec,omitempty"` 44 | CompactorSpec *StatefulSetSpec `json:"compactor_spec,omitempty"` 45 | StoreGatewaySpec *StatefulSetSpec `json:"store_gateway_spec,omitempty"` 46 | DistributorSpec *DeploymentSpec `json:"distributor_spec,omitempty"` 47 | QuerierSpec *DeploymentSpec `json:"querier_spec,omitempty"` 48 | QueryFrontendSpec *DeploymentSpec `json:"query_frontend_spec,omitempty"` 49 | AlertManagerSpec *DeploymentSpec `json:"alertmanager_spec,omitempty"` 50 | RulerSpec *DeploymentSpec `json:"ruler_spec,omitempty"` 51 | 52 | Memcached *MemcachedSpec `json:"memcached,omitempty"` 53 | 54 | // Config accepts any object, meaning it accepts any valid Cortex config 55 | // yaml. Defaulting and Validation are done in the webhooks. 56 | // +kubebuilder:pruning:PreserveUnknownFields 57 | Config runtime.RawExtension `json:"config,omitempty"` 58 | 59 | // +kubebuilder:pruning:PreserveUnknownFields 60 | RuntimeConfig *runtime.RawExtension `json:"runtime_config,omitempty"` 61 | } 62 | 63 | func (c *CortexSpec) ConfigSHA() string { 64 | sha := sha256.Sum256(c.Config.Raw) 65 | return hex.EncodeToString(sha[:]) 66 | } 67 | 68 | // CortexStatus defines the observed state of Cortex 69 | type CortexStatus struct { 70 | // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster 71 | // Important: Run "make" to regenerate code after modifying this file 72 | 73 | MemcachedRef *MemcachedReference `json:"memcached,omitempty"` 74 | IngesterRef *corev1.LocalObjectReference `json:"ingester,omitempty"` 75 | DistributorRef *corev1.LocalObjectReference `json:"distributor,omitempty"` 76 | QuerierRef *corev1.LocalObjectReference `json:"querier,omitempty"` 77 | QueryFrontendRef *corev1.LocalObjectReference `json:"query_frontend,omitempty"` 78 | CompactorRef *corev1.LocalObjectReference `json:"compactor,omitempty"` 79 | StoreGatewayRef *corev1.LocalObjectReference `json:"store_gateway,omitempty"` 80 | AlertManagerRef *corev1.LocalObjectReference `json:"alertmanager,omitempty"` 81 | RulerRef *corev1.LocalObjectReference `json:"ruler,omitempty"` 82 | } 83 | 84 | // MemcachedReference holds references to all the Memcached StatefulSets 85 | type MemcachedReference struct { 86 | ChunksCacheRef *corev1.LocalObjectReference `json:"chunks_cache,omitempty"` 87 | IndexQueriesCacheRef *corev1.LocalObjectReference `json:"index_queries_cache,omitempty"` 88 | IndexWritesCacheRef *corev1.LocalObjectReference `json:"index_writes_cache,omitempty"` 89 | ResultsCacheRef *corev1.LocalObjectReference `json:"results_cache,omitempty"` 90 | MetadataCacheRef *corev1.LocalObjectReference `json:"metadata_cache,omitempty"` 91 | } 92 | 93 | // IsSet returns true if all the resource references are not nil 94 | func (r *MemcachedReference) IsSet() bool { 95 | return r != nil && 96 | r.ChunksCacheRef != nil && 97 | r.IndexQueriesCacheRef != nil && 98 | r.IndexWritesCacheRef != nil && 99 | r.ResultsCacheRef != nil && 100 | r.MetadataCacheRef != nil 101 | } 102 | 103 | type StatefulSetSpec struct { 104 | DatadirSize *resource.Quantity `json:"datadir_size,omitempty"` 105 | StorageClassName *string `json:"storage_class_name,omitempty"` 106 | Replicas *int32 `json:"replicas,omitempty"` 107 | } 108 | 109 | func (s *StatefulSetSpec) Default() { 110 | if s.DatadirSize == nil { 111 | r := resource.MustParse("1Gi") 112 | s.DatadirSize = &r 113 | } 114 | 115 | if s.Replicas == nil { 116 | s.Replicas = pointer.Int32Ptr(2) 117 | } 118 | } 119 | 120 | type DeploymentSpec struct { 121 | Replicas *int32 `json:"replicas,omitempty"` 122 | } 123 | 124 | func (s *DeploymentSpec) Default() { 125 | if s.Replicas == nil { 126 | s.Replicas = pointer.Int32Ptr(2) 127 | } 128 | } 129 | 130 | type MemcachedSpec struct { 131 | Image string `json:"image,omitempty"` 132 | 133 | ChunksCacheSpec *MemcachedStatefulSetSpec `json:"chunks_cache_spec,omitempty"` 134 | IndexQueriesCacheSpec *MemcachedStatefulSetSpec `json:"index_queries_cache_spec,omitempty"` 135 | IndexWritesCacheSpec *MemcachedStatefulSetSpec `json:"index_writes_cache_spec,omitempty"` 136 | ResultsCacheSpec *MemcachedStatefulSetSpec `json:"results_cache_spec,omitempty"` 137 | MetadataCacheSpec *MemcachedStatefulSetSpec `json:"metadata_cache_spec,omitempty"` 138 | } 139 | 140 | func (m *MemcachedSpec) Default() { 141 | if m.Image == "" { 142 | m.Image = "memcached:1.6.9-alpine" 143 | } 144 | 145 | if m.ChunksCacheSpec == nil { 146 | m.ChunksCacheSpec = &MemcachedStatefulSetSpec{} 147 | } 148 | if m.IndexQueriesCacheSpec == nil { 149 | m.IndexQueriesCacheSpec = &MemcachedStatefulSetSpec{} 150 | } 151 | if m.IndexWritesCacheSpec == nil { 152 | m.IndexWritesCacheSpec = &MemcachedStatefulSetSpec{} 153 | } 154 | if m.ResultsCacheSpec == nil { 155 | m.ResultsCacheSpec = &MemcachedStatefulSetSpec{} 156 | } 157 | if m.MetadataCacheSpec == nil { 158 | m.MetadataCacheSpec = &MemcachedStatefulSetSpec{} 159 | } 160 | 161 | m.ChunksCacheSpec.Default() 162 | m.IndexQueriesCacheSpec.Default() 163 | m.IndexWritesCacheSpec.Default() 164 | m.ResultsCacheSpec.Default() 165 | m.MetadataCacheSpec.Default() 166 | } 167 | 168 | type MemcachedStatefulSetSpec struct { 169 | Replicas *int32 `json:"replicas,omitempty"` 170 | // MemoryLimit is the item memory in megabytes 171 | MemoryLimit *int32 `json:"memory_limit,omitempty"` 172 | // MaxItemSize adjusts max item size 173 | MaxItemSize *string `json:"max_item_size,omitempty"` 174 | } 175 | 176 | func (m *MemcachedStatefulSetSpec) AsArgs() []string { 177 | return []string{ 178 | fmt.Sprintf("-m %d", *m.MemoryLimit), 179 | fmt.Sprintf("-I %s", *m.MaxItemSize), 180 | } 181 | } 182 | 183 | func (m *MemcachedStatefulSetSpec) Default() { 184 | if m == nil { 185 | m = &MemcachedStatefulSetSpec{} 186 | } 187 | if m.Replicas == nil { 188 | m.Replicas = pointer.Int32Ptr(2) 189 | } 190 | if m.MemoryLimit == nil { 191 | m.MemoryLimit = pointer.Int32Ptr(4096) 192 | } 193 | if m.MaxItemSize == nil { 194 | m.MaxItemSize = pointer.StringPtr("2m") 195 | } 196 | } 197 | 198 | type ServiceAccountSpec struct { 199 | Annotations map[string]string `json:"annotations,omitempty"` 200 | // ImagePullSecrets is a list of references to secrets in the same namespace 201 | // to use for pulling any images in pods that reference this ServiceAccount. 202 | // More info: 203 | // https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod 204 | ImagePullSecrets []corev1.LocalObjectReference `json:"image_pull_secrets,omitempty"` 205 | } 206 | 207 | //+kubebuilder:object:root=true 208 | //+kubebuilder:subresource:status 209 | 210 | // Cortex is the Schema for the cortices API 211 | type Cortex struct { 212 | metav1.TypeMeta `json:",inline"` 213 | metav1.ObjectMeta `json:"metadata,omitempty"` 214 | 215 | Spec CortexSpec `json:"spec,omitempty"` 216 | Status CortexStatus `json:"status,omitempty"` 217 | } 218 | 219 | //+kubebuilder:object:root=true 220 | 221 | // CortexList contains a list of Cortex 222 | type CortexList struct { 223 | metav1.TypeMeta `json:",inline"` 224 | metav1.ListMeta `json:"metadata,omitempty"` 225 | Items []Cortex `json:"items"` 226 | } 227 | 228 | func init() { 229 | SchemeBuilder.Register(&Cortex{}, &CortexList{}) 230 | } 231 | -------------------------------------------------------------------------------- /api/v1alpha1/cortex_webhook.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "html/template" 23 | 24 | "github.com/miracl/conflate" 25 | yamlv2 "gopkg.in/yaml.v2" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | ctrl "sigs.k8s.io/controller-runtime" 28 | logf "sigs.k8s.io/controller-runtime/pkg/log" 29 | "sigs.k8s.io/controller-runtime/pkg/webhook" 30 | "sigs.k8s.io/yaml" 31 | 32 | "github.com/cortexproject/cortex/pkg/cortex" 33 | "github.com/cortexproject/cortex/pkg/util/flagext" 34 | ) 35 | 36 | // log is for logging in this package. 37 | var cortexlog = logf.Log.WithName("cortex-resource") 38 | 39 | func (r *Cortex) SetupWebhookWithManager(mgr ctrl.Manager) error { 40 | return ctrl.NewWebhookManagedBy(mgr). 41 | For(r). 42 | Complete() 43 | } 44 | 45 | //+kubebuilder:webhook:webhookVersions={v1beta1},path=/mutate-cortex-opstrace-io-v1alpha1-cortex,mutating=true,failurePolicy=fail,sideEffects=None,groups=cortex.opstrace.io,resources=cortices,verbs=create;update,versions=v1alpha1,name=mcortex.kb.io,admissionReviewVersions={v1,v1beta1} 46 | 47 | var _ webhook.Defaulter = &Cortex{} 48 | 49 | // Default implements webhook.Defaulter so a webhook will be registered for the type 50 | func (r *Cortex) Default() { 51 | cortexlog.Info("default", "name", r.Name) 52 | 53 | if r.Spec.Memcached == nil { 54 | r.Spec.Memcached = &MemcachedSpec{} 55 | } 56 | r.Spec.Memcached.Default() 57 | 58 | if r.Spec.IngesterSpec == nil { 59 | r.Spec.IngesterSpec = &StatefulSetSpec{} 60 | } 61 | r.Spec.IngesterSpec.Default() 62 | 63 | if r.Spec.CompactorSpec == nil { 64 | r.Spec.CompactorSpec = &StatefulSetSpec{} 65 | } 66 | r.Spec.CompactorSpec.Default() 67 | 68 | if r.Spec.StoreGatewaySpec == nil { 69 | r.Spec.StoreGatewaySpec = &StatefulSetSpec{} 70 | } 71 | r.Spec.StoreGatewaySpec.Default() 72 | 73 | if r.Spec.DistributorSpec == nil { 74 | r.Spec.DistributorSpec = &DeploymentSpec{} 75 | } 76 | r.Spec.DistributorSpec.Default() 77 | 78 | if r.Spec.QuerierSpec == nil { 79 | r.Spec.QuerierSpec = &DeploymentSpec{} 80 | } 81 | r.Spec.QuerierSpec.Default() 82 | 83 | if r.Spec.QueryFrontendSpec == nil { 84 | r.Spec.QueryFrontendSpec = &DeploymentSpec{} 85 | } 86 | r.Spec.QueryFrontendSpec.Default() 87 | 88 | if r.Spec.AlertManagerSpec == nil { 89 | r.Spec.AlertManagerSpec = &DeploymentSpec{} 90 | } 91 | r.Spec.AlertManagerSpec.Default() 92 | 93 | if r.Spec.RulerSpec == nil { 94 | r.Spec.RulerSpec = &DeploymentSpec{} 95 | } 96 | r.Spec.RulerSpec.Default() 97 | } 98 | 99 | // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. 100 | //+kubebuilder:webhook:webhookVersions={v1beta1},path=/validate-cortex-opstrace-io-v1alpha1-cortex,mutating=false,failurePolicy=fail,sideEffects=None,groups=cortex.opstrace.io,resources=cortices,verbs=create;update,versions=v1alpha1,name=vcortex.kb.io,admissionReviewVersions={v1,v1beta1} 101 | 102 | var _ webhook.Validator = &Cortex{} 103 | 104 | // ValidateCreate implements webhook.Validator so a webhook will be registered for the type 105 | func (r *Cortex) ValidateCreate() error { 106 | cortexlog.Info("validate create", "name", r.Name) 107 | return r.Validate() 108 | } 109 | 110 | // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type 111 | func (r *Cortex) ValidateUpdate(old runtime.Object) error { 112 | cortexlog.Info("validate update", "name", r.Name) 113 | return r.Validate() 114 | } 115 | 116 | // ValidateDelete implements webhook.Validator so a webhook will be registered for the type 117 | func (r *Cortex) ValidateDelete() error { 118 | cortexlog.Info("validate delete", "name", r.Name) 119 | 120 | // TODO(user): fill in your validation logic upon object deletion. 121 | return nil 122 | } 123 | 124 | func (r *Cortex) RuntimeConfigAsYAML() ([]byte, error) { 125 | if r == nil { 126 | return []byte{}, nil 127 | } 128 | 129 | return yaml.Marshal(r.Spec.RuntimeConfig) 130 | } 131 | 132 | // generateCortexConfig returns a config yaml with the cortex-operator default 133 | // configuration. 134 | func (r *Cortex) generateCortexConfig() ([]byte, error) { 135 | t := template.New("cortex-config") 136 | 137 | t, err := t.Parse(DefaultCortexConfigTemplate) 138 | if err != nil { 139 | return nil, err 140 | } 141 | 142 | var b bytes.Buffer 143 | err = t.Execute(&b, r) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | return b.Bytes(), nil 149 | } 150 | 151 | func (r *Cortex) AsJSON() ([]byte, error) { 152 | // Generate the default configuration. 153 | defaultCortexConfig, err := r.generateCortexConfig() 154 | if err != nil { 155 | return nil, fmt.Errorf("failed to generate default configuration: %w", err) 156 | } 157 | // Convert it to JSON to be able to merge with the user config. 158 | y, err := yaml.YAMLToJSON(defaultCortexConfig) 159 | if err != nil { 160 | return nil, fmt.Errorf("failed to convert default configuration: %w", err) 161 | } 162 | // Merge the user config and the defaults. The defaults will override fields 163 | // set by the user. 164 | c, err := conflate.FromData(r.Spec.Config.Raw, y) 165 | if err != nil { 166 | return nil, fmt.Errorf("failed to merge default configuration with user settings: %w", err) 167 | } 168 | // Convert the data to json. 169 | j, err := c.MarshalJSON() 170 | if err != nil { 171 | return nil, fmt.Errorf("failed to convert user configuration: %w", err) 172 | } 173 | return j, nil 174 | } 175 | 176 | func (r *Cortex) AsYAML() ([]byte, error) { 177 | j, err := r.AsJSON() 178 | if err != nil { 179 | return nil, err 180 | } 181 | return yaml.JSONToYAML(j) 182 | } 183 | 184 | // AsCortexConfig converts the configuration to an upstream cortex.Config 185 | // object. 186 | func (r *Cortex) AsCortexConfig() (*cortex.Config, error) { 187 | j, err := r.AsJSON() 188 | if err != nil { 189 | return nil, err 190 | } 191 | // Unmarshal the json to a cortex.Config structure to validate it. 192 | cfg := &cortex.Config{} 193 | // Set up the cortex config defaults otherwise validation will fail because 194 | // fields are not set. 195 | flagext.DefaultValues(cfg) 196 | // Unmarshal the desired cortex config into the object overriding the 197 | // defaults. 198 | err = yamlv2.UnmarshalStrict(j, cfg) 199 | if err != nil { 200 | return nil, fmt.Errorf("failed to : %w", err) 201 | } 202 | return cfg, nil 203 | } 204 | 205 | func (r *Cortex) Validate() error { 206 | cfg, err := r.AsCortexConfig() 207 | if err != nil { 208 | return err 209 | } 210 | // Validate the cortex configuration. 211 | return cfg.Validate(nil) 212 | } 213 | 214 | const DefaultCortexConfigTemplate = ` 215 | http_prefix: '' 216 | api: 217 | alertmanager_http_prefix: /alertmanager 218 | response_compression_enabled: true 219 | auth_enabled: true 220 | distributor: 221 | shard_by_all_labels: true 222 | pool: 223 | health_check_ingesters: true 224 | ha_tracker: 225 | enable_ha_tracker: false 226 | memberlist: 227 | abort_if_cluster_join_fails: true 228 | bind_port: 7946 229 | join_members: 230 | - 'gossip-ring.{{.Namespace}}.svc.cluster.local:7946' 231 | querier: 232 | batch_iterators: true 233 | ingester_streaming: true 234 | store_gateway_addresses: 'store-gateway.{{.Namespace}}.svc.cluster.local:9095' 235 | query_range: 236 | align_queries_with_step: true 237 | cache_results: true 238 | results_cache: 239 | cache: 240 | memcached_client: 241 | consistent_hash: true 242 | host: memcached-results.{{.Namespace}}.svc.cluster.local 243 | service: memcached-client 244 | frontend_worker: 245 | frontend_address: 'query-frontend.{{.Namespace}}.svc.cluster.local:9095' 246 | ingester: 247 | lifecycler: 248 | ring: 249 | kvstore: 250 | store: memberlist 251 | blocks_storage: 252 | tsdb: 253 | dir: /cortex/tsdb 254 | wal_compression_enabled: true 255 | bucket_store: 256 | sync_dir: /cortex/tsdb-sync 257 | index_cache: 258 | backend: memcached 259 | memcached: 260 | addresses: 'dnssrv+memcached-index-queries.{{.Namespace}}.svc.cluster.local:11211' 261 | chunks_cache: 262 | backend: memcached 263 | memcached: 264 | addresses: 'dnssrv+memcached-chunks.{{.Namespace}}.svc.cluster.local:11211' 265 | metadata_cache: 266 | backend: memcached 267 | memcached: 268 | addresses: 'dnssrv+memcached-metadata.{{.Namespace}}.svc.cluster.local:11211' 269 | store_gateway: 270 | sharding_enabled: true 271 | sharding_ring: 272 | kvstore: 273 | store: memberlist 274 | compactor: 275 | data_dir: /cortex/compactor 276 | sharding_enabled: true 277 | sharding_ring: 278 | kvstore: 279 | store: memberlist 280 | purger: 281 | enable: true 282 | storage: 283 | engine: blocks 284 | alertmanager: 285 | enable_api: true 286 | cluster: 287 | peers: 'alertmanager.{{.Namespace}}.svc.cluster.local:9094' 288 | sharding_enabled: true 289 | sharding_ring: 290 | kvstore: 291 | store: memberlist 292 | external_url: /alertmanager 293 | ruler: 294 | enable_api: true 295 | enable_sharding: true 296 | sharding_strategy: shuffle-sharding 297 | ring: 298 | kvstore: 299 | store: memberlist 300 | alertmanager_url: 'http://alertmanager.{{.Namespace}}.svc.cluster.local/alertmanager/' 301 | runtime_config: 302 | file: /etc/cortex/runtime-config.yaml 303 | period: 5s 304 | ` 305 | -------------------------------------------------------------------------------- /api/v1alpha1/cortex_webhook_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "io/ioutil" 23 | "path/filepath" 24 | 25 | . "github.com/onsi/ginkgo" 26 | . "github.com/onsi/gomega" 27 | "k8s.io/apimachinery/pkg/runtime/serializer" 28 | ) 29 | 30 | var _ = Describe("Cortex validation webhook", func() { 31 | 32 | BeforeEach(func(done Done) { 33 | fmt.Fprintf(GinkgoWriter, "BeforeEach\n") 34 | close(done) 35 | }) 36 | 37 | It("should validate sample", func() { 38 | manifest := filepath.Join("..", "..", "config", "samples", "cortex_v1alpha1_cortex.yaml") 39 | c := GetCortexTestSample(manifest) 40 | err := k8sClient.Create(context.Background(), c) 41 | Expect(err).ToNot(HaveOccurred()) 42 | }) 43 | 44 | }) 45 | 46 | func GetCortexTestSample(filepath string) *Cortex { 47 | b, err := ioutil.ReadFile(filepath) 48 | Expect(err).ToNot(HaveOccurred()) 49 | 50 | deserializer := serializer.NewCodecFactory(k8sClientScheme).UniversalDeserializer() 51 | obj, _, err := deserializer.Decode(b, nil, nil) 52 | Expect(err).ToNot(HaveOccurred()) 53 | Expect(obj).To(BeAssignableToTypeOf(&Cortex{})) 54 | 55 | c := obj.(*Cortex) 56 | c.Namespace = "default" 57 | 58 | return c 59 | } 60 | -------------------------------------------------------------------------------- /api/v1alpha1/groupversion_info.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package v1alpha1 contains API Schema definitions for the cortex v1alpha1 API group 18 | //+kubebuilder:object:generate=true 19 | //+groupName=cortex.opstrace.io 20 | package v1alpha1 21 | 22 | import ( 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "sigs.k8s.io/controller-runtime/pkg/scheme" 25 | ) 26 | 27 | var ( 28 | // GroupVersion is group version used to register these objects 29 | GroupVersion = schema.GroupVersion{Group: "cortex.opstrace.io", Version: "v1alpha1"} 30 | 31 | // SchemeBuilder is used to add go types to the GroupVersionKind scheme 32 | SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} 33 | 34 | // AddToScheme adds the types in this group-version to the given scheme. 35 | AddToScheme = SchemeBuilder.AddToScheme 36 | ) 37 | -------------------------------------------------------------------------------- /api/v1alpha1/webhook_suite_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package v1alpha1 18 | 19 | import ( 20 | "context" 21 | "crypto/tls" 22 | "fmt" 23 | "net" 24 | "path/filepath" 25 | "testing" 26 | "time" 27 | 28 | . "github.com/onsi/ginkgo" 29 | . "github.com/onsi/gomega" 30 | 31 | admissionv1beta1 "k8s.io/api/admission/v1beta1" 32 | //+kubebuilder:scaffold:imports 33 | "k8s.io/apimachinery/pkg/runtime" 34 | "k8s.io/client-go/rest" 35 | ctrl "sigs.k8s.io/controller-runtime" 36 | "sigs.k8s.io/controller-runtime/pkg/client" 37 | "sigs.k8s.io/controller-runtime/pkg/envtest" 38 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 39 | logf "sigs.k8s.io/controller-runtime/pkg/log" 40 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 41 | ) 42 | 43 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 44 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 45 | 46 | var ( 47 | cfg *rest.Config 48 | k8sClient client.Client 49 | testEnv *envtest.Environment 50 | ctx context.Context 51 | cancel context.CancelFunc 52 | k8sClientScheme = runtime.NewScheme() 53 | ) 54 | 55 | func TestAPIs(t *testing.T) { 56 | RegisterFailHandler(Fail) 57 | 58 | RunSpecsWithDefaultAndCustomReporters(t, 59 | "Webhook Suite", 60 | []Reporter{printer.NewlineReporter{}}) 61 | } 62 | 63 | var _ = BeforeSuite(func() { 64 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 65 | 66 | ctx, cancel = context.WithCancel(context.TODO()) 67 | 68 | By("bootstrapping test environment") 69 | testEnv = &envtest.Environment{ 70 | CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, 71 | ErrorIfCRDPathMissing: false, 72 | WebhookInstallOptions: envtest.WebhookInstallOptions{ 73 | Paths: []string{filepath.Join("..", "..", "config", "webhook")}, 74 | }, 75 | } 76 | 77 | cfg, err := testEnv.Start() 78 | Expect(err).NotTo(HaveOccurred()) 79 | Expect(cfg).NotTo(BeNil()) 80 | 81 | err = AddToScheme(k8sClientScheme) 82 | Expect(err).NotTo(HaveOccurred()) 83 | 84 | err = admissionv1beta1.AddToScheme(k8sClientScheme) 85 | Expect(err).NotTo(HaveOccurred()) 86 | 87 | //+kubebuilder:scaffold:scheme 88 | 89 | k8sClient, err = client.New(cfg, client.Options{Scheme: k8sClientScheme}) 90 | Expect(err).NotTo(HaveOccurred()) 91 | Expect(k8sClient).NotTo(BeNil()) 92 | 93 | // start webhook server using Manager 94 | webhookInstallOptions := &testEnv.WebhookInstallOptions 95 | mgr, err := ctrl.NewManager(cfg, ctrl.Options{ 96 | Scheme: k8sClientScheme, 97 | Host: webhookInstallOptions.LocalServingHost, 98 | Port: webhookInstallOptions.LocalServingPort, 99 | CertDir: webhookInstallOptions.LocalServingCertDir, 100 | LeaderElection: false, 101 | MetricsBindAddress: "0", 102 | }) 103 | Expect(err).NotTo(HaveOccurred()) 104 | 105 | err = (&Cortex{}).SetupWebhookWithManager(mgr) 106 | Expect(err).NotTo(HaveOccurred()) 107 | 108 | //+kubebuilder:scaffold:webhook 109 | 110 | go func() { 111 | err = mgr.Start(ctx) 112 | if err != nil { 113 | Expect(err).NotTo(HaveOccurred()) 114 | } 115 | }() 116 | 117 | // wait for the webhook server to get ready 118 | dialer := &net.Dialer{Timeout: time.Second} 119 | addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) 120 | Eventually(func() error { 121 | conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) 122 | if err != nil { 123 | return err 124 | } 125 | conn.Close() 126 | return nil 127 | }).Should(Succeed()) 128 | 129 | }, 60) 130 | 131 | var _ = AfterSuite(func() { 132 | cancel() 133 | By("tearing down the test environment") 134 | err := testEnv.Stop() 135 | Expect(err).NotTo(HaveOccurred()) 136 | }) 137 | -------------------------------------------------------------------------------- /api/v1alpha1/zz_generated.deepcopy.go: -------------------------------------------------------------------------------- 1 | // +build !ignore_autogenerated 2 | 3 | /** 4 | * Copyright 2021 Opstrace, Inc. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // Code generated by controller-gen. DO NOT EDIT. 20 | 21 | package v1alpha1 22 | 23 | import ( 24 | "k8s.io/api/core/v1" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | ) 27 | 28 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 29 | func (in *Cortex) DeepCopyInto(out *Cortex) { 30 | *out = *in 31 | out.TypeMeta = in.TypeMeta 32 | in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) 33 | in.Spec.DeepCopyInto(&out.Spec) 34 | in.Status.DeepCopyInto(&out.Status) 35 | } 36 | 37 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cortex. 38 | func (in *Cortex) DeepCopy() *Cortex { 39 | if in == nil { 40 | return nil 41 | } 42 | out := new(Cortex) 43 | in.DeepCopyInto(out) 44 | return out 45 | } 46 | 47 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 48 | func (in *Cortex) DeepCopyObject() runtime.Object { 49 | if c := in.DeepCopy(); c != nil { 50 | return c 51 | } 52 | return nil 53 | } 54 | 55 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 56 | func (in *CortexList) DeepCopyInto(out *CortexList) { 57 | *out = *in 58 | out.TypeMeta = in.TypeMeta 59 | in.ListMeta.DeepCopyInto(&out.ListMeta) 60 | if in.Items != nil { 61 | in, out := &in.Items, &out.Items 62 | *out = make([]Cortex, len(*in)) 63 | for i := range *in { 64 | (*in)[i].DeepCopyInto(&(*out)[i]) 65 | } 66 | } 67 | } 68 | 69 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CortexList. 70 | func (in *CortexList) DeepCopy() *CortexList { 71 | if in == nil { 72 | return nil 73 | } 74 | out := new(CortexList) 75 | in.DeepCopyInto(out) 76 | return out 77 | } 78 | 79 | // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. 80 | func (in *CortexList) DeepCopyObject() runtime.Object { 81 | if c := in.DeepCopy(); c != nil { 82 | return c 83 | } 84 | return nil 85 | } 86 | 87 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 88 | func (in *CortexSpec) DeepCopyInto(out *CortexSpec) { 89 | *out = *in 90 | if in.ServiceAccountSpec != nil { 91 | in, out := &in.ServiceAccountSpec, &out.ServiceAccountSpec 92 | *out = new(ServiceAccountSpec) 93 | (*in).DeepCopyInto(*out) 94 | } 95 | if in.IngesterSpec != nil { 96 | in, out := &in.IngesterSpec, &out.IngesterSpec 97 | *out = new(StatefulSetSpec) 98 | (*in).DeepCopyInto(*out) 99 | } 100 | if in.CompactorSpec != nil { 101 | in, out := &in.CompactorSpec, &out.CompactorSpec 102 | *out = new(StatefulSetSpec) 103 | (*in).DeepCopyInto(*out) 104 | } 105 | if in.StoreGatewaySpec != nil { 106 | in, out := &in.StoreGatewaySpec, &out.StoreGatewaySpec 107 | *out = new(StatefulSetSpec) 108 | (*in).DeepCopyInto(*out) 109 | } 110 | if in.DistributorSpec != nil { 111 | in, out := &in.DistributorSpec, &out.DistributorSpec 112 | *out = new(DeploymentSpec) 113 | (*in).DeepCopyInto(*out) 114 | } 115 | if in.QuerierSpec != nil { 116 | in, out := &in.QuerierSpec, &out.QuerierSpec 117 | *out = new(DeploymentSpec) 118 | (*in).DeepCopyInto(*out) 119 | } 120 | if in.QueryFrontendSpec != nil { 121 | in, out := &in.QueryFrontendSpec, &out.QueryFrontendSpec 122 | *out = new(DeploymentSpec) 123 | (*in).DeepCopyInto(*out) 124 | } 125 | if in.AlertManagerSpec != nil { 126 | in, out := &in.AlertManagerSpec, &out.AlertManagerSpec 127 | *out = new(DeploymentSpec) 128 | (*in).DeepCopyInto(*out) 129 | } 130 | if in.RulerSpec != nil { 131 | in, out := &in.RulerSpec, &out.RulerSpec 132 | *out = new(DeploymentSpec) 133 | (*in).DeepCopyInto(*out) 134 | } 135 | if in.Memcached != nil { 136 | in, out := &in.Memcached, &out.Memcached 137 | *out = new(MemcachedSpec) 138 | (*in).DeepCopyInto(*out) 139 | } 140 | in.Config.DeepCopyInto(&out.Config) 141 | if in.RuntimeConfig != nil { 142 | in, out := &in.RuntimeConfig, &out.RuntimeConfig 143 | *out = new(runtime.RawExtension) 144 | (*in).DeepCopyInto(*out) 145 | } 146 | } 147 | 148 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CortexSpec. 149 | func (in *CortexSpec) DeepCopy() *CortexSpec { 150 | if in == nil { 151 | return nil 152 | } 153 | out := new(CortexSpec) 154 | in.DeepCopyInto(out) 155 | return out 156 | } 157 | 158 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 159 | func (in *CortexStatus) DeepCopyInto(out *CortexStatus) { 160 | *out = *in 161 | if in.MemcachedRef != nil { 162 | in, out := &in.MemcachedRef, &out.MemcachedRef 163 | *out = new(MemcachedReference) 164 | (*in).DeepCopyInto(*out) 165 | } 166 | if in.IngesterRef != nil { 167 | in, out := &in.IngesterRef, &out.IngesterRef 168 | *out = new(v1.LocalObjectReference) 169 | **out = **in 170 | } 171 | if in.DistributorRef != nil { 172 | in, out := &in.DistributorRef, &out.DistributorRef 173 | *out = new(v1.LocalObjectReference) 174 | **out = **in 175 | } 176 | if in.QuerierRef != nil { 177 | in, out := &in.QuerierRef, &out.QuerierRef 178 | *out = new(v1.LocalObjectReference) 179 | **out = **in 180 | } 181 | if in.QueryFrontendRef != nil { 182 | in, out := &in.QueryFrontendRef, &out.QueryFrontendRef 183 | *out = new(v1.LocalObjectReference) 184 | **out = **in 185 | } 186 | if in.CompactorRef != nil { 187 | in, out := &in.CompactorRef, &out.CompactorRef 188 | *out = new(v1.LocalObjectReference) 189 | **out = **in 190 | } 191 | if in.StoreGatewayRef != nil { 192 | in, out := &in.StoreGatewayRef, &out.StoreGatewayRef 193 | *out = new(v1.LocalObjectReference) 194 | **out = **in 195 | } 196 | if in.AlertManagerRef != nil { 197 | in, out := &in.AlertManagerRef, &out.AlertManagerRef 198 | *out = new(v1.LocalObjectReference) 199 | **out = **in 200 | } 201 | if in.RulerRef != nil { 202 | in, out := &in.RulerRef, &out.RulerRef 203 | *out = new(v1.LocalObjectReference) 204 | **out = **in 205 | } 206 | } 207 | 208 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CortexStatus. 209 | func (in *CortexStatus) DeepCopy() *CortexStatus { 210 | if in == nil { 211 | return nil 212 | } 213 | out := new(CortexStatus) 214 | in.DeepCopyInto(out) 215 | return out 216 | } 217 | 218 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 219 | func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { 220 | *out = *in 221 | if in.Replicas != nil { 222 | in, out := &in.Replicas, &out.Replicas 223 | *out = new(int32) 224 | **out = **in 225 | } 226 | } 227 | 228 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec. 229 | func (in *DeploymentSpec) DeepCopy() *DeploymentSpec { 230 | if in == nil { 231 | return nil 232 | } 233 | out := new(DeploymentSpec) 234 | in.DeepCopyInto(out) 235 | return out 236 | } 237 | 238 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 239 | func (in *MemcachedReference) DeepCopyInto(out *MemcachedReference) { 240 | *out = *in 241 | if in.ChunksCacheRef != nil { 242 | in, out := &in.ChunksCacheRef, &out.ChunksCacheRef 243 | *out = new(v1.LocalObjectReference) 244 | **out = **in 245 | } 246 | if in.IndexQueriesCacheRef != nil { 247 | in, out := &in.IndexQueriesCacheRef, &out.IndexQueriesCacheRef 248 | *out = new(v1.LocalObjectReference) 249 | **out = **in 250 | } 251 | if in.IndexWritesCacheRef != nil { 252 | in, out := &in.IndexWritesCacheRef, &out.IndexWritesCacheRef 253 | *out = new(v1.LocalObjectReference) 254 | **out = **in 255 | } 256 | if in.ResultsCacheRef != nil { 257 | in, out := &in.ResultsCacheRef, &out.ResultsCacheRef 258 | *out = new(v1.LocalObjectReference) 259 | **out = **in 260 | } 261 | if in.MetadataCacheRef != nil { 262 | in, out := &in.MetadataCacheRef, &out.MetadataCacheRef 263 | *out = new(v1.LocalObjectReference) 264 | **out = **in 265 | } 266 | } 267 | 268 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemcachedReference. 269 | func (in *MemcachedReference) DeepCopy() *MemcachedReference { 270 | if in == nil { 271 | return nil 272 | } 273 | out := new(MemcachedReference) 274 | in.DeepCopyInto(out) 275 | return out 276 | } 277 | 278 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 279 | func (in *MemcachedSpec) DeepCopyInto(out *MemcachedSpec) { 280 | *out = *in 281 | if in.ChunksCacheSpec != nil { 282 | in, out := &in.ChunksCacheSpec, &out.ChunksCacheSpec 283 | *out = new(MemcachedStatefulSetSpec) 284 | (*in).DeepCopyInto(*out) 285 | } 286 | if in.IndexQueriesCacheSpec != nil { 287 | in, out := &in.IndexQueriesCacheSpec, &out.IndexQueriesCacheSpec 288 | *out = new(MemcachedStatefulSetSpec) 289 | (*in).DeepCopyInto(*out) 290 | } 291 | if in.IndexWritesCacheSpec != nil { 292 | in, out := &in.IndexWritesCacheSpec, &out.IndexWritesCacheSpec 293 | *out = new(MemcachedStatefulSetSpec) 294 | (*in).DeepCopyInto(*out) 295 | } 296 | if in.ResultsCacheSpec != nil { 297 | in, out := &in.ResultsCacheSpec, &out.ResultsCacheSpec 298 | *out = new(MemcachedStatefulSetSpec) 299 | (*in).DeepCopyInto(*out) 300 | } 301 | if in.MetadataCacheSpec != nil { 302 | in, out := &in.MetadataCacheSpec, &out.MetadataCacheSpec 303 | *out = new(MemcachedStatefulSetSpec) 304 | (*in).DeepCopyInto(*out) 305 | } 306 | } 307 | 308 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemcachedSpec. 309 | func (in *MemcachedSpec) DeepCopy() *MemcachedSpec { 310 | if in == nil { 311 | return nil 312 | } 313 | out := new(MemcachedSpec) 314 | in.DeepCopyInto(out) 315 | return out 316 | } 317 | 318 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 319 | func (in *MemcachedStatefulSetSpec) DeepCopyInto(out *MemcachedStatefulSetSpec) { 320 | *out = *in 321 | if in.Replicas != nil { 322 | in, out := &in.Replicas, &out.Replicas 323 | *out = new(int32) 324 | **out = **in 325 | } 326 | if in.MemoryLimit != nil { 327 | in, out := &in.MemoryLimit, &out.MemoryLimit 328 | *out = new(int32) 329 | **out = **in 330 | } 331 | if in.MaxItemSize != nil { 332 | in, out := &in.MaxItemSize, &out.MaxItemSize 333 | *out = new(string) 334 | **out = **in 335 | } 336 | } 337 | 338 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemcachedStatefulSetSpec. 339 | func (in *MemcachedStatefulSetSpec) DeepCopy() *MemcachedStatefulSetSpec { 340 | if in == nil { 341 | return nil 342 | } 343 | out := new(MemcachedStatefulSetSpec) 344 | in.DeepCopyInto(out) 345 | return out 346 | } 347 | 348 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 349 | func (in *ServiceAccountSpec) DeepCopyInto(out *ServiceAccountSpec) { 350 | *out = *in 351 | if in.Annotations != nil { 352 | in, out := &in.Annotations, &out.Annotations 353 | *out = make(map[string]string, len(*in)) 354 | for key, val := range *in { 355 | (*out)[key] = val 356 | } 357 | } 358 | if in.ImagePullSecrets != nil { 359 | in, out := &in.ImagePullSecrets, &out.ImagePullSecrets 360 | *out = make([]v1.LocalObjectReference, len(*in)) 361 | copy(*out, *in) 362 | } 363 | } 364 | 365 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceAccountSpec. 366 | func (in *ServiceAccountSpec) DeepCopy() *ServiceAccountSpec { 367 | if in == nil { 368 | return nil 369 | } 370 | out := new(ServiceAccountSpec) 371 | in.DeepCopyInto(out) 372 | return out 373 | } 374 | 375 | // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. 376 | func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { 377 | *out = *in 378 | if in.DatadirSize != nil { 379 | in, out := &in.DatadirSize, &out.DatadirSize 380 | x := (*in).DeepCopy() 381 | *out = &x 382 | } 383 | if in.StorageClassName != nil { 384 | in, out := &in.StorageClassName, &out.StorageClassName 385 | *out = new(string) 386 | **out = **in 387 | } 388 | if in.Replicas != nil { 389 | in, out := &in.Replicas, &out.Replicas 390 | *out = new(int32) 391 | **out = **in 392 | } 393 | } 394 | 395 | // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec. 396 | func (in *StatefulSetSpec) DeepCopy() *StatefulSetSpec { 397 | if in == nil { 398 | return nil 399 | } 400 | out := new(StatefulSetSpec) 401 | in.DeepCopyInto(out) 402 | return out 403 | } 404 | -------------------------------------------------------------------------------- /config/certmanager/certificate.yaml: -------------------------------------------------------------------------------- 1 | # The following manifests contain a self-signed issuer CR and a certificate CR. 2 | # More document can be found at https://docs.cert-manager.io 3 | # WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. 4 | apiVersion: cert-manager.io/v1 5 | kind: Issuer 6 | metadata: 7 | name: selfsigned-issuer 8 | namespace: system 9 | spec: 10 | selfSigned: {} 11 | --- 12 | apiVersion: cert-manager.io/v1 13 | kind: Certificate 14 | metadata: 15 | name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml 16 | namespace: system 17 | spec: 18 | # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize 19 | dnsNames: 20 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc 21 | - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local 22 | issuerRef: 23 | kind: Issuer 24 | name: selfsigned-issuer 25 | secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize 26 | -------------------------------------------------------------------------------- /config/certmanager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - certificate.yaml 3 | 4 | configurations: 5 | - kustomizeconfig.yaml 6 | -------------------------------------------------------------------------------- /config/certmanager/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This configuration is for teaching kustomize how to update name ref and var substitution 2 | nameReference: 3 | - kind: Issuer 4 | group: cert-manager.io 5 | fieldSpecs: 6 | - kind: Certificate 7 | group: cert-manager.io 8 | path: spec/issuerRef/name 9 | 10 | varReference: 11 | - kind: Certificate 12 | group: cert-manager.io 13 | path: spec/commonName 14 | - kind: Certificate 15 | group: cert-manager.io 16 | path: spec/dnsNames 17 | -------------------------------------------------------------------------------- /config/crd/bases/cortex.opstrace.io_cortices.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: apiextensions.k8s.io/v1 4 | kind: CustomResourceDefinition 5 | metadata: 6 | annotations: 7 | controller-gen.kubebuilder.io/version: v0.4.1 8 | creationTimestamp: null 9 | name: cortices.cortex.opstrace.io 10 | spec: 11 | group: cortex.opstrace.io 12 | names: 13 | kind: Cortex 14 | listKind: CortexList 15 | plural: cortices 16 | singular: cortex 17 | scope: Namespaced 18 | versions: 19 | - name: v1alpha1 20 | schema: 21 | openAPIV3Schema: 22 | description: Cortex is the Schema for the cortices API 23 | properties: 24 | apiVersion: 25 | description: 'APIVersion defines the versioned schema of this representation 26 | of an object. Servers should convert recognized schemas to the latest 27 | internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' 28 | type: string 29 | kind: 30 | description: 'Kind is a string value representing the REST resource this 31 | object represents. Servers may infer this from the endpoint the client 32 | submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' 33 | type: string 34 | metadata: 35 | type: object 36 | spec: 37 | description: CortexSpec defines the desired state of Cortex 38 | properties: 39 | alertmanager_spec: 40 | properties: 41 | replicas: 42 | format: int32 43 | type: integer 44 | type: object 45 | compactor_spec: 46 | properties: 47 | datadir_size: 48 | anyOf: 49 | - type: integer 50 | - type: string 51 | pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ 52 | x-kubernetes-int-or-string: true 53 | replicas: 54 | format: int32 55 | type: integer 56 | storage_class_name: 57 | type: string 58 | type: object 59 | config: 60 | description: Config accepts any object, meaning it accepts any valid 61 | Cortex config yaml. Defaulting and Validation are done in the webhooks. 62 | type: object 63 | x-kubernetes-preserve-unknown-fields: true 64 | distributor_spec: 65 | properties: 66 | replicas: 67 | format: int32 68 | type: integer 69 | type: object 70 | image: 71 | description: Image of Cortex to deploy. 72 | type: string 73 | ingester_spec: 74 | properties: 75 | datadir_size: 76 | anyOf: 77 | - type: integer 78 | - type: string 79 | pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ 80 | x-kubernetes-int-or-string: true 81 | replicas: 82 | format: int32 83 | type: integer 84 | storage_class_name: 85 | type: string 86 | type: object 87 | memcached: 88 | properties: 89 | chunks_cache_spec: 90 | properties: 91 | max_item_size: 92 | description: MaxItemSize adjusts max item size 93 | type: string 94 | memory_limit: 95 | description: MemoryLimit is the item memory in megabytes 96 | format: int32 97 | type: integer 98 | replicas: 99 | format: int32 100 | type: integer 101 | type: object 102 | image: 103 | type: string 104 | index_queries_cache_spec: 105 | properties: 106 | max_item_size: 107 | description: MaxItemSize adjusts max item size 108 | type: string 109 | memory_limit: 110 | description: MemoryLimit is the item memory in megabytes 111 | format: int32 112 | type: integer 113 | replicas: 114 | format: int32 115 | type: integer 116 | type: object 117 | index_writes_cache_spec: 118 | properties: 119 | max_item_size: 120 | description: MaxItemSize adjusts max item size 121 | type: string 122 | memory_limit: 123 | description: MemoryLimit is the item memory in megabytes 124 | format: int32 125 | type: integer 126 | replicas: 127 | format: int32 128 | type: integer 129 | type: object 130 | metadata_cache_spec: 131 | properties: 132 | max_item_size: 133 | description: MaxItemSize adjusts max item size 134 | type: string 135 | memory_limit: 136 | description: MemoryLimit is the item memory in megabytes 137 | format: int32 138 | type: integer 139 | replicas: 140 | format: int32 141 | type: integer 142 | type: object 143 | results_cache_spec: 144 | properties: 145 | max_item_size: 146 | description: MaxItemSize adjusts max item size 147 | type: string 148 | memory_limit: 149 | description: MemoryLimit is the item memory in megabytes 150 | format: int32 151 | type: integer 152 | replicas: 153 | format: int32 154 | type: integer 155 | type: object 156 | type: object 157 | querier_spec: 158 | properties: 159 | replicas: 160 | format: int32 161 | type: integer 162 | type: object 163 | query_frontend_spec: 164 | properties: 165 | replicas: 166 | format: int32 167 | type: integer 168 | type: object 169 | ruler_spec: 170 | properties: 171 | replicas: 172 | format: int32 173 | type: integer 174 | type: object 175 | runtime_config: 176 | type: object 177 | x-kubernetes-preserve-unknown-fields: true 178 | service_account_spec: 179 | properties: 180 | annotations: 181 | additionalProperties: 182 | type: string 183 | type: object 184 | image_pull_secrets: 185 | description: 'ImagePullSecrets is a list of references to secrets 186 | in the same namespace to use for pulling any images in pods 187 | that reference this ServiceAccount. More info: https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod' 188 | items: 189 | description: LocalObjectReference contains enough information 190 | to let you locate the referenced object inside the same namespace. 191 | properties: 192 | name: 193 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 194 | TODO: Add other useful fields. apiVersion, kind, uid?' 195 | type: string 196 | type: object 197 | type: array 198 | type: object 199 | store_gateway_spec: 200 | properties: 201 | datadir_size: 202 | anyOf: 203 | - type: integer 204 | - type: string 205 | pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ 206 | x-kubernetes-int-or-string: true 207 | replicas: 208 | format: int32 209 | type: integer 210 | storage_class_name: 211 | type: string 212 | type: object 213 | type: object 214 | status: 215 | description: CortexStatus defines the observed state of Cortex 216 | properties: 217 | alertmanager: 218 | description: LocalObjectReference contains enough information to let 219 | you locate the referenced object inside the same namespace. 220 | properties: 221 | name: 222 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 223 | TODO: Add other useful fields. apiVersion, kind, uid?' 224 | type: string 225 | type: object 226 | compactor: 227 | description: LocalObjectReference contains enough information to let 228 | you locate the referenced object inside the same namespace. 229 | properties: 230 | name: 231 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 232 | TODO: Add other useful fields. apiVersion, kind, uid?' 233 | type: string 234 | type: object 235 | distributor: 236 | description: LocalObjectReference contains enough information to let 237 | you locate the referenced object inside the same namespace. 238 | properties: 239 | name: 240 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 241 | TODO: Add other useful fields. apiVersion, kind, uid?' 242 | type: string 243 | type: object 244 | ingester: 245 | description: LocalObjectReference contains enough information to let 246 | you locate the referenced object inside the same namespace. 247 | properties: 248 | name: 249 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 250 | TODO: Add other useful fields. apiVersion, kind, uid?' 251 | type: string 252 | type: object 253 | memcached: 254 | description: MemcachedReference holds references to all the Memcached 255 | StatefulSets 256 | properties: 257 | chunks_cache: 258 | description: LocalObjectReference contains enough information 259 | to let you locate the referenced object inside the same namespace. 260 | properties: 261 | name: 262 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 263 | TODO: Add other useful fields. apiVersion, kind, uid?' 264 | type: string 265 | type: object 266 | index_queries_cache: 267 | description: LocalObjectReference contains enough information 268 | to let you locate the referenced object inside the same namespace. 269 | properties: 270 | name: 271 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 272 | TODO: Add other useful fields. apiVersion, kind, uid?' 273 | type: string 274 | type: object 275 | index_writes_cache: 276 | description: LocalObjectReference contains enough information 277 | to let you locate the referenced object inside the same namespace. 278 | properties: 279 | name: 280 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 281 | TODO: Add other useful fields. apiVersion, kind, uid?' 282 | type: string 283 | type: object 284 | metadata_cache: 285 | description: LocalObjectReference contains enough information 286 | to let you locate the referenced object inside the same namespace. 287 | properties: 288 | name: 289 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 290 | TODO: Add other useful fields. apiVersion, kind, uid?' 291 | type: string 292 | type: object 293 | results_cache: 294 | description: LocalObjectReference contains enough information 295 | to let you locate the referenced object inside the same namespace. 296 | properties: 297 | name: 298 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 299 | TODO: Add other useful fields. apiVersion, kind, uid?' 300 | type: string 301 | type: object 302 | type: object 303 | querier: 304 | description: LocalObjectReference contains enough information to let 305 | you locate the referenced object inside the same namespace. 306 | properties: 307 | name: 308 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 309 | TODO: Add other useful fields. apiVersion, kind, uid?' 310 | type: string 311 | type: object 312 | query_frontend: 313 | description: LocalObjectReference contains enough information to let 314 | you locate the referenced object inside the same namespace. 315 | properties: 316 | name: 317 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 318 | TODO: Add other useful fields. apiVersion, kind, uid?' 319 | type: string 320 | type: object 321 | ruler: 322 | description: LocalObjectReference contains enough information to let 323 | you locate the referenced object inside the same namespace. 324 | properties: 325 | name: 326 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 327 | TODO: Add other useful fields. apiVersion, kind, uid?' 328 | type: string 329 | type: object 330 | store_gateway: 331 | description: LocalObjectReference contains enough information to let 332 | you locate the referenced object inside the same namespace. 333 | properties: 334 | name: 335 | description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names 336 | TODO: Add other useful fields. apiVersion, kind, uid?' 337 | type: string 338 | type: object 339 | type: object 340 | type: object 341 | served: true 342 | storage: true 343 | subresources: 344 | status: {} 345 | status: 346 | acceptedNames: 347 | kind: "" 348 | plural: "" 349 | conditions: [] 350 | storedVersions: [] 351 | -------------------------------------------------------------------------------- /config/crd/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # This kustomization.yaml is not intended to be run by itself, 2 | # since it depends on service name and namespace that are out of this kustomize package. 3 | # It should be run by config/default 4 | resources: 5 | - bases/cortex.opstrace.io_cortices.yaml 6 | #+kubebuilder:scaffold:crdkustomizeresource 7 | 8 | patchesStrategicMerge: 9 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. 10 | # patches here are for enabling the conversion webhook for each CRD 11 | #- patches/webhook_in_cortices.yaml 12 | #+kubebuilder:scaffold:crdkustomizewebhookpatch 13 | 14 | # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. 15 | # patches here are for enabling the CA injection for each CRD 16 | #- patches/cainjection_in_cortices.yaml 17 | #+kubebuilder:scaffold:crdkustomizecainjectionpatch 18 | 19 | # the following config is for teaching kustomize how to do kustomization for CRDs. 20 | configurations: 21 | - kustomizeconfig.yaml 22 | -------------------------------------------------------------------------------- /config/crd/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # This file is for teaching kustomize how to substitute name and namespace reference in CRD 2 | nameReference: 3 | - kind: Service 4 | version: v1 5 | fieldSpecs: 6 | - kind: CustomResourceDefinition 7 | version: v1 8 | group: apiextensions.k8s.io 9 | path: spec/conversion/webhook/clientConfig/service/name 10 | 11 | namespace: 12 | - kind: CustomResourceDefinition 13 | version: v1 14 | group: apiextensions.k8s.io 15 | path: spec/conversion/webhook/clientConfig/service/namespace 16 | create: false 17 | 18 | varReference: 19 | - path: metadata/annotations 20 | -------------------------------------------------------------------------------- /config/crd/patches/cainjection_in_cortices.yaml: -------------------------------------------------------------------------------- 1 | # The following patch adds a directive for certmanager to inject CA into the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | annotations: 6 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 7 | name: cortices.cortex.opstrace.io 8 | -------------------------------------------------------------------------------- /config/crd/patches/webhook_in_cortices.yaml: -------------------------------------------------------------------------------- 1 | # The following patch enables a conversion webhook for the CRD 2 | apiVersion: apiextensions.k8s.io/v1 3 | kind: CustomResourceDefinition 4 | metadata: 5 | name: cortices.cortex.opstrace.io 6 | spec: 7 | conversion: 8 | strategy: Webhook 9 | webhook: 10 | clientConfig: 11 | service: 12 | namespace: system 13 | name: webhook-service 14 | path: /convert 15 | -------------------------------------------------------------------------------- /config/default/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # Adds namespace to all resources. 2 | namespace: cortex-operator-system 3 | 4 | # Value of this field is prepended to the 5 | # names of all resources, e.g. a deployment named 6 | # "wordpress" becomes "alices-wordpress". 7 | # Note that it should also match with the prefix (text before '-') of the namespace 8 | # field above. 9 | namePrefix: cortex-operator- 10 | 11 | # Labels to add to all resources and selectors. 12 | #commonLabels: 13 | # someName: someValue 14 | 15 | bases: 16 | - ../crd 17 | - ../rbac 18 | - ../manager 19 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 20 | # crd/kustomization.yaml 21 | - ../webhook 22 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. 23 | - ../certmanager 24 | # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 25 | #- ../prometheus 26 | 27 | patchesStrategicMerge: 28 | # Protect the /metrics endpoint by putting it behind auth. 29 | # If you want your controller-manager to expose the /metrics 30 | # endpoint w/o any authn/z, please comment the following line. 31 | - manager_auth_proxy_patch.yaml 32 | 33 | # Mount the controller config file for loading manager configurations 34 | # through a ComponentConfig type 35 | #- manager_config_patch.yaml 36 | 37 | # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 38 | # crd/kustomization.yaml 39 | - manager_webhook_patch.yaml 40 | 41 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 42 | # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. 43 | # 'CERTMANAGER' needs to be enabled to use ca injection 44 | - webhookcainjection_patch.yaml 45 | 46 | # the following config is for teaching kustomize how to do var substitution 47 | vars: 48 | # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. 49 | - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR 50 | objref: 51 | kind: Certificate 52 | group: cert-manager.io 53 | version: v1 54 | name: serving-cert # this name should match the one in certificate.yaml 55 | fieldref: 56 | fieldpath: metadata.namespace 57 | - name: CERTIFICATE_NAME 58 | objref: 59 | kind: Certificate 60 | group: cert-manager.io 61 | version: v1 62 | name: serving-cert # this name should match the one in certificate.yaml 63 | - name: SERVICE_NAMESPACE # namespace of the service 64 | objref: 65 | kind: Service 66 | version: v1 67 | name: webhook-service 68 | fieldref: 69 | fieldpath: metadata.namespace 70 | - name: SERVICE_NAME 71 | objref: 72 | kind: Service 73 | version: v1 74 | name: webhook-service 75 | -------------------------------------------------------------------------------- /config/default/manager_auth_proxy_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch inject a sidecar container which is a HTTP proxy for the 2 | # controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: controller-manager 7 | namespace: system 8 | spec: 9 | template: 10 | spec: 11 | containers: 12 | - name: kube-rbac-proxy 13 | image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 14 | args: 15 | - "--secure-listen-address=0.0.0.0:8443" 16 | - "--upstream=http://127.0.0.1:8080/" 17 | - "--logtostderr=true" 18 | - "--v=10" 19 | ports: 20 | - containerPort: 8443 21 | name: https 22 | - name: manager 23 | args: 24 | - "--health-probe-bind-address=:8081" 25 | - "--metrics-bind-address=127.0.0.1:8080" 26 | - "--leader-elect" 27 | -------------------------------------------------------------------------------- /config/default/manager_config_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | args: 12 | - "--config=controller_manager_config.yaml" 13 | volumeMounts: 14 | - name: manager-config 15 | mountPath: /controller_manager_config.yaml 16 | subPath: controller_manager_config.yaml 17 | volumes: 18 | - name: manager-config 19 | configMap: 20 | name: manager-config 21 | -------------------------------------------------------------------------------- /config/default/manager_webhook_patch.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | spec: 7 | template: 8 | spec: 9 | containers: 10 | - name: manager 11 | ports: 12 | - containerPort: 9443 13 | name: webhook-server 14 | protocol: TCP 15 | volumeMounts: 16 | - mountPath: /tmp/k8s-webhook-server/serving-certs 17 | name: cert 18 | readOnly: true 19 | volumes: 20 | - name: cert 21 | secret: 22 | defaultMode: 420 23 | secretName: webhook-server-cert 24 | -------------------------------------------------------------------------------- /config/default/webhookcainjection_patch.yaml: -------------------------------------------------------------------------------- 1 | # This patch add annotation to admission webhook config and 2 | # the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. 3 | apiVersion: admissionregistration.k8s.io/v1beta1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | name: mutating-webhook-configuration 7 | annotations: 8 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 9 | --- 10 | apiVersion: admissionregistration.k8s.io/v1beta1 11 | kind: ValidatingWebhookConfiguration 12 | metadata: 13 | name: validating-webhook-configuration 14 | annotations: 15 | cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) 16 | -------------------------------------------------------------------------------- /config/manager/controller_manager_config.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 2 | kind: ControllerManagerConfig 3 | health: 4 | healthProbeBindAddress: :8081 5 | metrics: 6 | bindAddress: 127.0.0.1:8080 7 | webhook: 8 | port: 9443 9 | leaderElection: 10 | leaderElect: true 11 | resourceName: 5b355f38.opstrace.com 12 | -------------------------------------------------------------------------------- /config/manager/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manager.yaml 3 | 4 | generatorOptions: 5 | disableNameSuffixHash: true 6 | 7 | configMapGenerator: 8 | - files: 9 | - controller_manager_config.yaml 10 | name: manager-config 11 | apiVersion: kustomize.config.k8s.io/v1beta1 12 | kind: Kustomization 13 | images: 14 | - name: controller 15 | newName: opstrace/cortex-operator 16 | newTag: "latest" 17 | -------------------------------------------------------------------------------- /config/manager/manager.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: system 7 | --- 8 | apiVersion: apps/v1 9 | kind: Deployment 10 | metadata: 11 | name: controller-manager 12 | namespace: system 13 | labels: 14 | control-plane: controller-manager 15 | spec: 16 | selector: 17 | matchLabels: 18 | control-plane: controller-manager 19 | replicas: 1 20 | template: 21 | metadata: 22 | labels: 23 | control-plane: controller-manager 24 | spec: 25 | securityContext: 26 | runAsNonRoot: true 27 | containers: 28 | - command: 29 | - /manager 30 | args: 31 | - --leader-elect 32 | image: controller:latest 33 | name: manager 34 | securityContext: 35 | allowPrivilegeEscalation: false 36 | livenessProbe: 37 | httpGet: 38 | path: /healthz 39 | port: 8081 40 | initialDelaySeconds: 15 41 | periodSeconds: 20 42 | readinessProbe: 43 | httpGet: 44 | path: /readyz 45 | port: 8081 46 | initialDelaySeconds: 5 47 | periodSeconds: 10 48 | resources: 49 | limits: 50 | cpu: 100m 51 | memory: 30Mi 52 | requests: 53 | cpu: 100m 54 | memory: 20Mi 55 | serviceAccountName: controller-manager 56 | terminationGracePeriodSeconds: 10 57 | -------------------------------------------------------------------------------- /config/prometheus/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - monitor.yaml 3 | -------------------------------------------------------------------------------- /config/prometheus/monitor.yaml: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus Monitor Service (Metrics) 3 | apiVersion: monitoring.coreos.com/v1 4 | kind: ServiceMonitor 5 | metadata: 6 | labels: 7 | control-plane: controller-manager 8 | name: controller-manager-metrics-monitor 9 | namespace: system 10 | spec: 11 | endpoints: 12 | - path: /metrics 13 | port: https 14 | scheme: https 15 | bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token 16 | tlsConfig: 17 | insecureSkipVerify: true 18 | selector: 19 | matchLabels: 20 | control-plane: controller-manager 21 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_client_clusterrole.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: metrics-reader 5 | rules: 6 | - nonResourceURLs: 7 | - "/metrics" 8 | verbs: 9 | - get 10 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRole 3 | metadata: 4 | name: proxy-role 5 | rules: 6 | - apiGroups: 7 | - authentication.k8s.io 8 | resources: 9 | - tokenreviews 10 | verbs: 11 | - create 12 | - apiGroups: 13 | - authorization.k8s.io 14 | resources: 15 | - subjectaccessreviews 16 | verbs: 17 | - create 18 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: proxy-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: proxy-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/auth_proxy_service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | control-plane: controller-manager 6 | name: controller-manager-metrics-service 7 | namespace: system 8 | spec: 9 | ports: 10 | - name: https 11 | port: 8443 12 | targetPort: https 13 | selector: 14 | control-plane: controller-manager 15 | -------------------------------------------------------------------------------- /config/rbac/cortex_editor_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to edit cortices. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: cortex-editor-role 6 | rules: 7 | - apiGroups: 8 | - cortex.opstrace.io 9 | resources: 10 | - cortices 11 | verbs: 12 | - create 13 | - delete 14 | - get 15 | - list 16 | - patch 17 | - update 18 | - watch 19 | - apiGroups: 20 | - cortex.opstrace.io 21 | resources: 22 | - cortices/status 23 | verbs: 24 | - get 25 | -------------------------------------------------------------------------------- /config/rbac/cortex_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions for end users to view cortices. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: cortex-viewer-role 6 | rules: 7 | - apiGroups: 8 | - cortex.opstrace.io 9 | resources: 10 | - cortices 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - apiGroups: 16 | - cortex.opstrace.io 17 | resources: 18 | - cortices/status 19 | verbs: 20 | - get 21 | -------------------------------------------------------------------------------- /config/rbac/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | # All RBAC will be applied under this service account in 3 | # the deployment namespace. You may comment out this resource 4 | # if your manager will use a service account that exists at 5 | # runtime. Be sure to update RoleBinding and ClusterRoleBinding 6 | # subjects if changing service account names. 7 | - service_account.yaml 8 | - role.yaml 9 | - role_binding.yaml 10 | - leader_election_role.yaml 11 | - leader_election_role_binding.yaml 12 | # Comment the following 4 lines if you want to disable 13 | # the auth proxy (https://github.com/brancz/kube-rbac-proxy) 14 | # which protects your /metrics endpoint. 15 | - auth_proxy_service.yaml 16 | - auth_proxy_role.yaml 17 | - auth_proxy_role_binding.yaml 18 | - auth_proxy_client_clusterrole.yaml 19 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role.yaml: -------------------------------------------------------------------------------- 1 | # permissions to do leader election. 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: Role 4 | metadata: 5 | name: leader-election-role 6 | rules: 7 | - apiGroups: 8 | - "" 9 | resources: 10 | - configmaps 11 | verbs: 12 | - get 13 | - list 14 | - watch 15 | - create 16 | - update 17 | - patch 18 | - delete 19 | - apiGroups: 20 | - coordination.k8s.io 21 | resources: 22 | - leases 23 | verbs: 24 | - get 25 | - list 26 | - watch 27 | - create 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - events 35 | verbs: 36 | - create 37 | - patch 38 | -------------------------------------------------------------------------------- /config/rbac/leader_election_role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: RoleBinding 3 | metadata: 4 | name: leader-election-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: Role 8 | name: leader-election-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/role.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | kind: ClusterRole 5 | metadata: 6 | creationTimestamp: null 7 | name: manager-role 8 | rules: 9 | - apiGroups: 10 | - "" 11 | resources: 12 | - configmaps 13 | verbs: 14 | - create 15 | - delete 16 | - get 17 | - list 18 | - patch 19 | - scope=Cluster 20 | - update 21 | - watch 22 | - apiGroups: 23 | - "" 24 | resources: 25 | - serviceaccounts 26 | verbs: 27 | - create 28 | - delete 29 | - get 30 | - list 31 | - patch 32 | - scope=Cluster 33 | - update 34 | - watch 35 | - apiGroups: 36 | - "" 37 | resources: 38 | - services 39 | verbs: 40 | - create 41 | - delete 42 | - get 43 | - list 44 | - patch 45 | - scope=Cluster 46 | - update 47 | - watch 48 | - apiGroups: 49 | - apps 50 | resources: 51 | - deployments 52 | verbs: 53 | - create 54 | - delete 55 | - get 56 | - list 57 | - patch 58 | - scope=Cluster 59 | - update 60 | - watch 61 | - apiGroups: 62 | - apps 63 | resources: 64 | - statefulsets 65 | verbs: 66 | - create 67 | - delete 68 | - get 69 | - list 70 | - patch 71 | - scope=Cluster 72 | - update 73 | - watch 74 | - apiGroups: 75 | - cortex.opstrace.io 76 | resources: 77 | - cortices 78 | verbs: 79 | - create 80 | - delete 81 | - get 82 | - list 83 | - patch 84 | - update 85 | - watch 86 | - apiGroups: 87 | - cortex.opstrace.io 88 | resources: 89 | - cortices/finalizers 90 | verbs: 91 | - update 92 | - apiGroups: 93 | - cortex.opstrace.io 94 | resources: 95 | - cortices/status 96 | verbs: 97 | - get 98 | - patch 99 | - update 100 | -------------------------------------------------------------------------------- /config/rbac/role_binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: manager-rolebinding 5 | roleRef: 6 | apiGroup: rbac.authorization.k8s.io 7 | kind: ClusterRole 8 | name: manager-role 9 | subjects: 10 | - kind: ServiceAccount 11 | name: controller-manager 12 | namespace: system 13 | -------------------------------------------------------------------------------- /config/rbac/service_account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: controller-manager 5 | namespace: system 6 | -------------------------------------------------------------------------------- /config/samples/cortex_v1alpha1_cortex.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: cortex.opstrace.io/v1alpha1 2 | kind: Cortex 3 | metadata: 4 | name: cortex-sample 5 | spec: 6 | image: "cortexproject/cortex:v1.9.0" 7 | 8 | config: 9 | server: 10 | grpc_server_max_recv_msg_size: 41943040 11 | grpc_server_max_send_msg_size: 41943040 12 | memberlist: 13 | max_join_backoff: 1m 14 | max_join_retries: 20 15 | min_join_backoff: 1s 16 | query_range: 17 | split_queries_by_interval: 24h 18 | limits: 19 | compactor_blocks_retention_period: 192h 20 | ingestion_rate: 100000 21 | ingestion_rate_strategy: local 22 | ingestion_burst_size: 200000 23 | max_global_series_per_user: 10000000 24 | max_series_per_user: 5000000 25 | accept_ha_samples: true 26 | ha_cluster_label: prometheus 27 | ha_replica_label: prometheus_replica 28 | ruler_tenant_shard_size: 3 29 | ingester: 30 | lifecycler: 31 | join_after: 30s 32 | observe_period: 30s 33 | num_tokens: 512 34 | blocks_storage: 35 | tsdb: 36 | retention_period: 6h 37 | backend: s3 38 | s3: 39 | bucket_name: cortex-operator-example-209f-data 40 | endpoint: s3.us-west-2.amazonaws.com 41 | configs: 42 | database: 43 | uri: https://someuri.com 44 | migrations_dir: /migrations 45 | alertmanager_storage: 46 | backend: s3 47 | s3: 48 | bucket_name: cortex-operator-example-209f-config 49 | endpoint: s3.us-west-2.amazonaws.com 50 | ruler_storage: 51 | backend: s3 52 | s3: 53 | bucket_name: cortex-operator-example-209f-config 54 | endpoint: s3.us-west-2.amazonaws.com 55 | -------------------------------------------------------------------------------- /config/webhook/kustomization.yaml: -------------------------------------------------------------------------------- 1 | resources: 2 | - manifests.v1beta1.yaml 3 | - service.yaml 4 | 5 | configurations: 6 | - kustomizeconfig.yaml 7 | -------------------------------------------------------------------------------- /config/webhook/kustomizeconfig.yaml: -------------------------------------------------------------------------------- 1 | # the following config is for teaching kustomize where to look at when substituting vars. 2 | # It requires kustomize v2.1.0 or newer to work properly. 3 | nameReference: 4 | - kind: Service 5 | version: v1 6 | fieldSpecs: 7 | - kind: MutatingWebhookConfiguration 8 | group: admissionregistration.k8s.io 9 | path: webhooks/clientConfig/service/name 10 | - kind: ValidatingWebhookConfiguration 11 | group: admissionregistration.k8s.io 12 | path: webhooks/clientConfig/service/name 13 | 14 | namespace: 15 | - kind: MutatingWebhookConfiguration 16 | group: admissionregistration.k8s.io 17 | path: webhooks/clientConfig/service/namespace 18 | create: true 19 | - kind: ValidatingWebhookConfiguration 20 | group: admissionregistration.k8s.io 21 | path: webhooks/clientConfig/service/namespace 22 | create: true 23 | 24 | varReference: 25 | - path: metadata/annotations 26 | -------------------------------------------------------------------------------- /config/webhook/manifests.v1beta1.yaml: -------------------------------------------------------------------------------- 1 | 2 | --- 3 | apiVersion: admissionregistration.k8s.io/v1beta1 4 | kind: MutatingWebhookConfiguration 5 | metadata: 6 | creationTimestamp: null 7 | name: mutating-webhook-configuration 8 | webhooks: 9 | - admissionReviewVersions: 10 | - v1 11 | - v1beta1 12 | clientConfig: 13 | service: 14 | name: webhook-service 15 | namespace: system 16 | path: /mutate-cortex-opstrace-io-v1alpha1-cortex 17 | failurePolicy: Fail 18 | name: mcortex.kb.io 19 | rules: 20 | - apiGroups: 21 | - cortex.opstrace.io 22 | apiVersions: 23 | - v1alpha1 24 | operations: 25 | - CREATE 26 | - UPDATE 27 | resources: 28 | - cortices 29 | sideEffects: None 30 | 31 | --- 32 | apiVersion: admissionregistration.k8s.io/v1beta1 33 | kind: ValidatingWebhookConfiguration 34 | metadata: 35 | creationTimestamp: null 36 | name: validating-webhook-configuration 37 | webhooks: 38 | - admissionReviewVersions: 39 | - v1 40 | - v1beta1 41 | clientConfig: 42 | service: 43 | name: webhook-service 44 | namespace: system 45 | path: /validate-cortex-opstrace-io-v1alpha1-cortex 46 | failurePolicy: Fail 47 | name: vcortex.kb.io 48 | rules: 49 | - apiGroups: 50 | - cortex.opstrace.io 51 | apiVersions: 52 | - v1alpha1 53 | operations: 54 | - CREATE 55 | - UPDATE 56 | resources: 57 | - cortices 58 | sideEffects: None 59 | -------------------------------------------------------------------------------- /config/webhook/service.yaml: -------------------------------------------------------------------------------- 1 | 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: webhook-service 6 | namespace: system 7 | spec: 8 | ports: 9 | - port: 443 10 | targetPort: 9443 11 | selector: 12 | control-plane: controller-manager 13 | -------------------------------------------------------------------------------- /controllers/cortex_alertmanager_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/go-logr/logr" 24 | appsv1 "k8s.io/api/apps/v1" 25 | corev1 "k8s.io/api/core/v1" 26 | apierrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | "k8s.io/apimachinery/pkg/util/intstr" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/client" 32 | 33 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 34 | ) 35 | 36 | const AlertManagerName = "alertmanager" 37 | 38 | // CortexAlertManagerReconciler reconciles a Cortex object and ensures the Cortex 39 | // Alert Manager is deployed 40 | type CortexAlertManagerReconciler struct { 41 | client.Client 42 | Log logr.Logger 43 | Scheme *runtime.Scheme 44 | } 45 | 46 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 47 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 48 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 49 | 50 | // 51 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 52 | // 53 | 54 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 55 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 56 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 57 | 58 | // 59 | // For more details, check Reconcile and its Result here: 60 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 61 | func (r *CortexAlertManagerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 62 | log := r.Log.WithValues("cortex", req.NamespacedName) 63 | 64 | cortex := &cortexv1alpha1.Cortex{} 65 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 66 | if !apierrors.IsNotFound(err) { 67 | log.Error(err, "unable to fetch Cortex") 68 | } 69 | // Ignore not-found errors, since they can't be fixed by an immediate 70 | // requeue (we'll need to wait for a new notification), and we can get 71 | // them on deleted requests. 72 | return ctrl.Result{}, client.IgnoreNotFound(err) 73 | } 74 | 75 | krr := KubernetesResourceReconciler{ 76 | scheme: r.Scheme, 77 | client: r.Client, 78 | cortex: cortex, 79 | log: log, 80 | } 81 | 82 | svc := NewAlertManagerService(req) 83 | err := krr.Reconcile(ctx, svc) 84 | if err != nil { 85 | return ctrl.Result{}, err 86 | } 87 | 88 | deploy := NewAlertManagerDeployment(req, cortex, cortex.Spec.AlertManagerSpec) 89 | cortex.Status.AlertManagerRef = deploy.ref 90 | err = krr.Reconcile(ctx, deploy) 91 | if err != nil { 92 | return ctrl.Result{}, err 93 | } 94 | 95 | return ctrl.Result{}, nil 96 | } 97 | 98 | // SetupWithManager sets up the controller with the Manager. 99 | func (r *CortexAlertManagerReconciler) SetupWithManager(mgr ctrl.Manager) error { 100 | return ctrl.NewControllerManagedBy(mgr). 101 | For(&cortexv1alpha1.Cortex{}). 102 | Complete(r) 103 | } 104 | 105 | func NewAlertManagerService(req ctrl.Request) *KubernetesResource { 106 | svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: AlertManagerName, Namespace: req.Namespace}} 107 | ref := &corev1.LocalObjectReference{Name: AlertManagerName} 108 | return &KubernetesResource{ 109 | obj: svc, 110 | ref: ref, 111 | mutator: func() error { 112 | svc.Labels = map[string]string{ 113 | "name": AlertManagerName, 114 | "job": fmt.Sprintf("%s.%s", req.Namespace, AlertManagerName), 115 | } 116 | svc.Spec.Ports = make([]corev1.ServicePort, 0) 117 | svc.Spec.Ports = []corev1.ServicePort{ 118 | { 119 | Name: "http", 120 | Port: 80, 121 | TargetPort: intstr.FromInt(80), 122 | }, 123 | { 124 | Name: AlertManagerName, 125 | Port: 9094, 126 | TargetPort: intstr.FromInt(9094), 127 | }, 128 | } 129 | svc.Spec.Selector = map[string]string{"name": AlertManagerName} 130 | 131 | return nil 132 | }, 133 | } 134 | } 135 | 136 | func NewAlertManagerDeployment( 137 | req ctrl.Request, 138 | cortex *cortexv1alpha1.Cortex, 139 | spec *cortexv1alpha1.DeploymentSpec, 140 | ) *KubernetesResource { 141 | deploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: AlertManagerName, Namespace: req.Namespace}} 142 | labels := map[string]string{ 143 | "name": AlertManagerName, 144 | } 145 | annotations := map[string]string{ 146 | CortexConfigShasumAnnotationName: cortex.Spec.ConfigSHA(), 147 | } 148 | ref := &corev1.LocalObjectReference{Name: AlertManagerName} 149 | 150 | configMapName := cortex.Name + CortexConfigMapNameSuffix 151 | runtimeConfigMapName := cortex.Name + CortexRuntimeConfigMapNameSuffix 152 | 153 | return &KubernetesResource{ 154 | obj: deploy, 155 | ref: ref, 156 | mutator: func() error { 157 | deploy.Spec.Replicas = spec.Replicas 158 | deploy.Spec.Selector = &metav1.LabelSelector{ 159 | MatchLabels: labels, 160 | } 161 | deploy.Spec.Template.Labels = labels 162 | deploy.Spec.Template.Annotations = annotations 163 | deploy.Spec.Template.Spec.Affinity = WithPodAntiAffinity(AlertManagerName) 164 | deploy.Spec.Template.Spec.Containers = []corev1.Container{ 165 | { 166 | Name: AlertManagerName, 167 | Image: cortex.Spec.Image, 168 | ImagePullPolicy: corev1.PullIfNotPresent, 169 | Args: []string{ 170 | "-target=" + AlertManagerName, 171 | "-config.file=/etc/cortex/config.yaml", 172 | }, 173 | Ports: []corev1.ContainerPort{ 174 | { 175 | Name: "http", 176 | ContainerPort: 80, 177 | }, 178 | { 179 | Name: AlertManagerName, 180 | ContainerPort: 9094, 181 | }, 182 | }, 183 | VolumeMounts: []corev1.VolumeMount{ 184 | { 185 | MountPath: "/etc/cortex", 186 | Name: "cortex", 187 | }, 188 | }, 189 | }, 190 | } 191 | deploy.Spec.Template.Spec.ServiceAccountName = ServiceAccountName 192 | deploy.Spec.Template.Spec.Volumes = []corev1.Volume{ 193 | { 194 | Name: "cortex", 195 | VolumeSource: corev1.VolumeSource{ 196 | Projected: &corev1.ProjectedVolumeSource{ 197 | Sources: []corev1.VolumeProjection{ 198 | { 199 | ConfigMap: &corev1.ConfigMapProjection{ 200 | LocalObjectReference: corev1.LocalObjectReference{ 201 | Name: configMapName, 202 | }, 203 | }, 204 | }, 205 | { 206 | ConfigMap: &corev1.ConfigMapProjection{ 207 | LocalObjectReference: corev1.LocalObjectReference{ 208 | Name: runtimeConfigMapName, 209 | }, 210 | }, 211 | }, 212 | }, 213 | }, 214 | }, 215 | }, 216 | } 217 | 218 | return nil 219 | }, 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /controllers/cortex_compactor_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | apierrors "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ctrl "sigs.k8s.io/controller-runtime" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | 28 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 29 | ) 30 | 31 | // CortexCompactorReconciler reconciles a Cortex object and ensures the Cortex 32 | // Compactor is deployed 33 | type CortexCompactorReconciler struct { 34 | client.Client 35 | Log logr.Logger 36 | Scheme *runtime.Scheme 37 | } 38 | 39 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 40 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 41 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 42 | 43 | // 44 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 45 | // 46 | 47 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 48 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 49 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 50 | 51 | // 52 | // For more details, check Reconcile and its Result here: 53 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 54 | func (r *CortexCompactorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 55 | log := r.Log.WithValues("cortex", req.NamespacedName) 56 | 57 | cortex := &cortexv1alpha1.Cortex{} 58 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 59 | if !apierrors.IsNotFound(err) { 60 | log.Error(err, "unable to fetch Cortex") 61 | } 62 | // Ignore not-found errors, since they can't be fixed by an immediate 63 | // requeue (we'll need to wait for a new notification), and we can get 64 | // them on deleted requests. 65 | return ctrl.Result{}, client.IgnoreNotFound(err) 66 | } 67 | 68 | krr := KubernetesResourceReconciler{ 69 | scheme: r.Scheme, 70 | client: r.Client, 71 | cortex: cortex, 72 | log: log, 73 | } 74 | 75 | svc := NewService(req, "compactor") 76 | err := krr.Reconcile(ctx, svc) 77 | if err != nil { 78 | return ctrl.Result{}, err 79 | } 80 | 81 | sts := NewStatefulSet(req, "compactor", cortex, cortex.Spec.CompactorSpec) 82 | cortex.Status.CompactorRef = sts.ref 83 | err = krr.Reconcile(ctx, sts) 84 | if err != nil { 85 | return ctrl.Result{}, err 86 | } 87 | 88 | return ctrl.Result{}, nil 89 | } 90 | 91 | // SetupWithManager sets up the controller with the Manager. 92 | func (r *CortexCompactorReconciler) SetupWithManager(mgr ctrl.Manager) error { 93 | return ctrl.NewControllerManagedBy(mgr). 94 | For(&cortexv1alpha1.Cortex{}). 95 | Complete(r) 96 | } 97 | -------------------------------------------------------------------------------- /controllers/cortex_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | corev1 "k8s.io/api/core/v1" 24 | apierrors "k8s.io/apimachinery/pkg/api/errors" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | "k8s.io/apimachinery/pkg/util/intstr" 28 | ctrl "sigs.k8s.io/controller-runtime" 29 | "sigs.k8s.io/controller-runtime/pkg/client" 30 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 31 | 32 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 33 | ) 34 | 35 | // CortexReconciler reconciles a Cortex object 36 | type CortexReconciler struct { 37 | client.Client 38 | Log logr.Logger 39 | Scheme *runtime.Scheme 40 | } 41 | 42 | type kubernetesResource struct { 43 | obj client.Object 44 | mutator controllerutil.MutateFn 45 | } 46 | 47 | const FinalizerName = "cortex.opstrace.io/finalizer" 48 | const ServiceAccountName = "cortex" 49 | const CortexConfigShasumAnnotationName = "cortex-operator/cortex-config-shasum" 50 | const CortexConfigMapNameSuffix = "-config" 51 | const CortexRuntimeConfigMapNameSuffix = "-runtime-config" 52 | const GossipRingServiceName = "gossip-ring" 53 | 54 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 55 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 56 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 57 | 58 | // 59 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 60 | // 61 | 62 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 63 | //+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 64 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 65 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 66 | //+kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 67 | 68 | // 69 | // For more details, check Reconcile and its Result here: 70 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 71 | func (r *CortexReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 72 | log := r.Log.WithValues("cortex", req.NamespacedName) 73 | 74 | cortex := &cortexv1alpha1.Cortex{} 75 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 76 | if !apierrors.IsNotFound(err) { 77 | log.Error(err, "unable to fetch Cortex") 78 | } 79 | // Ignore not-found errors, since they can't be fixed by an immediate 80 | // requeue (we'll need to wait for a new notification), and we can get 81 | // them on deleted requests. 82 | return ctrl.Result{}, client.IgnoreNotFound(err) 83 | } 84 | 85 | krr := KubernetesResourceReconciler{ 86 | scheme: r.Scheme, 87 | client: r.Client, 88 | cortex: cortex, 89 | log: log, 90 | } 91 | 92 | cm := NewCortexConfigMap(req, cortex) 93 | err := krr.Reconcile(ctx, cm) 94 | if err != nil { 95 | return ctrl.Result{}, err 96 | } 97 | 98 | sa := NewServiceAccount(req, cortex) 99 | err = krr.Reconcile(ctx, sa) 100 | if err != nil { 101 | return ctrl.Result{}, err 102 | } 103 | 104 | svc := NewGossipRingService(req) 105 | err = krr.Reconcile(ctx, svc) 106 | if err != nil { 107 | return ctrl.Result{}, err 108 | } 109 | 110 | return ctrl.Result{}, nil 111 | } 112 | 113 | // SetupWithManager sets up the controller with the Manager. 114 | func (r *CortexReconciler) SetupWithManager(mgr ctrl.Manager) error { 115 | return ctrl.NewControllerManagedBy(mgr). 116 | For(&cortexv1alpha1.Cortex{}). 117 | Complete(r) 118 | } 119 | 120 | func NewCortexConfigMap( 121 | req ctrl.Request, 122 | cortex *cortexv1alpha1.Cortex, 123 | ) *KubernetesResource { 124 | name := cortex.Name + CortexConfigMapNameSuffix 125 | // Validation errors were handled by the webhooks. 126 | y, _ := cortex.AsYAML() 127 | configMap := &corev1.ConfigMap{ 128 | ObjectMeta: metav1.ObjectMeta{ 129 | Name: name, 130 | Namespace: req.Namespace, 131 | }, 132 | } 133 | 134 | return &KubernetesResource{ 135 | obj: configMap, 136 | mutator: func() error { 137 | configMap.Data = map[string]string{ 138 | "config.yaml": string(y), 139 | } 140 | return nil 141 | }, 142 | } 143 | } 144 | 145 | func NewServiceAccount(req ctrl.Request, cortex *cortexv1alpha1.Cortex) *KubernetesResource { 146 | serviceAccount := &corev1.ServiceAccount{ 147 | ObjectMeta: metav1.ObjectMeta{ 148 | Name: ServiceAccountName, 149 | Namespace: req.Namespace, 150 | }, 151 | } 152 | 153 | return &KubernetesResource{ 154 | obj: serviceAccount, 155 | mutator: func() error { 156 | // nothing to do if nothing is set 157 | if cortex.Spec.ServiceAccountSpec == nil { 158 | return nil 159 | } 160 | 161 | serviceAccount.Annotations = cortex.Spec.ServiceAccountSpec.Annotations 162 | if len(cortex.Spec.ServiceAccountSpec.ImagePullSecrets) > 0 { 163 | serviceAccount.ImagePullSecrets = make( 164 | []corev1.LocalObjectReference, 165 | len(cortex.Spec.ServiceAccountSpec.ImagePullSecrets), 166 | ) 167 | copy( 168 | serviceAccount.ImagePullSecrets, 169 | cortex.Spec.ServiceAccountSpec.ImagePullSecrets, 170 | ) 171 | } 172 | 173 | return nil 174 | }, 175 | } 176 | } 177 | 178 | func NewGossipRingService(req ctrl.Request) *KubernetesResource { 179 | service := &corev1.Service{ 180 | ObjectMeta: metav1.ObjectMeta{ 181 | Name: GossipRingServiceName, 182 | Namespace: req.Namespace, 183 | }, 184 | } 185 | 186 | return &KubernetesResource{ 187 | obj: service, 188 | mutator: func() error { 189 | service.Labels = map[string]string{ 190 | "name": GossipRingServiceName, 191 | } 192 | service.Spec.Ports = make([]corev1.ServicePort, 0) 193 | service.Spec.ClusterIP = "None" 194 | service.Spec.Ports = []corev1.ServicePort{ 195 | { 196 | Name: GossipRingServiceName, 197 | Port: int32(7946), 198 | TargetPort: intstr.FromInt(7946), 199 | }, 200 | } 201 | service.Spec.Selector = map[string]string{"memberlist": GossipRingServiceName} 202 | return nil 203 | }, 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /controllers/cortex_controller_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "io/ioutil" 22 | "path/filepath" 23 | "time" 24 | 25 | . "github.com/onsi/ginkgo" 26 | . "github.com/onsi/ginkgo/extensions/table" 27 | . "github.com/onsi/gomega" 28 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 29 | corev1 "k8s.io/api/core/v1" 30 | "k8s.io/apimachinery/pkg/runtime/serializer" 31 | "k8s.io/apimachinery/pkg/types" 32 | "sigs.k8s.io/controller-runtime/pkg/client" 33 | ) 34 | 35 | var _ = Describe("Cortex controller", func() { 36 | const ( 37 | CortexName = "test-cortex" 38 | configMapName = CortexName + CortexConfigMapNameSuffix 39 | CortexNamespace = "default" 40 | 41 | timeout = time.Second * 10 42 | duration = time.Second * 10 43 | interval = time.Millisecond * 250 44 | ) 45 | 46 | Context("should manage resouces necessary for a cortex deployment ", func() { 47 | 48 | It("should allow to create a resource", func() { 49 | ctx := context.Background() 50 | manifest := filepath.Join("..", "config", "samples", "cortex_v1alpha1_cortex.yaml") 51 | testCortex := GetCortexTestSample(manifest) 52 | testCortex.Name = CortexName 53 | Expect(k8sClient.Create(ctx, testCortex)).Should(Succeed()) 54 | }) 55 | 56 | DescribeTable("should create resources necessary for a cortex deployment", 57 | func(lookupKey types.NamespacedName, createdObj client.Object, validator func(createdObj client.Object)) { 58 | ctx := context.Background() 59 | Eventually(func() error { 60 | return k8sClient.Get(ctx, lookupKey, createdObj) 61 | }, timeout, interval).Should(Succeed()) 62 | 63 | validator(createdObj) 64 | }, 65 | Entry( 66 | "creates a config map with cortex configuration", 67 | types.NamespacedName{Name: configMapName, Namespace: CortexNamespace}, 68 | &corev1.ConfigMap{}, 69 | func(createdObj client.Object) { 70 | _, ok := createdObj.(*corev1.ConfigMap) 71 | 72 | Expect(ok).To(BeTrue()) 73 | // TODO(sreis): validate config map without webhook defaulting? 74 | }, 75 | ), 76 | Entry( 77 | "creates a service account", 78 | types.NamespacedName{Name: ServiceAccountName, Namespace: CortexNamespace}, 79 | &corev1.ServiceAccount{}, 80 | func(createdObj client.Object) { 81 | _, ok := createdObj.(*corev1.ServiceAccount) 82 | Expect(ok).To(BeTrue()) 83 | // nothing else to check, service account only has a name 84 | // and a namespace 85 | }, 86 | ), 87 | ) 88 | }) 89 | }) 90 | 91 | func GetCortexTestSample(filepath string) *cortexv1alpha1.Cortex { 92 | b, err := ioutil.ReadFile(filepath) 93 | Expect(err).ToNot(HaveOccurred()) 94 | 95 | deserializer := serializer.NewCodecFactory(k8sClientScheme).UniversalDeserializer() 96 | obj, _, err := deserializer.Decode(b, nil, nil) 97 | Expect(err).ToNot(HaveOccurred()) 98 | Expect(obj).To(BeAssignableToTypeOf(&cortexv1alpha1.Cortex{})) 99 | 100 | c := obj.(*cortexv1alpha1.Cortex) 101 | c.Namespace = "default" 102 | 103 | return c 104 | } 105 | -------------------------------------------------------------------------------- /controllers/cortex_default_config.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | const CortexConfigTemplate = ` 20 | ` 21 | -------------------------------------------------------------------------------- /controllers/cortex_distributor_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | apierrors "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ctrl "sigs.k8s.io/controller-runtime" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | 28 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 29 | ) 30 | 31 | // CortexDistributorReconciler reconciles a Cortex object and ensures the Cortex 32 | // Distributors are deployed 33 | type CortexDistributorReconciler struct { 34 | client.Client 35 | Log logr.Logger 36 | Scheme *runtime.Scheme 37 | } 38 | 39 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 40 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 41 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 42 | 43 | // 44 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 45 | // 46 | 47 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 48 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 49 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 50 | 51 | // 52 | // For more details, check Reconcile and its Result here: 53 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 54 | func (r *CortexDistributorReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 55 | log := r.Log.WithValues("cortex", req.NamespacedName) 56 | 57 | cortex := &cortexv1alpha1.Cortex{} 58 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 59 | if !apierrors.IsNotFound(err) { 60 | log.Error(err, "unable to fetch Cortex") 61 | } 62 | // Ignore not-found errors, since they can't be fixed by an immediate 63 | // requeue (we'll need to wait for a new notification), and we can get 64 | // them on deleted requests. 65 | return ctrl.Result{}, client.IgnoreNotFound(err) 66 | } 67 | 68 | if cortex.Status.IngesterRef == nil { 69 | log.Info("waiting for ingester") 70 | return ctrl.Result{}, nil 71 | } 72 | 73 | krr := KubernetesResourceReconciler{ 74 | scheme: r.Scheme, 75 | client: r.Client, 76 | cortex: cortex, 77 | log: log, 78 | } 79 | 80 | svc := NewService(req, "distributor") 81 | err := krr.Reconcile(ctx, svc) 82 | if err != nil { 83 | return ctrl.Result{}, err 84 | } 85 | 86 | deploy := NewDeployment(req, "distributor", cortex, cortex.Spec.DistributorSpec) 87 | cortex.Status.DistributorRef = deploy.ref 88 | err = krr.Reconcile(ctx, deploy) 89 | if err != nil { 90 | return ctrl.Result{}, err 91 | } 92 | 93 | return ctrl.Result{}, nil 94 | } 95 | 96 | // SetupWithManager sets up the controller with the Manager. 97 | func (r *CortexDistributorReconciler) SetupWithManager(mgr ctrl.Manager) error { 98 | return ctrl.NewControllerManagedBy(mgr). 99 | For(&cortexv1alpha1.Cortex{}). 100 | Complete(r) 101 | } 102 | -------------------------------------------------------------------------------- /controllers/cortex_ingester_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | appsv1 "k8s.io/api/apps/v1" 24 | corev1 "k8s.io/api/core/v1" 25 | apierrors "k8s.io/apimachinery/pkg/api/errors" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/apimachinery/pkg/util/intstr" 29 | "k8s.io/utils/pointer" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/client" 32 | 33 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 34 | ) 35 | 36 | // CortexIngesterReconciler reconciles a Cortex object and ensures the Cortex 37 | // Ingesters are deployed 38 | type CortexIngesterReconciler struct { 39 | client.Client 40 | Log logr.Logger 41 | Scheme *runtime.Scheme 42 | } 43 | 44 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 45 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 46 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 47 | 48 | // 49 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 50 | // 51 | 52 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 53 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 54 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 55 | 56 | // 57 | // For more details, check Reconcile and its Result here: 58 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 59 | func (r *CortexIngesterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 60 | log := r.Log.WithValues("cortex", req.NamespacedName) 61 | 62 | cortex := &cortexv1alpha1.Cortex{} 63 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 64 | if !apierrors.IsNotFound(err) { 65 | log.Error(err, "unable to fetch Cortex") 66 | } 67 | // Ignore not-found errors, since they can't be fixed by an immediate 68 | // requeue (we'll need to wait for a new notification), and we can get 69 | // them on deleted requests. 70 | return ctrl.Result{}, client.IgnoreNotFound(err) 71 | } 72 | 73 | krr := KubernetesResourceReconciler{ 74 | scheme: r.Scheme, 75 | client: r.Client, 76 | cortex: cortex, 77 | log: log, 78 | } 79 | 80 | svc := NewService(req, "ingester") 81 | err := krr.Reconcile(ctx, svc) 82 | if err != nil { 83 | return ctrl.Result{}, err 84 | } 85 | 86 | sts := NewIngesterStatefulSet(req, "ingester", cortex) 87 | cortex.Status.IngesterRef = sts.ref 88 | err = krr.Reconcile(ctx, sts) 89 | if err != nil { 90 | return ctrl.Result{}, err 91 | } 92 | 93 | return ctrl.Result{}, nil 94 | } 95 | 96 | // SetupWithManager sets up the controller with the Manager. 97 | func (r *CortexIngesterReconciler) SetupWithManager(mgr ctrl.Manager) error { 98 | return ctrl.NewControllerManagedBy(mgr). 99 | For(&cortexv1alpha1.Cortex{}). 100 | Complete(r) 101 | } 102 | 103 | func NewIngesterStatefulSet( 104 | req ctrl.Request, 105 | name string, 106 | cortex *cortexv1alpha1.Cortex, 107 | ) *KubernetesResource { 108 | sts := &appsv1.StatefulSet{ 109 | ObjectMeta: metav1.ObjectMeta{Name: "ingester", Namespace: req.Namespace}, 110 | } 111 | labels := map[string]string{ 112 | "name": "ingester", 113 | } 114 | annotations := map[string]string{ 115 | CortexConfigShasumAnnotationName: cortex.Spec.ConfigSHA(), 116 | } 117 | ref := &corev1.LocalObjectReference{Name: name} 118 | configMapName := cortex.Name + CortexConfigMapNameSuffix 119 | runtimeConfigMapName := cortex.Name + CortexRuntimeConfigMapNameSuffix 120 | 121 | return &KubernetesResource{ 122 | obj: sts, 123 | ref: ref, 124 | mutator: func() error { 125 | sts.Spec.ServiceName = "ingester" 126 | sts.Spec.Replicas = cortex.Spec.IngesterSpec.Replicas 127 | sts.Spec.PodManagementPolicy = appsv1.ParallelPodManagement 128 | sts.Spec.Selector = &metav1.LabelSelector{ 129 | MatchLabels: map[string]string{"name": "ingester"}, 130 | } 131 | sts.Spec.Template.Labels = labels 132 | sts.Spec.Template.Annotations = annotations 133 | sts.Spec.Template.Spec.Affinity = WithPodAntiAffinity("ingester") 134 | sts.Spec.Template.Spec.Containers = []corev1.Container{ 135 | { 136 | Name: "ingester", 137 | Image: cortex.Spec.Image, 138 | Args: []string{ 139 | "-target=ingester", 140 | "-ingester.chunk-encoding=3", 141 | "-config.file=/etc/cortex/config.yaml", 142 | }, 143 | ImagePullPolicy: corev1.PullIfNotPresent, 144 | Ports: []corev1.ContainerPort{ 145 | { 146 | Name: "http", 147 | ContainerPort: 80, 148 | }, 149 | { 150 | Name: "grpc", 151 | ContainerPort: 9095, 152 | }, 153 | }, 154 | ReadinessProbe: &corev1.Probe{ 155 | Handler: corev1.Handler{ 156 | HTTPGet: &corev1.HTTPGetAction{ 157 | Path: "/ready", 158 | Port: intstr.FromInt(80), 159 | Scheme: corev1.URISchemeHTTP, 160 | }, 161 | }, 162 | InitialDelaySeconds: 45, 163 | TimeoutSeconds: 1, 164 | PeriodSeconds: 10, 165 | SuccessThreshold: 1, 166 | FailureThreshold: 3, 167 | }, 168 | VolumeMounts: []corev1.VolumeMount{ 169 | { 170 | MountPath: "/etc/cortex", 171 | Name: "cortex", 172 | }, 173 | { 174 | Name: "datadir", 175 | MountPath: "/cortex", 176 | }, 177 | }, 178 | }, 179 | } 180 | sts.Spec.Template.Spec.ServiceAccountName = ServiceAccountName 181 | // https://cortexmetrics.io/docs/guides/running-cortex-on-kubernetes/#take-extra-care-with-ingesters 182 | sts.Spec.Template.Spec.TerminationGracePeriodSeconds = pointer.Int64Ptr(2400) 183 | sts.Spec.Template.Spec.Volumes = []corev1.Volume{ 184 | { 185 | Name: "cortex", 186 | VolumeSource: corev1.VolumeSource{ 187 | Projected: &corev1.ProjectedVolumeSource{ 188 | Sources: []corev1.VolumeProjection{ 189 | { 190 | ConfigMap: &corev1.ConfigMapProjection{ 191 | LocalObjectReference: corev1.LocalObjectReference{ 192 | Name: configMapName, 193 | }, 194 | }, 195 | }, 196 | { 197 | ConfigMap: &corev1.ConfigMapProjection{ 198 | LocalObjectReference: corev1.LocalObjectReference{ 199 | Name: runtimeConfigMapName, 200 | }, 201 | }, 202 | }, 203 | }, 204 | }, 205 | }, 206 | }, 207 | { 208 | Name: "datadir", 209 | VolumeSource: corev1.VolumeSource{ 210 | PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ 211 | ClaimName: "datadir", 212 | }, 213 | }, 214 | }, 215 | } 216 | sts.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ 217 | { 218 | ObjectMeta: metav1.ObjectMeta{Name: "datadir"}, 219 | Spec: corev1.PersistentVolumeClaimSpec{ 220 | // Uses the default storage class. 221 | StorageClassName: cortex.Spec.IngesterSpec.StorageClassName, 222 | AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, 223 | Resources: corev1.ResourceRequirements{ 224 | Requests: corev1.ResourceList{ 225 | "storage": *cortex.Spec.IngesterSpec.DatadirSize, 226 | }, 227 | }, 228 | }, 229 | }, 230 | } 231 | 232 | return nil 233 | }, 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /controllers/cortex_querier_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | apierrors "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ctrl "sigs.k8s.io/controller-runtime" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | 28 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 29 | ) 30 | 31 | // CortexQuerierReconciler reconciles a Cortex object and ensures the Cortex 32 | // Queriers are deployed 33 | type CortexQuerierReconciler struct { 34 | client.Client 35 | Log logr.Logger 36 | Scheme *runtime.Scheme 37 | } 38 | 39 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 40 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 41 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 42 | 43 | // 44 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 45 | // 46 | 47 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 48 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 49 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 50 | 51 | // 52 | // For more details, check Reconcile and its Result here: 53 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 54 | func (r *CortexQuerierReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 55 | log := r.Log.WithValues("cortex", req.NamespacedName) 56 | 57 | cortex := &cortexv1alpha1.Cortex{} 58 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 59 | if !apierrors.IsNotFound(err) { 60 | log.Error(err, "unable to fetch Cortex") 61 | } 62 | // Ignore not-found errors, since they can't be fixed by an immediate 63 | // requeue (we'll need to wait for a new notification), and we can get 64 | // them on deleted requests. 65 | return ctrl.Result{}, client.IgnoreNotFound(err) 66 | } 67 | 68 | if cortex.Status.IngesterRef == nil { 69 | log.Info("waiting for ingester") 70 | return ctrl.Result{}, nil 71 | } 72 | 73 | krr := KubernetesResourceReconciler{ 74 | scheme: r.Scheme, 75 | client: r.Client, 76 | cortex: cortex, 77 | log: log, 78 | } 79 | 80 | svc := NewService(req, "querier") 81 | err := krr.Reconcile(ctx, svc) 82 | if err != nil { 83 | return ctrl.Result{}, err 84 | } 85 | 86 | deploy := NewDeployment(req, "querier", cortex, cortex.Spec.QuerierSpec) 87 | cortex.Status.QuerierRef = deploy.ref 88 | err = krr.Reconcile(ctx, deploy) 89 | if err != nil { 90 | return ctrl.Result{}, err 91 | } 92 | 93 | return ctrl.Result{}, nil 94 | } 95 | 96 | // SetupWithManager sets up the controller with the Manager. 97 | func (r *CortexQuerierReconciler) SetupWithManager(mgr ctrl.Manager) error { 98 | return ctrl.NewControllerManagedBy(mgr). 99 | For(&cortexv1alpha1.Cortex{}). 100 | Complete(r) 101 | } 102 | -------------------------------------------------------------------------------- /controllers/cortex_query_frontend_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | apierrors "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ctrl "sigs.k8s.io/controller-runtime" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | 28 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 29 | ) 30 | 31 | // CortexQueryFrontendReconciler reconciles a Cortex object and ensures the Cortex 32 | // Query Frontend are deployed 33 | type CortexQueryFrontendReconciler struct { 34 | client.Client 35 | Log logr.Logger 36 | Scheme *runtime.Scheme 37 | } 38 | 39 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 40 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 41 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 42 | 43 | // 44 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 45 | // 46 | 47 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 48 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 49 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 50 | 51 | // 52 | // For more details, check Reconcile and its Result here: 53 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 54 | func (r *CortexQueryFrontendReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 55 | log := r.Log.WithValues("cortex", req.NamespacedName) 56 | 57 | cortex := &cortexv1alpha1.Cortex{} 58 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 59 | if !apierrors.IsNotFound(err) { 60 | log.Error(err, "unable to fetch Cortex") 61 | } 62 | // Ignore not-found errors, since they can't be fixed by an immediate 63 | // requeue (we'll need to wait for a new notification), and we can get 64 | // them on deleted requests. 65 | return ctrl.Result{}, client.IgnoreNotFound(err) 66 | } 67 | 68 | if cortex.Status.QuerierRef == nil { 69 | log.Info("waiting for querier") 70 | return ctrl.Result{}, nil 71 | } 72 | 73 | krr := KubernetesResourceReconciler{ 74 | scheme: r.Scheme, 75 | client: r.Client, 76 | cortex: cortex, 77 | log: log, 78 | } 79 | 80 | svc := NewService(req, "query-frontend") 81 | err := krr.Reconcile(ctx, svc) 82 | if err != nil { 83 | return ctrl.Result{}, err 84 | } 85 | 86 | deploy := NewDeployment(req, "query-frontend", cortex, cortex.Spec.QueryFrontendSpec) 87 | cortex.Status.QueryFrontendRef = deploy.ref 88 | err = krr.Reconcile(ctx, deploy) 89 | if err != nil { 90 | return ctrl.Result{}, err 91 | } 92 | 93 | return ctrl.Result{}, nil 94 | } 95 | 96 | // SetupWithManager sets up the controller with the Manager. 97 | func (r *CortexQueryFrontendReconciler) SetupWithManager(mgr ctrl.Manager) error { 98 | return ctrl.NewControllerManagedBy(mgr). 99 | For(&cortexv1alpha1.Cortex{}). 100 | Complete(r) 101 | } 102 | -------------------------------------------------------------------------------- /controllers/cortex_ruler_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/go-logr/logr" 24 | appsv1 "k8s.io/api/apps/v1" 25 | corev1 "k8s.io/api/core/v1" 26 | apierrors "k8s.io/apimachinery/pkg/api/errors" 27 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 | "k8s.io/apimachinery/pkg/runtime" 29 | "k8s.io/apimachinery/pkg/util/intstr" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/client" 32 | 33 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 34 | ) 35 | 36 | const RulerName = "ruler" 37 | 38 | // CortexRulerReconciler reconciles a Cortex object and ensures the Cortex 39 | // Ruler is deployed 40 | type CortexRulerReconciler struct { 41 | client.Client 42 | Log logr.Logger 43 | Scheme *runtime.Scheme 44 | } 45 | 46 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 47 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 48 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 49 | 50 | // 51 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 52 | // 53 | 54 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 55 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 56 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 57 | 58 | // 59 | // For more details, check Reconcile and its Result here: 60 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 61 | func (r *CortexRulerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 62 | log := r.Log.WithValues("cortex", req.NamespacedName) 63 | 64 | cortex := &cortexv1alpha1.Cortex{} 65 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 66 | if !apierrors.IsNotFound(err) { 67 | log.Error(err, "unable to fetch Cortex") 68 | } 69 | // Ignore not-found errors, since they can't be fixed by an immediate 70 | // requeue (we'll need to wait for a new notification), and we can get 71 | // them on deleted requests. 72 | return ctrl.Result{}, client.IgnoreNotFound(err) 73 | } 74 | 75 | if !cortex.Status.MemcachedRef.IsSet() { 76 | log.Info("waiting for memcached") 77 | return ctrl.Result{Requeue: true}, nil 78 | } 79 | 80 | krr := KubernetesResourceReconciler{ 81 | scheme: r.Scheme, 82 | client: r.Client, 83 | cortex: cortex, 84 | log: log, 85 | } 86 | 87 | svc := NewRulerService(req) 88 | err := krr.Reconcile(ctx, svc) 89 | if err != nil { 90 | return ctrl.Result{}, err 91 | } 92 | 93 | deploy := NewRulerDeployment(req, cortex, cortex.Spec.RulerSpec) 94 | cortex.Status.RulerRef = deploy.ref 95 | err = krr.Reconcile(ctx, deploy) 96 | if err != nil { 97 | return ctrl.Result{}, err 98 | } 99 | 100 | return ctrl.Result{}, nil 101 | } 102 | 103 | // SetupWithManager sets up the controller with the Manager. 104 | func (r *CortexRulerReconciler) SetupWithManager(mgr ctrl.Manager) error { 105 | return ctrl.NewControllerManagedBy(mgr). 106 | For(&cortexv1alpha1.Cortex{}). 107 | Complete(r) 108 | } 109 | 110 | func NewRulerService(req ctrl.Request) *KubernetesResource { 111 | svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: RulerName, Namespace: req.Namespace}} 112 | ref := &corev1.LocalObjectReference{Name: RulerName} 113 | return &KubernetesResource{ 114 | obj: svc, 115 | ref: ref, 116 | mutator: func() error { 117 | svc.Labels = map[string]string{ 118 | "name": RulerName, 119 | "job": fmt.Sprintf("%s.%s", req.Namespace, RulerName), 120 | } 121 | svc.Spec.Ports = make([]corev1.ServicePort, 0) 122 | svc.Spec.Ports = []corev1.ServicePort{ 123 | { 124 | Name: "http", 125 | Port: 80, 126 | TargetPort: intstr.FromInt(80), 127 | }, 128 | } 129 | svc.Spec.Selector = map[string]string{"name": RulerName} 130 | 131 | return nil 132 | }, 133 | } 134 | } 135 | 136 | func NewRulerDeployment( 137 | req ctrl.Request, 138 | cortex *cortexv1alpha1.Cortex, 139 | spec *cortexv1alpha1.DeploymentSpec, 140 | ) *KubernetesResource { 141 | deploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: RulerName, Namespace: req.Namespace}} 142 | labels := map[string]string{ 143 | "name": RulerName, 144 | } 145 | annotations := map[string]string{ 146 | CortexConfigShasumAnnotationName: cortex.Spec.ConfigSHA(), 147 | } 148 | ref := &corev1.LocalObjectReference{Name: RulerName} 149 | configMapName := cortex.Name + CortexConfigMapNameSuffix 150 | runtimeConfigMapName := cortex.Name + CortexRuntimeConfigMapNameSuffix 151 | 152 | return &KubernetesResource{ 153 | obj: deploy, 154 | ref: ref, 155 | mutator: func() error { 156 | deploy.Spec.Replicas = spec.Replicas 157 | deploy.Spec.Selector = &metav1.LabelSelector{ 158 | MatchLabels: labels, 159 | } 160 | deploy.Spec.Template.Labels = labels 161 | deploy.Spec.Template.Annotations = annotations 162 | deploy.Spec.Template.Spec.Affinity = WithPodAntiAffinity(RulerName) 163 | deploy.Spec.Template.Spec.Containers = []corev1.Container{ 164 | { 165 | Name: RulerName, 166 | Image: cortex.Spec.Image, 167 | ImagePullPolicy: corev1.PullIfNotPresent, 168 | Args: []string{ 169 | "-target=" + RulerName, 170 | "-config.file=/etc/cortex/config.yaml", 171 | }, 172 | Ports: []corev1.ContainerPort{ 173 | { 174 | Name: "http", 175 | ContainerPort: 80, 176 | }, 177 | }, 178 | VolumeMounts: []corev1.VolumeMount{ 179 | { 180 | MountPath: "/etc/cortex", 181 | Name: "cortex", 182 | }, 183 | }, 184 | }, 185 | } 186 | deploy.Spec.Template.Spec.ServiceAccountName = ServiceAccountName 187 | deploy.Spec.Template.Spec.Volumes = []corev1.Volume{ 188 | { 189 | Name: "cortex", 190 | VolumeSource: corev1.VolumeSource{ 191 | Projected: &corev1.ProjectedVolumeSource{ 192 | Sources: []corev1.VolumeProjection{ 193 | { 194 | ConfigMap: &corev1.ConfigMapProjection{ 195 | LocalObjectReference: corev1.LocalObjectReference{ 196 | Name: configMapName, 197 | }, 198 | }, 199 | }, 200 | { 201 | ConfigMap: &corev1.ConfigMapProjection{ 202 | LocalObjectReference: corev1.LocalObjectReference{ 203 | Name: runtimeConfigMapName, 204 | }, 205 | }, 206 | }, 207 | }, 208 | }, 209 | }, 210 | }, 211 | } 212 | 213 | return nil 214 | }, 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /controllers/cortex_runtime_config_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | corev1 "k8s.io/api/core/v1" 24 | apierrors "k8s.io/apimachinery/pkg/api/errors" 25 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 26 | "k8s.io/apimachinery/pkg/runtime" 27 | ctrl "sigs.k8s.io/controller-runtime" 28 | "sigs.k8s.io/controller-runtime/pkg/client" 29 | 30 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 31 | ) 32 | 33 | // CortexRuntimeConfigReconciler reconciles a Cortex object and ensures the 34 | // Cortex runtime config map is updated accordingly. 35 | type CortexRuntimeConfigReconciler struct { 36 | client.Client 37 | Log logr.Logger 38 | Scheme *runtime.Scheme 39 | } 40 | 41 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 42 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 43 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 44 | 45 | // 46 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 47 | // 48 | 49 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 50 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 51 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 52 | 53 | // 54 | // For more details, check Reconcile and its Result here: 55 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 56 | func (r *CortexRuntimeConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 57 | log := r.Log.WithValues("cortex", req.NamespacedName) 58 | 59 | cortex := &cortexv1alpha1.Cortex{} 60 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 61 | if !apierrors.IsNotFound(err) { 62 | log.Error(err, "unable to fetch Cortex") 63 | } 64 | // Ignore not-found errors, since they can't be fixed by an immediate 65 | // requeue (we'll need to wait for a new notification), and we can get 66 | // them on deleted requests. 67 | return ctrl.Result{}, client.IgnoreNotFound(err) 68 | } 69 | 70 | krr := KubernetesResourceReconciler{ 71 | scheme: r.Scheme, 72 | client: r.Client, 73 | cortex: cortex, 74 | log: log, 75 | } 76 | 77 | cm, err := NewCortexRuntimeConfigMap(req, cortex) 78 | if err != nil { 79 | log.Error(err, "failed to generate cortex runtime config map, will not retry") 80 | return ctrl.Result{Requeue: false}, err 81 | } 82 | 83 | err = krr.Reconcile(ctx, cm) 84 | if err != nil { 85 | return ctrl.Result{}, err 86 | } 87 | 88 | return ctrl.Result{}, nil 89 | } 90 | 91 | // SetupWithManager sets up the controller with the Manager. 92 | func (r *CortexRuntimeConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { 93 | return ctrl.NewControllerManagedBy(mgr). 94 | For(&cortexv1alpha1.Cortex{}). 95 | Complete(r) 96 | } 97 | 98 | func NewCortexRuntimeConfigMap( 99 | req ctrl.Request, 100 | cortex *cortexv1alpha1.Cortex, 101 | ) (*KubernetesResource, error) { 102 | name := cortex.Name + CortexRuntimeConfigMapNameSuffix 103 | configMap := &corev1.ConfigMap{ 104 | ObjectMeta: metav1.ObjectMeta{ 105 | Name: name, 106 | Namespace: req.Namespace, 107 | }, 108 | } 109 | 110 | y, err := cortex.RuntimeConfigAsYAML() 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | data := map[string]string{ 116 | "runtime-config.yaml": string(y), 117 | } 118 | 119 | return &KubernetesResource{ 120 | obj: configMap, 121 | mutator: func() error { 122 | configMap.Data = data 123 | return nil 124 | }, 125 | }, nil 126 | } 127 | -------------------------------------------------------------------------------- /controllers/cortex_store_gateway_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | apierrors "k8s.io/apimachinery/pkg/api/errors" 24 | "k8s.io/apimachinery/pkg/runtime" 25 | ctrl "sigs.k8s.io/controller-runtime" 26 | "sigs.k8s.io/controller-runtime/pkg/client" 27 | 28 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 29 | ) 30 | 31 | // CortexStoreGatewayReconciler reconciles a Cortex object and ensures the Cortex 32 | // Store Gateway is deployed 33 | type CortexStoreGatewayReconciler struct { 34 | client.Client 35 | Log logr.Logger 36 | Scheme *runtime.Scheme 37 | } 38 | 39 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 40 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 41 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 42 | 43 | // 44 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 45 | // 46 | 47 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 48 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 49 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 50 | 51 | // 52 | // For more details, check Reconcile and its Result here: 53 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 54 | func (r *CortexStoreGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 55 | log := r.Log.WithValues("cortex", req.NamespacedName) 56 | 57 | cortex := &cortexv1alpha1.Cortex{} 58 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 59 | if !apierrors.IsNotFound(err) { 60 | log.Error(err, "unable to fetch Cortex") 61 | } 62 | // Ignore not-found errors, since they can't be fixed by an immediate 63 | // requeue (we'll need to wait for a new notification), and we can get 64 | // them on deleted requests. 65 | return ctrl.Result{}, client.IgnoreNotFound(err) 66 | } 67 | 68 | krr := KubernetesResourceReconciler{ 69 | scheme: r.Scheme, 70 | client: r.Client, 71 | cortex: cortex, 72 | log: log, 73 | } 74 | 75 | svc := NewService(req, "store-gateway") 76 | err := krr.Reconcile(ctx, svc) 77 | if err != nil { 78 | return ctrl.Result{}, err 79 | } 80 | 81 | sts := NewStatefulSet(req, "store-gateway", cortex, cortex.Spec.StoreGatewaySpec) 82 | cortex.Status.StoreGatewayRef = sts.ref 83 | err = krr.Reconcile(ctx, sts) 84 | if err != nil { 85 | return ctrl.Result{}, err 86 | } 87 | 88 | return ctrl.Result{}, nil 89 | } 90 | 91 | // SetupWithManager sets up the controller with the Manager. 92 | func (r *CortexStoreGatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { 93 | return ctrl.NewControllerManagedBy(mgr). 94 | For(&cortexv1alpha1.Cortex{}). 95 | Complete(r) 96 | } 97 | -------------------------------------------------------------------------------- /controllers/helpers.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | corev1 "k8s.io/api/core/v1" 21 | v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | ) 23 | 24 | func WithPodAntiAffinity(name string) *corev1.Affinity { 25 | return &corev1.Affinity{ 26 | PodAntiAffinity: &corev1.PodAntiAffinity{ 27 | PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ 28 | { 29 | PodAffinityTerm: corev1.PodAffinityTerm{ 30 | LabelSelector: &v1.LabelSelector{ 31 | MatchLabels: map[string]string{"name": name}, 32 | }, 33 | TopologyKey: "kubernetes.io/hostname", 34 | }, 35 | // 36 | // https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/ 37 | // 38 | // "The weight field in preferredDuringSchedulingIgnoredDuringExecution 39 | // is in the range 1-100. For each node that meets all of the scheduling 40 | // requirements (resource request, RequiredDuringScheduling affinity 41 | // expressions, etc.), the scheduler will compute a sum by iterating 42 | // through the elements of this field and adding "weight" to the sum if 43 | // the node matches the corresponding MatchExpressions. This score is 44 | // then combined with the scores of other priority functions for the 45 | // node. The node(s) with the highest total score are the most 46 | // preferred." 47 | // 48 | // Give the maximum weight to this rule to prevent the pod from being 49 | // scheduled to a node already running a pod for the given 50 | // deployment. 51 | Weight: 100, 52 | }, 53 | }, 54 | }, 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /controllers/kubernetes_resource.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | "github.com/go-logr/logr" 24 | appsv1 "k8s.io/api/apps/v1" 25 | corev1 "k8s.io/api/core/v1" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/apimachinery/pkg/util/intstr" 29 | ctrl "sigs.k8s.io/controller-runtime" 30 | "sigs.k8s.io/controller-runtime/pkg/client" 31 | "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" 32 | 33 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 34 | ) 35 | 36 | type KubernetesResource struct { 37 | obj client.Object 38 | ref *corev1.LocalObjectReference 39 | mutator controllerutil.MutateFn 40 | } 41 | 42 | type KubernetesResourceReconciler struct { 43 | log logr.Logger 44 | client client.Client 45 | cortex *cortexv1alpha1.Cortex 46 | scheme *runtime.Scheme 47 | } 48 | 49 | func (krr *KubernetesResourceReconciler) Reconcile( 50 | ctx context.Context, 51 | r *KubernetesResource, 52 | ) error { 53 | // Set up garbage collection. The object (resource.obj) will be 54 | // automatically deleted when he owner (cortex) is deleted. 55 | err := controllerutil.SetOwnerReference(krr.cortex, r.obj, krr.scheme) 56 | if err != nil { 57 | krr.log.Error( 58 | err, 59 | "failed to set owner reference on resource", 60 | "kind", r.obj.GetObjectKind().GroupVersionKind().Kind, 61 | "name", r.obj.GetName(), 62 | ) 63 | return err 64 | } 65 | 66 | op, err := controllerutil.CreateOrUpdate(ctx, krr.client, r.obj, r.mutator) 67 | if err != nil { 68 | krr.log.Error( 69 | err, 70 | "failed to reconcile resource", 71 | "kind", r.obj.GetObjectKind().GroupVersionKind().Kind, 72 | "name", r.obj.GetName(), 73 | ) 74 | return err 75 | } 76 | 77 | err = krr.client.Status().Update(ctx, krr.cortex) 78 | if err != nil { 79 | krr.log.Error( 80 | err, 81 | "failed to reconcile resource status", 82 | "kind", r.obj.GetObjectKind().GroupVersionKind().Kind, 83 | "name", r.obj.GetName(), 84 | ) 85 | return err 86 | } 87 | 88 | krr.log.Info( 89 | "Reconcile successful", 90 | "operation", op, 91 | "kind", r.obj.GetObjectKind().GroupVersionKind().Kind, 92 | "name", r.obj.GetName(), 93 | ) 94 | 95 | return nil 96 | } 97 | 98 | func NewService(req ctrl.Request, name string) *KubernetesResource { 99 | svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: req.Namespace}} 100 | ref := &corev1.LocalObjectReference{Name: name} 101 | return &KubernetesResource{ 102 | obj: svc, 103 | ref: ref, 104 | mutator: func() error { 105 | svc.Labels = map[string]string{ 106 | "name": name, 107 | "job": fmt.Sprintf("%s.%s", req.Namespace, name), 108 | } 109 | svc.Spec.Ports = make([]corev1.ServicePort, 0) 110 | svc.Spec.Ports = []corev1.ServicePort{ 111 | { 112 | Name: "http", 113 | Port: 80, 114 | TargetPort: intstr.FromInt(80), 115 | }, 116 | { 117 | Name: "grpc", 118 | Port: 9095, 119 | TargetPort: intstr.FromInt(9095), 120 | }, 121 | } 122 | svc.Spec.Selector = map[string]string{"name": name} 123 | 124 | return nil 125 | }, 126 | } 127 | } 128 | 129 | func NewDeployment( 130 | req ctrl.Request, 131 | name string, 132 | cortex *cortexv1alpha1.Cortex, 133 | spec *cortexv1alpha1.DeploymentSpec, 134 | ) *KubernetesResource { 135 | deploy := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: req.Namespace}} 136 | labels := map[string]string{ 137 | "name": name, 138 | "memberlist": "gossip-ring", 139 | } 140 | annotations := map[string]string{ 141 | CortexConfigShasumAnnotationName: cortex.Spec.ConfigSHA(), 142 | } 143 | ref := &corev1.LocalObjectReference{Name: name} 144 | configMapName := cortex.Name + CortexConfigMapNameSuffix 145 | runtimeConfigMapName := cortex.Name + CortexRuntimeConfigMapNameSuffix 146 | 147 | return &KubernetesResource{ 148 | obj: deploy, 149 | ref: ref, 150 | mutator: func() error { 151 | deploy.Spec.Replicas = spec.Replicas 152 | deploy.Spec.Selector = &metav1.LabelSelector{ 153 | MatchLabels: labels, 154 | } 155 | deploy.Spec.Template.Labels = labels 156 | deploy.Spec.Template.Annotations = annotations 157 | deploy.Spec.Template.Spec.Affinity = WithPodAntiAffinity(name) 158 | deploy.Spec.Template.Spec.Containers = []corev1.Container{ 159 | { 160 | Name: name, 161 | Image: cortex.Spec.Image, 162 | ImagePullPolicy: corev1.PullIfNotPresent, 163 | Args: []string{ 164 | "-target=" + name, 165 | "-config.file=/etc/cortex/config.yaml", 166 | }, 167 | Ports: []corev1.ContainerPort{ 168 | { 169 | Name: "http", 170 | ContainerPort: 80, 171 | }, 172 | { 173 | Name: "grpc", 174 | ContainerPort: 9095, 175 | }, 176 | }, 177 | VolumeMounts: []corev1.VolumeMount{ 178 | { 179 | MountPath: "/etc/cortex", 180 | Name: "cortex", 181 | }, 182 | }, 183 | }, 184 | } 185 | deploy.Spec.Template.Spec.ServiceAccountName = ServiceAccountName 186 | deploy.Spec.Template.Spec.Volumes = []corev1.Volume{ 187 | { 188 | Name: "cortex", 189 | VolumeSource: corev1.VolumeSource{ 190 | Projected: &corev1.ProjectedVolumeSource{ 191 | Sources: []corev1.VolumeProjection{ 192 | { 193 | ConfigMap: &corev1.ConfigMapProjection{ 194 | LocalObjectReference: corev1.LocalObjectReference{ 195 | Name: configMapName, 196 | }, 197 | }, 198 | }, 199 | { 200 | ConfigMap: &corev1.ConfigMapProjection{ 201 | LocalObjectReference: corev1.LocalObjectReference{ 202 | Name: runtimeConfigMapName, 203 | }, 204 | }, 205 | }, 206 | }, 207 | }, 208 | }, 209 | }, 210 | } 211 | 212 | return nil 213 | }, 214 | } 215 | } 216 | 217 | func NewStatefulSet( 218 | req ctrl.Request, 219 | name string, 220 | cortex *cortexv1alpha1.Cortex, 221 | spec *cortexv1alpha1.StatefulSetSpec, 222 | ) *KubernetesResource { 223 | sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: req.Namespace}} 224 | labels := map[string]string{ 225 | "name": name, 226 | } 227 | annotations := map[string]string{ 228 | CortexConfigShasumAnnotationName: cortex.Spec.ConfigSHA(), 229 | } 230 | ref := &corev1.LocalObjectReference{Name: name} 231 | configMapName := cortex.Name + CortexConfigMapNameSuffix 232 | runtimeConfigMapName := cortex.Name + CortexRuntimeConfigMapNameSuffix 233 | 234 | return &KubernetesResource{ 235 | obj: sts, 236 | ref: ref, 237 | mutator: func() error { 238 | sts.Spec.ServiceName = name 239 | sts.Spec.Replicas = spec.Replicas 240 | sts.Spec.PodManagementPolicy = appsv1.OrderedReadyPodManagement 241 | sts.Spec.Selector = &metav1.LabelSelector{ 242 | MatchLabels: map[string]string{"name": name}, 243 | } 244 | sts.Spec.Template.Labels = labels 245 | sts.Spec.Template.Annotations = annotations 246 | sts.Spec.Template.Spec.Affinity = WithPodAntiAffinity(name) 247 | sts.Spec.Template.Spec.Containers = []corev1.Container{ 248 | { 249 | Name: name, 250 | Image: cortex.Spec.Image, 251 | Args: []string{ 252 | "-target=" + name, 253 | "-config.file=/etc/cortex/config.yaml", 254 | }, 255 | ImagePullPolicy: corev1.PullIfNotPresent, 256 | Ports: []corev1.ContainerPort{ 257 | { 258 | Name: "http", 259 | ContainerPort: 80, 260 | }, 261 | { 262 | Name: "grpc", 263 | ContainerPort: 9095, 264 | }, 265 | }, 266 | VolumeMounts: []corev1.VolumeMount{ 267 | { 268 | MountPath: "/etc/cortex", 269 | Name: "cortex", 270 | }, 271 | { 272 | Name: "datadir", 273 | MountPath: "/cortex", 274 | }, 275 | }, 276 | }, 277 | } 278 | sts.Spec.Template.Spec.ServiceAccountName = ServiceAccountName 279 | sts.Spec.Template.Spec.Volumes = []corev1.Volume{ 280 | { 281 | Name: "cortex", 282 | VolumeSource: corev1.VolumeSource{ 283 | Projected: &corev1.ProjectedVolumeSource{ 284 | Sources: []corev1.VolumeProjection{ 285 | { 286 | ConfigMap: &corev1.ConfigMapProjection{ 287 | LocalObjectReference: corev1.LocalObjectReference{ 288 | Name: configMapName, 289 | }, 290 | }, 291 | }, 292 | { 293 | ConfigMap: &corev1.ConfigMapProjection{ 294 | LocalObjectReference: corev1.LocalObjectReference{ 295 | Name: runtimeConfigMapName, 296 | }, 297 | }, 298 | }, 299 | }, 300 | }, 301 | }, 302 | }, 303 | { 304 | Name: "datadir", 305 | VolumeSource: corev1.VolumeSource{ 306 | PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ 307 | ClaimName: "datadir", 308 | }, 309 | }, 310 | }, 311 | } 312 | sts.Spec.VolumeClaimTemplates = []corev1.PersistentVolumeClaim{ 313 | { 314 | ObjectMeta: metav1.ObjectMeta{Name: "datadir"}, 315 | Spec: corev1.PersistentVolumeClaimSpec{ 316 | StorageClassName: spec.StorageClassName, 317 | AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, 318 | Resources: corev1.ResourceRequirements{ 319 | Requests: corev1.ResourceList{ 320 | "storage": *spec.DatadirSize, 321 | }, 322 | }, 323 | }, 324 | }, 325 | } 326 | 327 | return nil 328 | }, 329 | } 330 | } 331 | -------------------------------------------------------------------------------- /controllers/memcached_controller.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "context" 21 | 22 | "github.com/go-logr/logr" 23 | appsv1 "k8s.io/api/apps/v1" 24 | corev1 "k8s.io/api/core/v1" 25 | apierrors "k8s.io/apimachinery/pkg/api/errors" 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/apimachinery/pkg/util/intstr" 29 | ctrl "sigs.k8s.io/controller-runtime" 30 | "sigs.k8s.io/controller-runtime/pkg/client" 31 | 32 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 33 | ) 34 | 35 | // MemacachedReconciler reconciles a Cortex object and ensures all Memcached 36 | // resources are deployed 37 | type MemcachedReconciler struct { 38 | client.Client 39 | Log logr.Logger 40 | Scheme *runtime.Scheme 41 | } 42 | 43 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices,verbs=get;list;watch;create;update;patch;delete 44 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/status,verbs=get;update;patch 45 | //+kubebuilder:rbac:groups=cortex.opstrace.io,resources=cortices/finalizers,verbs=update 46 | 47 | // 48 | // Setup RBAC to create and manage Kubernetes resources required to deploy Cortex. 49 | // 50 | 51 | //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 52 | //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 53 | //+kubebuilder:rbac:groups="apps",resources=statefulsets,verbs=get;list;watch;create;update;patch;delete;scope=Cluster 54 | 55 | // 56 | // For more details, check Reconcile and its Result here: 57 | // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile 58 | func (r *MemcachedReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { 59 | log := r.Log.WithValues("cortex", req.NamespacedName) 60 | 61 | cortex := &cortexv1alpha1.Cortex{} 62 | if err := r.Get(ctx, req.NamespacedName, cortex); err != nil { 63 | if !apierrors.IsNotFound(err) { 64 | log.Error(err, "unable to fetch Cortex") 65 | } 66 | // Ignore not-found errors, since they can't be fixed by an immediate 67 | // requeue (we'll need to wait for a new notification), and we can get 68 | // them on deleted requests. 69 | return ctrl.Result{}, client.IgnoreNotFound(err) 70 | } 71 | 72 | if cortex.Status.MemcachedRef == nil { 73 | cortex.Status.MemcachedRef = &cortexv1alpha1.MemcachedReference{} 74 | } 75 | 76 | krr := KubernetesResourceReconciler{ 77 | scheme: r.Scheme, 78 | client: r.Client, 79 | cortex: cortex, 80 | log: log, 81 | } 82 | 83 | svc := NewMemcachedService(req, "memcached-chunks") 84 | err := krr.Reconcile(ctx, svc) 85 | if err != nil { 86 | return ctrl.Result{}, err 87 | } 88 | 89 | sts := NewMemcachedStatefulSet( 90 | req, 91 | "memcached-chunks", 92 | cortex.Spec.Memcached.Image, 93 | cortex.Spec.Memcached.ChunksCacheSpec, 94 | ) 95 | cortex.Status.MemcachedRef.ChunksCacheRef = sts.ref 96 | err = krr.Reconcile(ctx, sts) 97 | if err != nil { 98 | return ctrl.Result{}, err 99 | } 100 | 101 | svc = NewMemcachedService(req, "memcached-index-queries") 102 | err = krr.Reconcile(ctx, svc) 103 | if err != nil { 104 | return ctrl.Result{}, err 105 | } 106 | 107 | sts = NewMemcachedStatefulSet( 108 | req, 109 | "memcached-index-queries", 110 | cortex.Spec.Memcached.Image, 111 | cortex.Spec.Memcached.IndexQueriesCacheSpec, 112 | ) 113 | cortex.Status.MemcachedRef.IndexQueriesCacheRef = sts.ref 114 | err = krr.Reconcile(ctx, sts) 115 | if err != nil { 116 | return ctrl.Result{}, err 117 | } 118 | 119 | svc = NewMemcachedService(req, "memcached-index-writes") 120 | err = krr.Reconcile(ctx, svc) 121 | if err != nil { 122 | return ctrl.Result{}, err 123 | } 124 | 125 | sts = NewMemcachedStatefulSet( 126 | req, 127 | "memcached-index-writes", 128 | cortex.Spec.Memcached.Image, 129 | cortex.Spec.Memcached.IndexWritesCacheSpec, 130 | ) 131 | cortex.Status.MemcachedRef.IndexWritesCacheRef = sts.ref 132 | err = krr.Reconcile(ctx, sts) 133 | if err != nil { 134 | return ctrl.Result{}, err 135 | } 136 | 137 | svc = NewMemcachedService(req, "memcached-results") 138 | err = krr.Reconcile(ctx, svc) 139 | if err != nil { 140 | return ctrl.Result{}, err 141 | } 142 | 143 | sts = NewMemcachedStatefulSet( 144 | req, 145 | "memcached-results", 146 | cortex.Spec.Memcached.Image, 147 | cortex.Spec.Memcached.ResultsCacheSpec, 148 | ) 149 | cortex.Status.MemcachedRef.ResultsCacheRef = sts.ref 150 | err = krr.Reconcile(ctx, sts) 151 | if err != nil { 152 | return ctrl.Result{}, err 153 | } 154 | 155 | svc = NewMemcachedService(req, "memcached-metadata") 156 | err = krr.Reconcile(ctx, svc) 157 | if err != nil { 158 | return ctrl.Result{}, err 159 | } 160 | 161 | sts = NewMemcachedStatefulSet( 162 | req, 163 | "memcached-metadata", 164 | cortex.Spec.Memcached.Image, 165 | cortex.Spec.Memcached.MetadataCacheSpec, 166 | ) 167 | cortex.Status.MemcachedRef.MetadataCacheRef = sts.ref 168 | err = krr.Reconcile(ctx, sts) 169 | if err != nil { 170 | return ctrl.Result{}, err 171 | } 172 | 173 | return ctrl.Result{}, nil 174 | } 175 | 176 | // SetupWithManager sets up the controller with the Manager. 177 | func (r *MemcachedReconciler) SetupWithManager(mgr ctrl.Manager) error { 178 | return ctrl.NewControllerManagedBy(mgr). 179 | For(&cortexv1alpha1.Cortex{}). 180 | Complete(r) 181 | } 182 | 183 | func NewMemcachedService(req ctrl.Request, name string) *KubernetesResource { 184 | svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: req.Namespace}} 185 | return &KubernetesResource{ 186 | obj: svc, 187 | mutator: func() error { 188 | svc.Labels = map[string]string{ 189 | "name": name, 190 | } 191 | svc.Spec.Ports = make([]corev1.ServicePort, 0) 192 | svc.Spec.ClusterIP = "None" 193 | svc.Spec.Ports = []corev1.ServicePort{ 194 | { 195 | Name: "memcached-client", 196 | Port: 11211, 197 | TargetPort: intstr.FromInt(11211), 198 | }, 199 | } 200 | svc.Spec.Selector = map[string]string{"name": name} 201 | 202 | return nil 203 | }, 204 | } 205 | } 206 | 207 | func NewMemcachedStatefulSet( 208 | req ctrl.Request, 209 | name string, 210 | image string, 211 | spec *cortexv1alpha1.MemcachedStatefulSetSpec, 212 | ) *KubernetesResource { 213 | statefulSet := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: req.Namespace}} 214 | ref := &corev1.LocalObjectReference{Name: name} 215 | args := append([]string{"-v"}, spec.AsArgs()...) 216 | 217 | return &KubernetesResource{ 218 | obj: statefulSet, 219 | ref: ref, 220 | mutator: func() error { 221 | statefulSet.Spec.ServiceName = name 222 | statefulSet.Spec.Replicas = spec.Replicas 223 | statefulSet.Spec.PodManagementPolicy = appsv1.ParallelPodManagement 224 | statefulSet.Spec.Selector = &metav1.LabelSelector{ 225 | MatchLabels: map[string]string{"name": name}, 226 | } 227 | statefulSet.Spec.Template.ObjectMeta.Labels = map[string]string{ 228 | "name": name, 229 | } 230 | statefulSet.Spec.Template.Spec.Affinity = WithPodAntiAffinity(name) 231 | statefulSet.Spec.Template.Spec.Containers = []corev1.Container{ 232 | { 233 | Name: "memcached", 234 | Image: image, 235 | Args: args, 236 | ImagePullPolicy: corev1.PullIfNotPresent, 237 | Ports: []corev1.ContainerPort{ 238 | { 239 | Name: "client", 240 | ContainerPort: 11211, 241 | }, 242 | }, 243 | }, 244 | } 245 | statefulSet.Spec.UpdateStrategy = appsv1.StatefulSetUpdateStrategy{ 246 | Type: appsv1.RollingUpdateStatefulSetStrategyType, 247 | } 248 | 249 | return nil 250 | }, 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /controllers/suite_test.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package controllers 18 | 19 | import ( 20 | "path/filepath" 21 | "testing" 22 | 23 | . "github.com/onsi/ginkgo" 24 | . "github.com/onsi/gomega" 25 | appsv1 "k8s.io/api/apps/v1" 26 | corev1 "k8s.io/api/core/v1" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/client-go/rest" 29 | ctrl "sigs.k8s.io/controller-runtime" 30 | "sigs.k8s.io/controller-runtime/pkg/client" 31 | "sigs.k8s.io/controller-runtime/pkg/envtest" 32 | "sigs.k8s.io/controller-runtime/pkg/envtest/printer" 33 | logf "sigs.k8s.io/controller-runtime/pkg/log" 34 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 35 | 36 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 37 | //+kubebuilder:scaffold:imports 38 | ) 39 | 40 | // These tests use Ginkgo (BDD-style Go testing framework). Refer to 41 | // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. 42 | 43 | var ( 44 | cfg *rest.Config 45 | k8sClient client.Client 46 | testEnv *envtest.Environment 47 | k8sClientScheme = runtime.NewScheme() 48 | ) 49 | 50 | func TestAPIs(t *testing.T) { 51 | RegisterFailHandler(Fail) 52 | 53 | RunSpecsWithDefaultAndCustomReporters(t, 54 | "Controller Suite", 55 | []Reporter{printer.NewlineReporter{}}) 56 | } 57 | 58 | var _ = BeforeSuite(func() { 59 | logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) 60 | 61 | By("bootstrapping test environment") 62 | testEnv = &envtest.Environment{ 63 | CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, 64 | ErrorIfCRDPathMissing: true, 65 | } 66 | 67 | cfg, err := testEnv.Start() 68 | Expect(err).NotTo(HaveOccurred()) 69 | Expect(cfg).NotTo(BeNil()) 70 | 71 | err = corev1.AddToScheme(k8sClientScheme) 72 | Expect(err).NotTo(HaveOccurred()) 73 | err = appsv1.AddToScheme(k8sClientScheme) 74 | Expect(err).NotTo(HaveOccurred()) 75 | err = cortexv1alpha1.AddToScheme(k8sClientScheme) 76 | Expect(err).NotTo(HaveOccurred()) 77 | 78 | //+kubebuilder:scaffold:scheme 79 | 80 | k8sClient, err = client.New(cfg, client.Options{Scheme: k8sClientScheme}) 81 | Expect(err).NotTo(HaveOccurred()) 82 | Expect(k8sClient).NotTo(BeNil()) 83 | 84 | k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ 85 | Scheme: k8sClientScheme, 86 | }) 87 | Expect(err).ToNot(HaveOccurred()) 88 | 89 | err = (&CortexReconciler{ 90 | Client: k8sManager.GetClient(), 91 | Scheme: k8sManager.GetScheme(), 92 | Log: ctrl.Log.WithName("controllers").WithName("Cortex"), 93 | }).SetupWithManager(k8sManager) 94 | Expect(err).ToNot(HaveOccurred()) 95 | 96 | go func() { 97 | err = k8sManager.Start(ctrl.SetupSignalHandler()) 98 | Expect(err).ToNot(HaveOccurred()) 99 | }() 100 | 101 | }, 60) 102 | 103 | var _ = AfterSuite(func() { 104 | By("tearing down the test environment") 105 | err := testEnv.Stop() 106 | Expect(err).NotTo(HaveOccurred()) 107 | }) 108 | -------------------------------------------------------------------------------- /docs/guides/terraform/README.md: -------------------------------------------------------------------------------- 1 | # How to create an EKS cluster using Terraform and deploy cortex-operator 2 | 3 | This guide shows how you can create an EKS cluster and the required S3 bucket with Terraform to deploy the cortex-operator. 4 | 5 | 6 | ## Requirements 7 | 8 | - AWS account and credentials set up for your environment 9 | - [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started) 10 | 11 | ## Acknowledgement 12 | 13 | The Terraform files are based on the [Terraform EKS provider examples](https://github.com/hashicorp/terraform-provider-kubernetes/tree/main/_examples/eks). 14 | 15 | ## Preamble 16 | 17 | The base directory to create the infrastructure must be `examples/terraform` in the repo root directory. 18 | 19 | ``` 20 | git clone https://github.com/opstrace/cortex-operator cortex-operator 21 | cd cortex-operator/examples/terraform 22 | ``` 23 | 24 | Proceed to initialize terraform. 25 | 26 | ``` 27 | terraform init 28 | ``` 29 | 30 | ## Configure 31 | 32 | Edit `example.tfvars` with your favorite editor and choose the cluster name prefix, the AWS region, and Kubernetes cluster version. 33 | 34 | The current defaults are: 35 | 36 | ``` 37 | cluster_name_prefix = "cortex-operator-example" 38 | aws_region = "us-west-2" 39 | kubernetes_version = "1.18" 40 | ``` 41 | 42 | ## Create Infrastructure 43 | 44 | To create all the required infrastructure to run the cortex-operator: 45 | 46 | ``` 47 | terraform apply -var-file=example.tfvars 48 | ``` 49 | 50 | Review the plan and apply it by typing `yes` when the following prompt appears: 51 | 52 | ``` 53 | Do you want to perform these actions? 54 | Terraform will perform the actions described above. 55 | Only 'yes' will be accepted to approve. 56 | 57 | Enter a value: yes 58 | ``` 59 | 60 | It takes, on average, 15 minutes to create the EKS cluster and required S3 buckets. When it's done, it'll print out the cluster name and names of S3 buckets it created. 61 | 62 | This is how it can look on a successful run when it's finished: 63 | 64 | ``` 65 | Outputs: 66 | 67 | aws_region = "us-west-2" 68 | cluster_name = "cortex-operator-example-209f" 69 | cortex_config_bucket_name = "cortex-operator-example-209f-config" 70 | cortex_data_bucket_name = "cortex-operator-example-209f-data" 71 | ``` 72 | 73 | ## Set up kubeconfig 74 | 75 | When you've provisioned your EKS cluster, you need to configure kubectl. 76 | 77 | Run the following command to retrieve the access credentials for your cluster and automatically configure kubectl. 78 | 79 | ``` 80 | aws eks --region $(terraform output --raw aws_region) update-kubeconfig --name $(terraform output --raw cluster_name) 81 | ``` 82 | 83 | ## Teardown Infrastructure 84 | 85 | When you are done, don't forget to clean up everything with: 86 | 87 | ``` 88 | terraform destroy 89 | ``` 90 | -------------------------------------------------------------------------------- /docs/guides/terraform/eks.tf: -------------------------------------------------------------------------------- 1 | resource "aws_eks_cluster" "k8s-acc" { 2 | name = local.cluster_name 3 | version = var.kubernetes_version 4 | role_arn = aws_iam_role.k8s-acc-cluster.arn 5 | 6 | vpc_config { 7 | subnet_ids = aws_subnet.k8s-acc.*.id 8 | } 9 | 10 | # Ensure that IAM Role permissions are created before and deleted after EKS Cluster handling. 11 | # Otherwise, EKS will not be able to properly delete EKS managed EC2 infrastructure such as Security Groups. 12 | depends_on = [ 13 | aws_iam_role_policy_attachment.k8s-acc-AmazonEKSClusterPolicy, 14 | aws_iam_role_policy_attachment.k8s-acc-AmazonEKSVPCResourceController, 15 | ] 16 | } 17 | 18 | resource "aws_eks_node_group" "k8s-acc" { 19 | cluster_name = aws_eks_cluster.k8s-acc.name 20 | node_group_name = local.cluster_name 21 | node_role_arn = aws_iam_role.k8s-acc-node.arn 22 | subnet_ids = aws_subnet.k8s-acc.*.id 23 | 24 | scaling_config { 25 | desired_size = 2 26 | max_size = 2 27 | min_size = 2 28 | } 29 | 30 | # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling. 31 | # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces. 32 | depends_on = [ 33 | aws_iam_role_policy_attachment.k8s-acc-AmazonEKSWorkerNodePolicy, 34 | aws_iam_role_policy_attachment.k8s-acc-AmazonEKS_CNI_Policy, 35 | aws_iam_role_policy_attachment.k8s-acc-AmazonEC2ContainerRegistryReadOnly, 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /docs/guides/terraform/example.tfvars: -------------------------------------------------------------------------------- 1 | cluster_name_prefix = "cortex-operator-example" 2 | aws_region = "us-west-2" 3 | kubernetes_version = "1.18" 4 | -------------------------------------------------------------------------------- /docs/guides/terraform/iam.tf: -------------------------------------------------------------------------------- 1 | resource "aws_iam_role" "k8s-acc-cluster" { 2 | name = local.cluster_name 3 | 4 | assume_role_policy = < github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999 22 | 23 | replace k8s.io/client-go => k8s.io/client-go v0.20.4 24 | 25 | replace k8s.io/api => k8s.io/api v0.20.4 26 | 27 | // Use fork of gocql that has gokit logs and Prometheus metrics. 28 | replace github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 29 | 30 | // Using a 3rd-party branch for custom dialer - see https://github.com/bradfitz/gomemcache/pull/86 31 | replace github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab 32 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Opstrace, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "flag" 21 | "os" 22 | 23 | // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) 24 | // to ensure that exec-entrypoint and run can make use of them. 25 | _ "k8s.io/client-go/plugin/pkg/client/auth" 26 | 27 | "k8s.io/apimachinery/pkg/runtime" 28 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 29 | clientgoscheme "k8s.io/client-go/kubernetes/scheme" 30 | ctrl "sigs.k8s.io/controller-runtime" 31 | "sigs.k8s.io/controller-runtime/pkg/healthz" 32 | "sigs.k8s.io/controller-runtime/pkg/log/zap" 33 | 34 | cortexv1alpha1 "github.com/opstrace/cortex-operator/api/v1alpha1" 35 | "github.com/opstrace/cortex-operator/controllers" 36 | //+kubebuilder:scaffold:imports 37 | ) 38 | 39 | var ( 40 | scheme = runtime.NewScheme() 41 | setupLog = ctrl.Log.WithName("setup") 42 | ) 43 | 44 | func init() { 45 | utilruntime.Must(clientgoscheme.AddToScheme(scheme)) 46 | 47 | utilruntime.Must(cortexv1alpha1.AddToScheme(scheme)) 48 | //+kubebuilder:scaffold:scheme 49 | } 50 | 51 | func main() { 52 | var metricsAddr string 53 | var enableLeaderElection bool 54 | var probeAddr string 55 | flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") 56 | flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") 57 | flag.BoolVar(&enableLeaderElection, "leader-elect", false, 58 | "Enable leader election for controller manager. "+ 59 | "Enabling this will ensure there is only one active controller manager.") 60 | opts := zap.Options{ 61 | Development: true, 62 | } 63 | opts.BindFlags(flag.CommandLine) 64 | flag.Parse() 65 | 66 | ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) 67 | 68 | mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ 69 | Scheme: scheme, 70 | MetricsBindAddress: metricsAddr, 71 | Port: 9443, 72 | HealthProbeBindAddress: probeAddr, 73 | LeaderElection: enableLeaderElection, 74 | LeaderElectionID: "5b355f38.opstrace.io", 75 | }) 76 | if err != nil { 77 | setupLog.Error(err, "unable to start manager") 78 | os.Exit(1) 79 | } 80 | 81 | if err = (&controllers.CortexReconciler{ 82 | Client: mgr.GetClient(), 83 | Log: ctrl.Log.WithName("controllers").WithName("Cortex"), 84 | Scheme: mgr.GetScheme(), 85 | }).SetupWithManager(mgr); err != nil { 86 | setupLog.Error(err, "unable to create controller", "controller", "Cortex") 87 | os.Exit(1) 88 | } 89 | if err = (&controllers.MemcachedReconciler{ 90 | Client: mgr.GetClient(), 91 | Log: ctrl.Log.WithName("controllers").WithName("Memcached"), 92 | Scheme: mgr.GetScheme(), 93 | }).SetupWithManager(mgr); err != nil { 94 | setupLog.Error(err, "unable to create controller", "controller", "Memcached") 95 | os.Exit(1) 96 | } 97 | if err = (&controllers.CortexIngesterReconciler{ 98 | Client: mgr.GetClient(), 99 | Log: ctrl.Log.WithName("controllers").WithName("CortexIngester"), 100 | Scheme: mgr.GetScheme(), 101 | }).SetupWithManager(mgr); err != nil { 102 | setupLog.Error(err, "unable to create controller", "controller", "CortexIngester") 103 | os.Exit(1) 104 | } 105 | if err = (&controllers.CortexDistributorReconciler{ 106 | Client: mgr.GetClient(), 107 | Log: ctrl.Log.WithName("controllers").WithName("CortexDistributor"), 108 | Scheme: mgr.GetScheme(), 109 | }).SetupWithManager(mgr); err != nil { 110 | setupLog.Error(err, "unable to create controller", "controller", "CortexDistributor") 111 | os.Exit(1) 112 | } 113 | if err = (&controllers.CortexQuerierReconciler{ 114 | Client: mgr.GetClient(), 115 | Log: ctrl.Log.WithName("controllers").WithName("CortexQuerier"), 116 | Scheme: mgr.GetScheme(), 117 | }).SetupWithManager(mgr); err != nil { 118 | setupLog.Error(err, "unable to create controller", "controller", "CortexQuerier") 119 | os.Exit(1) 120 | } 121 | if err = (&controllers.CortexQueryFrontendReconciler{ 122 | Client: mgr.GetClient(), 123 | Log: ctrl.Log.WithName("controllers").WithName("CortexQueryFrontend"), 124 | Scheme: mgr.GetScheme(), 125 | }).SetupWithManager(mgr); err != nil { 126 | setupLog.Error(err, "unable to create controller", "controller", "CortexQueryFrontend") 127 | os.Exit(1) 128 | } 129 | if err = (&controllers.CortexCompactorReconciler{ 130 | Client: mgr.GetClient(), 131 | Log: ctrl.Log.WithName("controllers").WithName("CortexCompactor"), 132 | Scheme: mgr.GetScheme(), 133 | }).SetupWithManager(mgr); err != nil { 134 | setupLog.Error(err, "unable to create controller", "controller", "CortexCompactor") 135 | os.Exit(1) 136 | } 137 | if err = (&controllers.CortexStoreGatewayReconciler{ 138 | Client: mgr.GetClient(), 139 | Log: ctrl.Log.WithName("controllers").WithName("CortexStoreGateway"), 140 | Scheme: mgr.GetScheme(), 141 | }).SetupWithManager(mgr); err != nil { 142 | setupLog.Error(err, "unable to create controller", "controller", "CortexStoreGateway") 143 | os.Exit(1) 144 | } 145 | if err = (&controllers.CortexRuntimeConfigReconciler{ 146 | Client: mgr.GetClient(), 147 | Log: ctrl.Log.WithName("controllers").WithName("CortexRuntimeConfig"), 148 | Scheme: mgr.GetScheme(), 149 | }).SetupWithManager(mgr); err != nil { 150 | setupLog.Error(err, "unable to create controller", "controller", "CortexRuntimeConfig") 151 | os.Exit(1) 152 | } 153 | if err = (&controllers.CortexAlertManagerReconciler{ 154 | Client: mgr.GetClient(), 155 | Log: ctrl.Log.WithName("controllers").WithName("CortexAlertManager"), 156 | Scheme: mgr.GetScheme(), 157 | }).SetupWithManager(mgr); err != nil { 158 | setupLog.Error(err, "unable to create controller", "controller", "CortexAlertManager") 159 | os.Exit(1) 160 | } 161 | if err = (&controllers.CortexRulerReconciler{ 162 | Client: mgr.GetClient(), 163 | Log: ctrl.Log.WithName("controllers").WithName("CortexRuler"), 164 | Scheme: mgr.GetScheme(), 165 | }).SetupWithManager(mgr); err != nil { 166 | setupLog.Error(err, "unable to create controller", "controller", "CortexRuler") 167 | os.Exit(1) 168 | } 169 | if err = (&cortexv1alpha1.Cortex{}).SetupWebhookWithManager(mgr); err != nil { 170 | setupLog.Error(err, "unable to create webhook", "webhook", "Cortex") 171 | os.Exit(1) 172 | } 173 | //+kubebuilder:scaffold:builder 174 | 175 | if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { 176 | setupLog.Error(err, "unable to set up health check") 177 | os.Exit(1) 178 | } 179 | if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { 180 | setupLog.Error(err, "unable to set up ready check") 181 | os.Exit(1) 182 | } 183 | 184 | setupLog.Info("starting manager") 185 | if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { 186 | setupLog.Error(err, "problem running manager") 187 | os.Exit(1) 188 | } 189 | } 190 | --------------------------------------------------------------------------------