├── .gitattributes ├── .github └── dependabot.yaml ├── .gitignore ├── .golangci.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── OWNERS ├── README.md ├── RELEASE.md ├── SECURITY_CONTACTS ├── code-of-conduct.md ├── docs ├── getting-started.md └── metrics-ref.md ├── go.mod ├── go.sum ├── hack ├── boilerplate.go.txt └── tools.go ├── pkg ├── apiserver │ ├── apiserver.go │ ├── cmapis.go │ ├── emapis.go │ ├── endpoints │ │ └── handlers │ │ │ ├── get.go │ │ │ └── rest.go │ ├── installer │ │ ├── apiserver_test.go │ │ ├── cmhandlers.go │ │ ├── conversion.go │ │ ├── emhandlers.go │ │ └── installer.go │ ├── metrics │ │ ├── metrics.go │ │ └── metrics_test.go │ └── registry │ │ └── rest │ │ └── rest.go ├── cmd │ ├── builder.go │ ├── builder_test.go │ └── options │ │ ├── options.go │ │ └── options_test.go ├── dynamicmapper │ ├── mapper.go │ └── mapper_test.go ├── generated │ ├── openapi-gen.go │ └── openapi │ │ ├── core │ │ ├── doc.go │ │ └── zz_generated.openapi.go │ │ ├── custommetrics │ │ ├── doc.go │ │ └── zz_generated.openapi.go │ │ └── externalmetrics │ │ ├── doc.go │ │ └── zz_generated.openapi.go ├── provider │ ├── defaults │ │ └── default_metric_providers.go │ ├── errors.go │ ├── fake │ │ └── fake.go │ ├── helpers │ │ └── helpers.go │ ├── interfaces.go │ ├── interfaces_test.go │ └── resource_lister.go └── registry │ ├── custom_metrics │ └── reststorage.go │ └── external_metrics │ └── reststorage.go ├── test-adapter-deploy ├── Dockerfile ├── README.md └── testing-adapter.yaml └── test-adapter ├── README.md ├── main.go └── provider └── provider.go /.gitattributes: -------------------------------------------------------------------------------- 1 | **/zz_generated.*.go linguist-generated=true 2 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | groups: 6 | k8s-dependencies: 7 | patterns: 8 | - "k8s.io*" 9 | schedule: 10 | interval: "weekly" 11 | - package-ecosystem: docker 12 | directory: "/" 13 | schedule: 14 | interval: weekly 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *~ 3 | _output 4 | apiserver.local.config/ 5 | .idea/ 6 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 7m 3 | 4 | linters: 5 | disable-all: true 6 | enable: 7 | - bodyclose 8 | - copyloopvar 9 | - dogsled 10 | - dupl 11 | - errcheck 12 | - gocritic 13 | - gocyclo 14 | - gofmt 15 | - goimports 16 | - gosec 17 | - goprintffuncname 18 | - gosimple 19 | - govet 20 | - ineffassign 21 | - misspell 22 | - nakedret 23 | - nolintlint 24 | - revive 25 | - staticcheck 26 | - stylecheck 27 | - typecheck 28 | - unconvert 29 | - unused 30 | - whitespace 31 | 32 | linters-settings: 33 | goimports: 34 | local-prefixes: sigs.k8s.io/custom-metrics-apiserver 35 | misspell: 36 | ignore-words: 37 | - "creater" # Cf. e.g. https://pkg.go.dev/k8s.io/apimachinery@v0.24.3/pkg/runtime#ObjectCreater 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing guidelines 2 | 3 | ## How to become a contributor and submit your own code 4 | 5 | ### Contributor License Agreements 6 | 7 | We'd love to accept your patches! Before we can take them, we have to jump a couple of legal hurdles. 8 | 9 | Please fill out either the individual or corporate Contributor License Agreement (CLA). 10 | 11 | * If you are an individual writing original source code and you're sure you own the intellectual property, then you'll need to sign an [individual CLA](https://identity.linuxfoundation.org/node/285/node/285/individual-signup). 12 | * If you work for a company that wants to allow you to contribute your work, then you'll need to sign a [corporate CLA](https://identity.linuxfoundation.org/node/285/organization-signup). 13 | 14 | Follow either of the two links above to access the appropriate CLA and instructions for how to sign and return it. Once we receive it, we'll be able to accept your pull requests. 15 | 16 | ### Contributing A Patch 17 | 18 | 1. Submit an issue describing your proposed change to the repo in question. 19 | 1. The [repo owners](OWNERS) will respond to your issue promptly. 20 | 1. If your proposed change is accepted, and you haven't already done so, sign a Contributor License Agreement (see details above). 21 | 1. Fork the desired repo, develop and test your code changes. 22 | 1. Submit a pull request. 23 | 24 | ### Adding dependencies 25 | 26 | The project follows a standard Go project layout, see more about [dependency-management](https://github.com/kubernetes/community/blob/master/contributors/devel/development.md#dependency-management). 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | REGISTRY?=kubernetes-sigs 2 | IMAGE?=k8s-test-metrics-adapter 3 | TEMP_DIR:=$(shell mktemp -d) 4 | ARCH?=amd64 5 | OUT_DIR?=./_output 6 | GOPATH:=$(shell go env GOPATH) 7 | 8 | VERSION?=latest 9 | 10 | GOLANGCI_VERSION:=1.64.8 11 | 12 | .PHONY: all 13 | all: build-test-adapter 14 | 15 | 16 | # Generate 17 | # -------- 18 | 19 | generated_openapis := core custommetrics externalmetrics 20 | generated_files := $(generated_openapis:%=pkg/generated/openapi/%/zz_generated.openapi.go) 21 | 22 | pkg/generated/openapi/core/zz_generated.openapi.go: INPUTS := "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/version" "k8s.io/api/core/v1" 23 | pkg/generated/openapi/custommetrics/zz_generated.openapi.go: INPUTS := "k8s.io/metrics/pkg/apis/custom_metrics" "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1" "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2" 24 | pkg/generated/openapi/externalmetrics/zz_generated.openapi.go: INPUTS := "k8s.io/metrics/pkg/apis/external_metrics" "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" 25 | 26 | pkg/generated/openapi/%/zz_generated.openapi.go: go.mod go.sum 27 | go install -mod=readonly k8s.io/kube-openapi/cmd/openapi-gen 28 | $(GOPATH)/bin/openapi-gen --logtostderr \ 29 | --go-header-file ./hack/boilerplate.go.txt \ 30 | --output-pkg ./$(@D) \ 31 | --output-file zz_generated.openapi.go \ 32 | --output-dir ./$(@D) \ 33 | -r /dev/null \ 34 | $(INPUTS) 35 | 36 | .PHONY: update-generated 37 | update-generated: $(generated_files) 38 | 39 | 40 | # Build 41 | # ----- 42 | 43 | .PHONY: build-test-adapter 44 | build-test-adapter: update-generated 45 | CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -o $(OUT_DIR)/$(ARCH)/test-adapter sigs.k8s.io/custom-metrics-apiserver/test-adapter 46 | 47 | 48 | # Format and lint 49 | # --------------- 50 | 51 | HAS_GOLANGCI_VERSION:=$(shell $(GOPATH)/bin/golangci-lint version --format=short) 52 | .PHONY: golangci 53 | golangci: 54 | ifneq ($(HAS_GOLANGCI_VERSION), $(GOLANGCI_VERSION)) 55 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v$(GOLANGCI_VERSION) 56 | endif 57 | 58 | .PHONY: verify-lint 59 | verify-lint: golangci 60 | $(GOPATH)/bin/golangci-lint run --modules-download-mode=readonly || (echo 'Run "make update-lint"' && exit 1) 61 | 62 | .PHONY: update-lint 63 | update-lint: golangci 64 | $(GOPATH)/bin/golangci-lint run --fix --modules-download-mode=readonly 65 | 66 | 67 | # License 68 | # ------- 69 | 70 | HAS_ADDLICENSE:=$(shell which $(GOPATH)/bin/addlicense) 71 | .PHONY: verify-licenses 72 | verify-licenses:addlicense 73 | find -type f -name "*.go" | xargs $(GOPATH)/bin/addlicense -check || (echo 'Run "make update-licenses"' && exit 1) 74 | 75 | .PHONY: update-licenses 76 | update-licenses: addlicense 77 | find -type f -name "*.go" | xargs $(GOPATH)/bin/addlicense -c "The Kubernetes Authors." 78 | 79 | .PHONY: addlicense 80 | addlicense: 81 | ifndef HAS_ADDLICENSE 82 | go install -mod=readonly github.com/google/addlicense 83 | endif 84 | 85 | 86 | # Verify 87 | # ------ 88 | 89 | .PHONY: verify 90 | verify: verify-deps verify-lint verify-licenses verify-generated 91 | 92 | .PHONY: verify-deps 93 | verify-deps: 94 | go mod verify 95 | go mod tidy 96 | @git diff --exit-code -- go.sum go.mod 97 | 98 | .PHONY: verify-generated 99 | verify-generated: update-generated 100 | @git diff --exit-code -- $(generated_files) 101 | 102 | 103 | # Test 104 | # ---- 105 | 106 | .PHONY: test 107 | test: 108 | CGO_ENABLED=0 go test ./pkg/... 109 | 110 | .PHONY: test-adapter-container 111 | test-adapter-container: build-test-adapter 112 | cp test-adapter-deploy/Dockerfile $(TEMP_DIR) 113 | cp $(OUT_DIR)/$(ARCH)/test-adapter $(TEMP_DIR)/adapter 114 | cd $(TEMP_DIR) && sed -i.bak "s|BASEIMAGE|scratch|g" Dockerfile 115 | docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(VERSION) $(TEMP_DIR) 116 | 117 | .PHONY: test-kind 118 | test-kind: 119 | kind load docker-image $(REGISTRY)/$(IMAGE)-$(ARCH):$(VERSION) 120 | sed 's|REGISTRY|'${REGISTRY}'|g' test-adapter-deploy/testing-adapter.yaml | kubectl apply -f - 121 | kubectl rollout restart -n custom-metrics deployment/custom-metrics-apiserver 122 | -------------------------------------------------------------------------------- /OWNERS: -------------------------------------------------------------------------------- 1 | approvers: 2 | - dgrisonnet 3 | - RainbowMango 4 | reviewers: 5 | - dgrisonnet 6 | - RainbowMango 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Custom Metrics Adapter Server Boilerplate 2 | 3 | [![Go Reference](https://pkg.go.dev/badge/sigs.k8s.io/custom-metrics-apiserver.svg)](https://pkg.go.dev/sigs.k8s.io/custom-metrics-apiserver) 4 | 5 | ## Purpose 6 | 7 | This repository contains boilerplate code for setting up an implementation 8 | of the [Metrics APIs](https://github.com/kubernetes/metrics): 9 | 10 | - Custom metrics (`k8s.io/metrics/pkg/apis/custom_metrics`) 11 | - External metrics (`k8s.io/metrics/pkg/apis/external_metrics`) 12 | 13 | It includes the necessary boilerplate for setting up an implementation 14 | (generic API server setup, registration of resources, etc), plus an 15 | implementation for testing that allows setting custom metric values over HTTP. 16 | 17 | ## How to use this repository 18 | 19 | This repository is designed to be used as a library. First, implement one 20 | or more of the metrics provider interfaces in `pkg/provider` (for example, 21 | `CustomMetricsProvider`), depending on which APIs you want to support. 22 | 23 | Then, use the `AdapterBase` in `pkg/cmd` to initialize the necessary flags 24 | and set up the API server, passing in your providers. 25 | 26 | More information can be found in the [getting started 27 | guide](/docs/getting-started.md), and the testing implementation can be 28 | found in the [test-adapter directory](/test-adapter). 29 | 30 | ### Prerequisites 31 | 32 | [Go](https://go.dev/doc/install): this library requires the same version of 33 | [Go as Kubernetes](https://git.k8s.io/community/contributors/devel/development.md#go). 34 | 35 | ## Test Adapter 36 | 37 | There is a test adapter in this repository that can be used for testing 38 | changes to the repository, as a mock implementation of the APIs for 39 | automated unit tests, and also as an example implementation. 40 | 41 | Note that this adapter **should not** be used for production. It's for 42 | writing automated e2e tests and serving as a sample only. 43 | 44 | To build it: 45 | 46 | ```bash 47 | # build the test-adapter container as $REGISTRY/k8s-test-metrics-adapter-amd64 48 | export REGISTRY= 49 | make test-adapter-container 50 | ``` 51 | 52 | To test it locally, install [kind](https://kind.sigs.k8s.io/docs/user/quick-start/), 53 | then create a cluster, and start the tests: 54 | ```bash 55 | kind create cluster 56 | make test-kind 57 | ``` 58 | 59 | To test it in an arbitraty Kubernetes cluster: 60 | ```bash 61 | # push the container up to a registry 62 | docker push $REGISTRY/k8s-test-metrics-adapter-amd64 63 | 64 | # launch the adapter using the test adapter deployment manifest 65 | kubectl apply -f test-adapter-deploy/testing-adapter.yaml 66 | ``` 67 | 68 | When the deployment is ready, you can define new metrics on the test adapter 69 | by querying the write endpoint: 70 | 71 | ```bash 72 | # set up a proxy to the API server so we can access write endpoints 73 | # of the testing adapter directly 74 | kubectl proxy & 75 | # write a sample metric -- the write paths match the same URL structure 76 | # as the read paths, but at the /write-metrics base path. 77 | # data needs to be in json, so we also need to set the content-type header 78 | curl -X POST \ 79 | -H 'Content-Type: application/json' \ 80 | http://localhost:8001/api/v1/namespaces/custom-metrics/services/custom-metrics-apiserver:http/proxy/write-metrics/namespaces/default/services/kubernetes/test-metric \ 81 | --data-raw '"300m"' 82 | ``` 83 | 84 | ```bash 85 | # you can pipe to `jq .` to pretty-print the output, if it's installed 86 | # (otherwise, it's not necessary) 87 | kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta2" | jq . 88 | # fetching certain custom metrics of namespaced resources 89 | kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta2/namespaces/default/services/kubernetes/test-metric" | jq . 90 | ``` 91 | 92 | If you wanted to target a simple nginx-deployment and then use this as an HPA scaler metric, something like this would work following the previous curl command: 93 | 94 | ```yaml 95 | apiVersion: autoscaling/v2 96 | kind: HorizontalPodAutoscaler 97 | metadata: 98 | name: nginx-deployment 99 | namespace: default 100 | spec: 101 | scaleTargetRef: 102 | apiVersion: apps/v1 103 | kind: Deployment 104 | name: nginx-deployment 105 | minReplicas: 1 106 | maxReplicas: 10 107 | metrics: 108 | - type: Object 109 | object: 110 | metric: 111 | name: test-metric 112 | describedObject: 113 | apiVersion: v1 114 | kind: Service 115 | name: kubernetes 116 | target: 117 | type: Value 118 | value: 300m 119 | ``` 120 | 121 | You can also query the external metrics: 122 | 123 | ```bash 124 | kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1" | jq . 125 | # fetching certain custom metrics of namespaced resources 126 | kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/my-external-metric" | jq . 127 | ``` 128 | 129 | ## Compatibility 130 | 131 | The APIs in this repository follow the standard guarantees for Kubernetes 132 | APIs, and will follow Kubernetes releases. 133 | 134 | ## Community, discussion, contribution, and support 135 | 136 | Learn how to engage with the Kubernetes community on the 137 | [community page](https://kubernetes.io/community/). 138 | 139 | You can reach the maintainers of this repository at: 140 | 141 | - [Slack](https://slack.k8s.io/): channel `#sig-instrumentation` 142 | - [Mailing List](https://groups.google.com/g/kubernetes-sig-instrumentation) 143 | 144 | ### Code of Conduct 145 | 146 | Participation in the Kubernetes community is governed by the [Kubernetes 147 | Code of Conduct](code-of-conduct.md). 148 | 149 | ### Contribution Guidelines 150 | 151 | See [CONTRIBUTING.md](CONTRIBUTING.md) for more information. 152 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Release Process 2 | 3 | The custom-metrics-apiserver library is released following Kubernetes releases. 4 | The major and minor versions of custom-metrics-apiserver are in sync with 5 | upstream Kubernetes and the patch version is reserved for changes in 6 | custom-metrics-apiserver that we want to release without having to wait for the 7 | next version of Kubernetes. 8 | 9 | The process is as follow and should always be in sync with Kubernetes: 10 | 11 | 1. A new Kubernetes minor version is released 12 | 1. An issue is proposing a new release with a changelog containing the changes 13 | since the last minor release 14 | 1. At least one [approver](OWNERS) must LGTM this release 15 | 1. A PR that bumps Kubernetes dependencies to the latest version is created and 16 | merged. The major and minor version of the dependencies should be in sync 17 | with the version we are releasing. 18 | 1. An OWNER creates a draft GitHub release 19 | 1. An OWNER creates a release tag using `git tag -s $VERSION`, inserts the 20 | changelog and pushes the tag with `git push $VERSION` 21 | 1. An OWNER creates and pushes a release branch named `release-X.Y` 22 | 1. An OWNER publishes the GitHub release 23 | 1. An announcement email is sent to 24 | `kubernetes-sig-instrumentation@googlegroups.com` with the subject 25 | `[ANNOUNCE] custom-metrics-apiserver $VERSION is released` 26 | 1. The release issue is closed 27 | -------------------------------------------------------------------------------- /SECURITY_CONTACTS: -------------------------------------------------------------------------------- 1 | # Defined below are the security contacts for this repo. 2 | # 3 | # They are the contact point for the Product Security Team to reach out 4 | # to for triaging and handling of incoming issues. 5 | # 6 | # The below names agree to abide by the 7 | # [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) 8 | # and will be removed and replaced if they violate that agreement. 9 | # 10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE 11 | # INSTRUCTIONS AT https://kubernetes.io/security/ 12 | 13 | dgrisonnet 14 | kawych 15 | luxas 16 | s-urbaniak 17 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Community Code of Conduct 2 | 3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) 4 | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting started with developing your own Custom Metrics API Server 2 | 3 | This will walk through writing a very basic custom metrics API server using 4 | this library. The implementation will be static. With a real adapter, you'd 5 | generally be reading from some external metrics system instead. 6 | 7 | The end result will look similar to the [test adapter](/test-adapter), but 8 | will generate sample metrics automatically instead of setting them via an 9 | HTTP endpoint. 10 | 11 | ## Prerequisites 12 | 13 | Create a project and initialize the dependencies like so: 14 | 15 | ```shell 16 | $ go mod init example.com/youradapter 17 | $ go get sigs.k8s.io/custom-metrics-apiserver@latest 18 | ``` 19 | 20 | ## Writing the Code 21 | 22 | There's two parts to an adapter: the setup code, and the providers. The 23 | setup code initializes the API server, and the providers handle requests 24 | from the API for metrics. 25 | 26 | ### Writing a provider 27 | 28 | There are currently two provider interfaces, corresponding to two 29 | different APIs: the custom metrics API (for metrics that describe 30 | Kubernetes objects), and the external metrics API (for metrics that don't 31 | describe Kubernetes objects, or are otherwise not attached to a particular 32 | object). For the sake of brevity, this walkthrough will show an example of 33 | the custom metrics API, but a full example including the external metrics 34 | API can be found in the [test adapter](/test-adapter). 35 | 36 | Put your provider in the `pkg/provider` directory in your repository. 37 | 38 |
39 | 40 | To get started, you'll need some imports: 41 | 42 | ```go 43 | package provider 44 | 45 | import ( 46 | "context" 47 | "time" 48 | 49 | apimeta "k8s.io/apimachinery/pkg/api/meta" 50 | "k8s.io/apimachinery/pkg/api/resource" 51 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 52 | "k8s.io/apimachinery/pkg/labels" 53 | "k8s.io/apimachinery/pkg/runtime/schema" 54 | "k8s.io/apimachinery/pkg/types" 55 | "k8s.io/client-go/dynamic" 56 | "k8s.io/metrics/pkg/apis/custom_metrics" 57 | 58 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 59 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/defaults" 60 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/helpers" 61 | ) 62 | ``` 63 | 64 |
65 | 66 | The custom metrics provider interface, which you'll need to implement, is 67 | called `CustomMetricsProvider`, and looks like this: 68 | 69 | ```go 70 | type CustomMetricsProvider interface { 71 | ListAllMetrics() []CustomMetricInfo 72 | 73 | GetMetricByName(ctx context.Context, name types.NamespacedName, info CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) 74 | GetMetricBySelector(ctx context.Context, namespace string, selector labels.Selector, info CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error)} 75 | ``` 76 | 77 | First, there's a method for listing all metrics available at any point in 78 | time. It's used to populate the discovery information in the API, so that 79 | clients can know what metrics are available. It's not allowed to fail (it 80 | doesn't return any error), and it should return quickly. 81 | 82 | You can list your metrics (asynchronously) and return them on every request. 83 | This is not mandatory because kubernetes can request metric values without 84 | listing them before, but maybe there are some cases where is useful. To 85 | provide a unified solution, a default implementation is provided thanks to 86 | `DefaultCustomMetricsProvider` (and `DefaultExternalMetricsProvider` for 87 | external metrics) 88 | 89 | Next, you'll need to implement the methods that actually fetch the 90 | metrics. There are methods for fetching metrics describing arbitrary Kubernetes 91 | resources, both root-scoped and namespaced-scoped. Those metrics can 92 | either be fetched for a single object, or for a list of objects by 93 | selector. 94 | 95 |
96 | 97 | Make sure you understand resources, scopes, and quantities before 98 | continuing. 99 | 100 | --- 101 | 102 | #### Kinds, Resources, and Scopes 103 | 104 | When working with these APIs (and Kubernetes APIs in general), you'll 105 | often see references to resources, kinds, and scope. 106 | 107 | Kinds refer to types within the API. For instance, `Deployment` is 108 | a kind. When you fully-qualify a kind, you write it as 109 | a *group-version-kind*, or *GVK* for short. The GVK for `Deployment` is 110 | `apps/v1.Deployment` (or `{Group: "apps", Version: "v1", Kind: 111 | "Deployment"}` in Go syntax). 112 | 113 | Resources refer to URLs in an API, in an abstract sense. Each resource 114 | has a corresponding kind, but that kind isn't necessarily unique to that 115 | particular resource. For instance, the resource `deployments` has kind 116 | `Deployment`, but so does the subresource `deployments/status`. On the 117 | other hand, all the `*/scale` subresources use the kind `Scale` (for the 118 | most part). When you fully-qualify a resource, you write it as 119 | a *group-version-resource*, or GVR for short. The GVR `{Group: "apps", 120 | Version: "v1", Resource: "deployments"}` corresponds to the URL form 121 | `/apis/apps/v1/namespaces//deployments`. Resources may be singular or 122 | plural -- both effectively refer to the same thing. 123 | 124 | Sometimes you might partially qualify a particular kind or resource as 125 | a *group-kind* or *group-resource*, leaving off the versions. You write 126 | group-resources as `.`, like `deployments.apps` in the 127 | custom metrics API (and in kubectl). 128 | 129 | Scope refers to whether or not a particular resource is grouped under 130 | namespaces. You say that namespaced resources, like `deployments` are 131 | *namespace-scoped*, while non-namespaced resources, like `nodes` are 132 | *root-scoped*. 133 | 134 | To figure out which kinds correspond to which resources, and which 135 | resources have what scope, you use a `RESTMapper`. The `RESTMapper` 136 | generally collects its information from *discovery* information, which 137 | lists which kinds and resources are available in a particular Kubernetes 138 | cluster. 139 | 140 | #### Quantities 141 | 142 | When dealing with metrics, you'll often need to deal with fractional 143 | numbers. While many systems use floating point numbers for that purpose, 144 | Kubernetes instead uses a system called *quantities*. 145 | 146 | Quantities are whole numbers suffixed with SI suffixes. You use the `m` 147 | suffix (for milli-units) to denote numbers with fractional components, 148 | down the thousandths place. 149 | 150 | For instance, `10500m` means `10.5` in decimal notation. To construct 151 | a new quantity out of a milli-unit value (e.g. millicores or millimeters), 152 | you'd use the `resource.NewMilliQuantity(valueInMilliUnits, 153 | resource.DecimalSI)` function. To construct a new quantity that's a whole 154 | number, you can either use `NewMilliQuantity` and multiple by `1000`, or 155 | use the `resource.NewQuantity(valueInWholeUnits, resource.DecimalSI)` 156 | function. 157 | 158 | Remember that in both cases, the argument *must* be an integer, so if you 159 | need to represent a number with a fractional component, use 160 | `NewMilliQuantity`. 161 | 162 | --- 163 | 164 |
165 | 166 | You'll need a handle to a RESTMapper (to map between resources and kinds) 167 | and dynamic client to fetch lists of objects in the cluster, if you don't 168 | already have sufficient information in your metrics pipeline: 169 | 170 | ```go 171 | type yourProvider struct { 172 | defaults.DefaultCustomMetricsProvider 173 | defaults.DefaultExternalMetricsProvider 174 | client dynamic.Interface 175 | mapper apimeta.RESTMapper 176 | 177 | // just increment values when they're requested 178 | values map[provider.CustomMetricInfo]int64 179 | } 180 | 181 | func NewProvider(client dynamic.Interface, mapper apimeta.RESTMapper) provider.CustomMetricsProvider { 182 | return &yourProvider{ 183 | client: client, 184 | mapper: mapper, 185 | values: make(map[provider.CustomMetricInfo]int64), 186 | } 187 | } 188 | ``` 189 | 190 | Then, you can implement the methods that fetch the metrics. In this 191 | walkthrough, those methods will just increment values for metrics as 192 | they're fetched. In real adapter, you'd want to fetch metrics from your 193 | backend in these methods. 194 | 195 | First, a couple of helpers, which support doing the fake "fetch" 196 | operation, and constructing a result object: 197 | 198 | ```go 199 | // valueFor fetches a value from the fake list and increments it. 200 | func (p *yourProvider) valueFor(info provider.CustomMetricInfo) (int64, error) { 201 | // normalize the value so that you treat plural resources and singular 202 | // resources the same (e.g. pods vs pod) 203 | info, _, err := info.Normalized(p.mapper) 204 | if err != nil { 205 | return 0, err 206 | } 207 | 208 | value := p.values[info] 209 | value += 1 210 | p.values[info] = value 211 | 212 | return value, nil 213 | } 214 | 215 | // metricFor constructs a result for a single metric value. 216 | func (p *yourProvider) metricFor(value int64, name types.NamespacedName, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) { 217 | // construct a reference referring to the described object 218 | objRef, err := helpers.ReferenceFor(p.mapper, name, info) 219 | if err != nil { 220 | return nil, err 221 | } 222 | 223 | return &custom_metrics.MetricValue{ 224 | DescribedObject: objRef, 225 | Metric: custom_metrics.MetricIdentifier{ 226 | Name: info.Metric, 227 | }, 228 | // you'll want to use the actual timestamp in a real adapter 229 | Timestamp: metav1.Time{time.Now()}, 230 | Value: *resource.NewMilliQuantity(value*100, resource.DecimalSI), 231 | }, nil 232 | } 233 | ``` 234 | 235 | Then, you'll need to implement the two main methods. The first fetches 236 | a single metric value for one object (for example, for the `object` metric 237 | type in the HorizontalPodAutoscaler): 238 | 239 | ```go 240 | func (p *yourProvider) GetMetricByName(ctx context.Context, name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) { 241 | value, err := p.valueFor(info) 242 | if err != nil { 243 | return nil, err 244 | } 245 | return p.metricFor(value, name, info) 246 | } 247 | ``` 248 | 249 | The second fetches multiple metric values, one for each object in a set 250 | (for example, for the `pods` metric type in the HorizontalPodAutoscaler). 251 | 252 | ```go 253 | func (p *yourProvider) GetMetricBySelector(ctx context.Context, namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) { 254 | totalValue, err := p.valueFor(info) 255 | if err != nil { 256 | return nil, err 257 | } 258 | 259 | names, err := helpers.ListObjectNames(p.mapper, p.client, namespace, selector, info) 260 | if err != nil { 261 | return nil, err 262 | } 263 | 264 | res := make([]custom_metrics.MetricValue, len(names)) 265 | for i, name := range names { 266 | // in a real adapter, you might want to consider pre-computing the 267 | // object reference created in metricFor, instead of recomputing it 268 | // for each object. 269 | value, err := p.metricFor(100*totalValue/int64(len(res)), types.NamespacedName{Namespace: namespace, Name: name}, info) 270 | if err != nil { 271 | return nil, err 272 | } 273 | res[i] = *value 274 | } 275 | 276 | return &custom_metrics.MetricValueList{ 277 | Items: res, 278 | }, nil 279 | } 280 | ``` 281 | 282 | Now, you just need to plug in your provider to an API server. 283 | 284 | ### Writing the setup code 285 | 286 | The adapter library provides helpers to construct an API server to serve 287 | the metrics provided by your provider. 288 | 289 |
290 | 291 | First, you'll need a few imports: 292 | 293 | ```go 294 | package main 295 | 296 | import ( 297 | "flag" 298 | "os" 299 | 300 | "k8s.io/apimachinery/pkg/util/wait" 301 | "k8s.io/component-base/logs" 302 | "k8s.io/klog/v2" 303 | 304 | basecmd "sigs.k8s.io/custom-metrics-apiserver/pkg/cmd" 305 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 306 | 307 | // make this the path to the provider that you just wrote 308 | yourprov "example.com/youradapter/pkg/provider" 309 | ) 310 | ``` 311 | 312 |
313 | 314 | With those out of the way, you can make use of the `basecmd.AdapterBase` 315 | struct to help set up the API server: 316 | 317 | ```go 318 | type YourAdapter struct { 319 | basecmd.AdapterBase 320 | 321 | // the message printed on startup 322 | Message string 323 | } 324 | 325 | func main() { 326 | logs.InitLogs() 327 | defer logs.FlushLogs() 328 | 329 | // initialize the flags, with one custom flag for the message 330 | cmd := &YourAdapter{} 331 | cmd.Flags().StringVar(&cmd.Message, "msg", "starting adapter...", "startup message") 332 | // make sure you get the klog flags 333 | logs.AddGoFlags(flag.CommandLine) 334 | cmd.Flags().AddGoFlagSet(flag.CommandLine) 335 | cmd.Flags().Parse(os.Args) 336 | 337 | provider := cmd.makeProviderOrDie() 338 | cmd.WithCustomMetrics(provider) 339 | // you could also set up external metrics support, 340 | // if your provider supported it: 341 | // cmd.WithExternalMetrics(provider) 342 | 343 | klog.Infof(cmd.Message) 344 | if err := cmd.Run(wait.NeverStop); err != nil { 345 | klog.Fatalf("unable to run custom metrics adapter: %v", err) 346 | } 347 | } 348 | ``` 349 | 350 | Finally, you'll need to add a bit of setup code for the specifics of your 351 | provider. This code will be specific to the options of your provider -- 352 | you might need to pass configuration for connecting to the backing metrics 353 | solution, extra credentials, or advanced configuration. For the provider 354 | you wrote above, the setup code looks something like this: 355 | 356 | ```go 357 | func (a *YourAdapter) makeProviderOrDie() provider.CustomMetricsProvider { 358 | client, err := a.DynamicClient() 359 | if err != nil { 360 | klog.Fatalf("unable to construct dynamic client: %v", err) 361 | } 362 | 363 | mapper, err := a.RESTMapper() 364 | if err != nil { 365 | klog.Fatalf("unable to construct discovery REST mapper: %v", err) 366 | } 367 | 368 | return yourprov.NewProvider(client, mapper) 369 | } 370 | ``` 371 | 372 | Then add the missing dependencies with: 373 | 374 | ```shell 375 | $ go mod tidy 376 | ``` 377 | 378 | ## Build the project 379 | 380 | Now that you have a working adapter, you can build it with `go build`, and 381 | stick in it a container, and deploy it onto the cluster. Check out the 382 | [test adapter deployment files](/test-adapter-deploy) for an example of 383 | how to do that. 384 | -------------------------------------------------------------------------------- /docs/metrics-ref.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Metrics Reference 4 | 5 | This page details the metrics that the custom metrics adapter exposes by default. Many of the exposed metrics are created in this project's dependencies. Generating this doc is currently a manual process. 6 | 7 | ### List of Stable Kubernetes Metrics 8 | 9 | Stable metrics observe strict API contracts and no labels can be added or removed from stable metrics during their lifetime. 10 | 11 | 12 | #### **apiserver_current_inflight_requests** 13 | Maximal number of currently used inflight request limit of this apiserver per request kind in last second. 14 | 15 | - **Stability Level:** STABLE 16 | - **Type:** Gauge 17 | - **Labels:** 18 | 19 | - request_kind 20 | 21 | #### **apiserver_request_duration_seconds** 22 | Response latency distribution in seconds for each verb, dry run value, group, version, resource, subresource, scope and component. 23 | 24 | - **Stability Level:** STABLE 25 | - **Type:** Histogram 26 | - **Labels:** 27 | 28 | - component 29 | - dry_run 30 | - group 31 | - resource 32 | - scope 33 | - subresource 34 | - verb 35 | - version 36 | 37 | #### **apiserver_request_total** 38 | Counter of apiserver requests broken out for each verb, dry run value, group, version, resource, scope, component, and HTTP response code. 39 | 40 | - **Stability Level:** STABLE 41 | - **Type:** Counter 42 | - **Labels:** 43 | 44 | - code 45 | - component 46 | - dry_run 47 | - group 48 | - resource 49 | - scope 50 | - subresource 51 | - verb 52 | - version 53 | 54 | #### **apiserver_response_sizes** 55 | Response size distribution in bytes for each group, version, verb, resource, subresource, scope and component. 56 | 57 | - **Stability Level:** STABLE 58 | - **Type:** Histogram 59 | - **Labels:** 60 | 61 | - component 62 | - group 63 | - resource 64 | - scope 65 | - subresource 66 | - verb 67 | - version 68 | 69 | 70 | ### List of Beta Kubernetes Metrics 71 | 72 | Beta metrics observe a looser API contract than its stable counterparts. No labels can be removed from beta metrics during their lifetime, however, labels can be added while the metric is in the beta stage. This offers the assurance that beta metrics will honor existing dashboards and alerts, while allowing for amendments in the future. 73 | 74 | 75 | #### **disabled_metrics_total** 76 | The count of disabled metrics. 77 | 78 | - **Stability Level:** BETA 79 | - **Type:** Counter 80 | 81 | 82 | #### **hidden_metrics_total** 83 | The count of hidden metrics. 84 | 85 | - **Stability Level:** BETA 86 | - **Type:** Counter 87 | 88 | 89 | #### **registered_metrics_total** 90 | The count of registered metrics broken by stability level and deprecation version. 91 | 92 | - **Stability Level:** BETA 93 | - **Type:** Counter 94 | - **Labels:** 95 | 96 | - deprecated_version 97 | - stability_level 98 | 99 | 100 | ### List of Alpha Kubernetes Metrics 101 | 102 | Alpha metrics do not have any API guarantees. These metrics must be used at your own risk, subsequent versions of Kubernetes may remove these metrics altogether, or mutate the API in such a way that breaks existing dashboards and alerts. 103 | 104 | 105 | #### **aggregator_discovery_aggregation_count_total** 106 | Counter of number of times discovery was aggregated 107 | 108 | - **Stability Level:** ALPHA 109 | - **Type:** Counter 110 | 111 | 112 | #### **apiserver_audit_event_total** 113 | Counter of audit events generated and sent to the audit backend. 114 | 115 | - **Stability Level:** ALPHA 116 | - **Type:** Counter 117 | 118 | 119 | #### **apiserver_audit_requests_rejected_total** 120 | Counter of apiserver requests rejected due to an error in audit logging backend. 121 | 122 | - **Stability Level:** ALPHA 123 | - **Type:** Counter 124 | 125 | 126 | #### **apiserver_client_certificate_expiration_seconds** 127 | Distribution of the remaining lifetime on the certificate used to authenticate a request. 128 | 129 | - **Stability Level:** ALPHA 130 | - **Type:** Histogram 131 | 132 | 133 | #### **apiserver_delegated_authz_request_duration_seconds** 134 | Request latency in seconds. Broken down by status code. 135 | 136 | - **Stability Level:** ALPHA 137 | - **Type:** Histogram 138 | - **Labels:** 139 | 140 | - code 141 | 142 | #### **apiserver_delegated_authz_request_total** 143 | Number of HTTP requests partitioned by status code. 144 | 145 | - **Stability Level:** ALPHA 146 | - **Type:** Counter 147 | - **Labels:** 148 | 149 | - code 150 | 151 | #### **apiserver_envelope_encryption_dek_cache_fill_percent** 152 | Percent of the cache slots currently occupied by cached DEKs. 153 | 154 | - **Stability Level:** ALPHA 155 | - **Type:** Gauge 156 | 157 | 158 | #### **apiserver_flowcontrol_read_vs_write_current_requests** 159 | Observations, at the end of every nanosecond, of the number of requests (as a fraction of the relevant limit) waiting or in regular stage of execution 160 | 161 | - **Stability Level:** ALPHA 162 | - **Type:** TimingRatioHistogram 163 | - **Labels:** 164 | 165 | - phase 166 | - request_kind 167 | 168 | #### **apiserver_flowcontrol_seat_fair_frac** 169 | Fair fraction of server's concurrency to allocate to each priority level that can use it 170 | 171 | - **Stability Level:** ALPHA 172 | - **Type:** Gauge 173 | 174 | 175 | #### **apiserver_request_filter_duration_seconds** 176 | Request filter latency distribution in seconds, for each filter type 177 | 178 | - **Stability Level:** ALPHA 179 | - **Type:** Histogram 180 | - **Labels:** 181 | 182 | - filter 183 | 184 | #### **apiserver_request_sli_duration_seconds** 185 | Response latency distribution (not counting webhook duration and priority & fairness queue wait times) in seconds for each verb, group, version, resource, subresource, scope and component. 186 | 187 | - **Stability Level:** ALPHA 188 | - **Type:** Histogram 189 | - **Labels:** 190 | 191 | - component 192 | - group 193 | - resource 194 | - scope 195 | - subresource 196 | - verb 197 | - version 198 | 199 | #### **apiserver_request_slo_duration_seconds** 200 | Response latency distribution (not counting webhook duration and priority & fairness queue wait times) in seconds for each verb, group, version, resource, subresource, scope and component. 201 | 202 | - **Stability Level:** ALPHA 203 | - **Type:** Histogram 204 | - **Labels:** 205 | 206 | - component 207 | - group 208 | - resource 209 | - scope 210 | - subresource 211 | - verb 212 | - version 213 | - **Deprecated Versions:** 1.27.0 214 | #### **apiserver_storage_data_key_generation_duration_seconds** 215 | Latencies in seconds of data encryption key(DEK) generation operations. 216 | 217 | - **Stability Level:** ALPHA 218 | - **Type:** Histogram 219 | 220 | 221 | #### **apiserver_storage_data_key_generation_failures_total** 222 | Total number of failed data encryption key(DEK) generation operations. 223 | 224 | - **Stability Level:** ALPHA 225 | - **Type:** Counter 226 | 227 | 228 | #### **apiserver_storage_envelope_transformation_cache_misses_total** 229 | Total number of cache misses while accessing key decryption key(KEK). 230 | 231 | - **Stability Level:** ALPHA 232 | - **Type:** Counter 233 | 234 | 235 | #### **apiserver_tls_handshake_errors_total** 236 | Number of requests dropped with 'TLS handshake error from' error 237 | 238 | - **Stability Level:** ALPHA 239 | - **Type:** Counter 240 | 241 | 242 | #### **apiserver_webhooks_x509_insecure_sha1_total** 243 | Counts the number of requests to servers with insecure SHA1 signatures in their serving certificate OR the number of connection failures due to the insecure SHA1 signatures (either/or, based on the runtime environment) 244 | 245 | - **Stability Level:** ALPHA 246 | - **Type:** Counter 247 | 248 | 249 | #### **apiserver_webhooks_x509_missing_san_total** 250 | Counts the number of requests to servers missing SAN extension in their serving certificate OR the number of connection failures due to the lack of x509 certificate SAN extension missing (either/or, based on the runtime environment) 251 | 252 | - **Stability Level:** ALPHA 253 | - **Type:** Counter 254 | 255 | 256 | #### **authenticated_user_requests** 257 | Counter of authenticated requests broken out by username. 258 | 259 | - **Stability Level:** ALPHA 260 | - **Type:** Counter 261 | - **Labels:** 262 | 263 | - username 264 | 265 | #### **authentication_attempts** 266 | Counter of authenticated attempts. 267 | 268 | - **Stability Level:** ALPHA 269 | - **Type:** Counter 270 | - **Labels:** 271 | 272 | - result 273 | 274 | #### **authentication_duration_seconds** 275 | Authentication duration in seconds broken out by result. 276 | 277 | - **Stability Level:** ALPHA 278 | - **Type:** Histogram 279 | - **Labels:** 280 | 281 | - result 282 | 283 | #### **authorization_attempts_total** 284 | Counter of authorization attempts broken down by result. It can be either 'allowed', 'denied', 'no-opinion' or 'error'. 285 | 286 | - **Stability Level:** ALPHA 287 | - **Type:** Counter 288 | - **Labels:** 289 | 290 | - result 291 | 292 | #### **authorization_duration_seconds** 293 | Authorization duration in seconds broken out by result. 294 | 295 | - **Stability Level:** ALPHA 296 | - **Type:** Histogram 297 | - **Labels:** 298 | 299 | - result 300 | 301 | #### **field_validation_request_duration_seconds** 302 | Response latency distribution in seconds for each field validation value 303 | 304 | - **Stability Level:** ALPHA 305 | - **Type:** Histogram 306 | - **Labels:** 307 | 308 | - field_validation 309 | 310 | #### **metrics_apiserver_metric_freshness_seconds** 311 | Freshness of metrics exported 312 | 313 | - **Stability Level:** ALPHA 314 | - **Type:** Histogram 315 | - **Labels:** 316 | 317 | - group 318 | 319 | #### **workqueue_adds_total** 320 | Total number of adds handled by workqueue 321 | 322 | - **Stability Level:** ALPHA 323 | - **Type:** Counter 324 | - **Labels:** 325 | 326 | - name 327 | 328 | #### **workqueue_depth** 329 | Current depth of workqueue 330 | 331 | - **Stability Level:** ALPHA 332 | - **Type:** Gauge 333 | - **Labels:** 334 | 335 | - name 336 | 337 | #### **workqueue_longest_running_processor_seconds** 338 | How many seconds has the longest running processor for workqueue been running. 339 | 340 | - **Stability Level:** ALPHA 341 | - **Type:** Gauge 342 | - **Labels:** 343 | 344 | - name 345 | 346 | #### **workqueue_queue_duration_seconds** 347 | How long in seconds an item stays in workqueue before being requested. 348 | 349 | - **Stability Level:** ALPHA 350 | - **Type:** Histogram 351 | - **Labels:** 352 | 353 | - name 354 | 355 | #### **workqueue_retries_total** 356 | Total number of retries handled by workqueue 357 | 358 | - **Stability Level:** ALPHA 359 | - **Type:** Counter 360 | - **Labels:** 361 | 362 | - name 363 | 364 | #### **workqueue_unfinished_work_seconds** 365 | How many seconds of work has done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads. One can deduce the number of stuck threads by observing the rate at which this increases. 366 | 367 | - **Stability Level:** ALPHA 368 | - **Type:** Gauge 369 | - **Labels:** 370 | 371 | - name 372 | 373 | #### **workqueue_work_duration_seconds** 374 | How long in seconds processing an item from workqueue takes. 375 | 376 | - **Stability Level:** ALPHA 377 | - **Type:** Histogram 378 | - **Labels:** 379 | 380 | - name 381 | 382 | 383 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module sigs.k8s.io/custom-metrics-apiserver 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.2 6 | 7 | require ( 8 | github.com/emicklei/go-restful/v3 v3.12.2 9 | github.com/google/addlicense v1.1.1 10 | github.com/spf13/pflag v1.0.6 11 | github.com/stretchr/testify v1.10.0 12 | k8s.io/api v0.33.1 13 | k8s.io/apimachinery v0.33.1 14 | k8s.io/apiserver v0.33.1 15 | k8s.io/client-go v0.33.1 16 | k8s.io/component-base v0.33.1 17 | k8s.io/klog/v2 v2.130.1 18 | k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff 19 | k8s.io/metrics v0.33.1 20 | k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 21 | ) 22 | 23 | require ( 24 | cel.dev/expr v0.19.1 // indirect 25 | github.com/NYTimes/gziphandler v1.1.1 // indirect 26 | github.com/antlr4-go/antlr/v4 v4.13.0 // indirect 27 | github.com/beorn7/perks v1.0.1 // indirect 28 | github.com/blang/semver/v4 v4.0.0 // indirect 29 | github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect 30 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 31 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 32 | github.com/coreos/go-semver v0.3.1 // indirect 33 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect 34 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 35 | github.com/felixge/httpsnoop v1.0.4 // indirect 36 | github.com/fsnotify/fsnotify v1.7.0 // indirect 37 | github.com/fxamacker/cbor/v2 v2.7.0 // indirect 38 | github.com/go-logr/logr v1.4.2 // indirect 39 | github.com/go-logr/stdr v1.2.2 // indirect 40 | github.com/go-openapi/jsonpointer v0.21.0 // indirect 41 | github.com/go-openapi/jsonreference v0.20.4 // indirect 42 | github.com/go-openapi/swag v0.23.0 // indirect 43 | github.com/gogo/protobuf v1.3.2 // indirect 44 | github.com/golang/protobuf v1.5.4 // indirect 45 | github.com/google/btree v1.1.3 // indirect 46 | github.com/google/cel-go v0.23.2 // indirect 47 | github.com/google/gnostic-models v0.6.9 // indirect 48 | github.com/google/go-cmp v0.7.0 // indirect 49 | github.com/google/uuid v1.6.0 // indirect 50 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 51 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect 52 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 53 | github.com/josharian/intern v1.0.0 // indirect 54 | github.com/json-iterator/go v1.1.12 // indirect 55 | github.com/kylelemons/godebug v1.1.0 // indirect 56 | github.com/mailru/easyjson v0.7.7 // indirect 57 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 58 | github.com/modern-go/reflect2 v1.0.2 // indirect 59 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 60 | github.com/pkg/errors v0.9.1 // indirect 61 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 62 | github.com/prometheus/client_golang v1.22.0 // indirect 63 | github.com/prometheus/client_model v0.6.1 // indirect 64 | github.com/prometheus/common v0.62.0 // indirect 65 | github.com/prometheus/procfs v0.15.1 // indirect 66 | github.com/spf13/cobra v1.8.1 // indirect 67 | github.com/stoewer/go-strcase v1.3.0 // indirect 68 | github.com/x448/float16 v0.8.4 // indirect 69 | go.etcd.io/etcd/api/v3 v3.5.21 // indirect 70 | go.etcd.io/etcd/client/pkg/v3 v3.5.21 // indirect 71 | go.etcd.io/etcd/client/v3 v3.5.21 // indirect 72 | go.opentelemetry.io/auto/sdk v1.1.0 // indirect 73 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect 74 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect 75 | go.opentelemetry.io/otel v1.33.0 // indirect 76 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect 77 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect 78 | go.opentelemetry.io/otel/metric v1.33.0 // indirect 79 | go.opentelemetry.io/otel/sdk v1.33.0 // indirect 80 | go.opentelemetry.io/otel/trace v1.33.0 // indirect 81 | go.opentelemetry.io/proto/otlp v1.4.0 // indirect 82 | go.uber.org/multierr v1.11.0 // indirect 83 | go.uber.org/zap v1.27.0 // indirect 84 | golang.org/x/crypto v0.36.0 // indirect 85 | golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect 86 | golang.org/x/mod v0.21.0 // indirect 87 | golang.org/x/net v0.38.0 // indirect 88 | golang.org/x/oauth2 v0.27.0 // indirect 89 | golang.org/x/sync v0.12.0 // indirect 90 | golang.org/x/sys v0.31.0 // indirect 91 | golang.org/x/term v0.30.0 // indirect 92 | golang.org/x/text v0.23.0 // indirect 93 | golang.org/x/time v0.9.0 // indirect 94 | golang.org/x/tools v0.26.0 // indirect 95 | google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect 96 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect 97 | google.golang.org/grpc v1.68.1 // indirect 98 | google.golang.org/protobuf v1.36.5 // indirect 99 | gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect 100 | gopkg.in/inf.v0 v0.9.1 // indirect 101 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 102 | gopkg.in/yaml.v3 v3.0.1 // indirect 103 | k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect 104 | k8s.io/kms v0.33.1 // indirect 105 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect 106 | sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect 107 | sigs.k8s.io/randfill v1.0.0 // indirect 108 | sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect 109 | sigs.k8s.io/yaml v1.4.0 // indirect 110 | ) 111 | -------------------------------------------------------------------------------- /hack/boilerplate.go.txt: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /hack/tools.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | //go:build tools 16 | 17 | // Package tools tracks dependencies for tools that used in the build process. 18 | package tools 19 | 20 | import ( 21 | _ "github.com/google/addlicense" 22 | _ "k8s.io/kube-openapi/cmd/openapi-gen" 23 | ) 24 | -------------------------------------------------------------------------------- /pkg/apiserver/apiserver.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apiserver 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apimachinery/pkg/runtime" 22 | "k8s.io/apimachinery/pkg/runtime/schema" 23 | "k8s.io/apimachinery/pkg/runtime/serializer" 24 | utilruntime "k8s.io/apimachinery/pkg/util/runtime" 25 | genericapiserver "k8s.io/apiserver/pkg/server" 26 | "k8s.io/apiserver/pkg/util/compatibility" 27 | "k8s.io/client-go/informers" 28 | cminstall "k8s.io/metrics/pkg/apis/custom_metrics/install" 29 | eminstall "k8s.io/metrics/pkg/apis/external_metrics/install" 30 | 31 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/installer" 32 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 33 | ) 34 | 35 | var ( 36 | Scheme = runtime.NewScheme() 37 | Codecs = serializer.NewCodecFactory(Scheme) 38 | ) 39 | 40 | func init() { 41 | cminstall.Install(Scheme) 42 | eminstall.Install(Scheme) 43 | 44 | // we need custom conversion functions to list resources with options 45 | utilruntime.Must(installer.RegisterConversions(Scheme)) 46 | 47 | // we need to add the options to empty v1 48 | // TODO fix the server code to avoid this 49 | metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) 50 | 51 | // TODO: keep the generic API server from wanting this 52 | unversioned := schema.GroupVersion{Group: "", Version: "v1"} 53 | Scheme.AddUnversionedTypes(unversioned, 54 | &metav1.Status{}, 55 | &metav1.APIVersions{}, 56 | &metav1.APIGroupList{}, 57 | &metav1.APIGroup{}, 58 | &metav1.APIResourceList{}, 59 | ) 60 | } 61 | 62 | type Config struct { 63 | GenericConfig *genericapiserver.Config 64 | } 65 | 66 | // CustomMetricsAdapterServer contains state for a Kubernetes cluster master/api server. 67 | type CustomMetricsAdapterServer struct { 68 | GenericAPIServer *genericapiserver.GenericAPIServer 69 | customMetricsProvider provider.CustomMetricsProvider 70 | externalMetricsProvider provider.ExternalMetricsProvider 71 | } 72 | 73 | type CompletedConfig struct { 74 | genericapiserver.CompletedConfig 75 | } 76 | 77 | // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. 78 | func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig { 79 | c.GenericConfig.EffectiveVersion = compatibility.DefaultBuildEffectiveVersion() 80 | return CompletedConfig{c.GenericConfig.Complete(informers)} 81 | } 82 | 83 | // New returns a new instance of CustomMetricsAdapterServer from the given config. 84 | // name is used to differentiate for logging. 85 | // Each of the arguments: customMetricsProvider, externalMetricsProvider can be set either to 86 | // a provider implementation, or to nil to disable one of the APIs. 87 | func (c CompletedConfig) New(name string, customMetricsProvider provider.CustomMetricsProvider, externalMetricsProvider provider.ExternalMetricsProvider) (*CustomMetricsAdapterServer, error) { 88 | genericServer, err := c.CompletedConfig.New(name, genericapiserver.NewEmptyDelegate()) // completion is done in Complete, no need for a second time 89 | if err != nil { 90 | return nil, err 91 | } 92 | 93 | s := &CustomMetricsAdapterServer{ 94 | GenericAPIServer: genericServer, 95 | customMetricsProvider: customMetricsProvider, 96 | externalMetricsProvider: externalMetricsProvider, 97 | } 98 | 99 | if customMetricsProvider != nil { 100 | if err := s.InstallCustomMetricsAPI(); err != nil { 101 | return nil, err 102 | } 103 | } 104 | if externalMetricsProvider != nil { 105 | if err := s.InstallExternalMetricsAPI(); err != nil { 106 | return nil, err 107 | } 108 | } 109 | 110 | return s, nil 111 | } 112 | -------------------------------------------------------------------------------- /pkg/apiserver/cmapis.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apiserver 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/api/meta" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | genericapi "k8s.io/apiserver/pkg/endpoints" 25 | "k8s.io/apiserver/pkg/endpoints/discovery" 26 | genericapiserver "k8s.io/apiserver/pkg/server" 27 | "k8s.io/metrics/pkg/apis/custom_metrics" 28 | 29 | specificapi "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/installer" 30 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 31 | metricstorage "sigs.k8s.io/custom-metrics-apiserver/pkg/registry/custom_metrics" 32 | ) 33 | 34 | func (s *CustomMetricsAdapterServer) InstallCustomMetricsAPI() error { 35 | groupInfo := genericapiserver.NewDefaultAPIGroupInfo(custom_metrics.GroupName, Scheme, runtime.NewParameterCodec(Scheme), Codecs) 36 | container := s.GenericAPIServer.Handler.GoRestfulContainer 37 | 38 | // Register custom metrics REST handler for all supported API versions. 39 | for versionIndex, mainGroupVer := range groupInfo.PrioritizedVersions { 40 | preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{ 41 | GroupVersion: mainGroupVer.String(), 42 | Version: mainGroupVer.Version, 43 | } 44 | groupVersion := metav1.GroupVersionForDiscovery{ 45 | GroupVersion: mainGroupVer.String(), 46 | Version: mainGroupVer.Version, 47 | } 48 | apiGroup := metav1.APIGroup{ 49 | Name: mainGroupVer.Group, 50 | Versions: []metav1.GroupVersionForDiscovery{groupVersion}, 51 | PreferredVersion: preferredVersionForDiscovery, 52 | } 53 | 54 | cmAPI := s.cmAPI(&groupInfo, mainGroupVer) 55 | if err := cmAPI.InstallREST(container); err != nil { 56 | return err 57 | } 58 | 59 | if versionIndex == 0 { 60 | s.GenericAPIServer.DiscoveryGroupManager.AddGroup(apiGroup) 61 | container.Add(discovery.NewAPIGroupHandler(s.GenericAPIServer.Serializer, apiGroup).WebService()) 62 | } 63 | } 64 | return nil 65 | } 66 | 67 | func (s *CustomMetricsAdapterServer) cmAPI(groupInfo *genericapiserver.APIGroupInfo, groupVersion schema.GroupVersion) *specificapi.MetricsAPIGroupVersion { 68 | resourceStorage := metricstorage.NewREST(s.customMetricsProvider) 69 | 70 | return &specificapi.MetricsAPIGroupVersion{ 71 | DynamicStorage: resourceStorage, 72 | APIGroupVersion: &genericapi.APIGroupVersion{ 73 | Root: genericapiserver.APIGroupPrefix, 74 | GroupVersion: groupVersion, 75 | MetaGroupVersion: groupInfo.MetaGroupVersion, 76 | 77 | ParameterCodec: groupInfo.ParameterCodec, 78 | Serializer: groupInfo.NegotiatedSerializer, 79 | Creater: groupInfo.Scheme, 80 | Convertor: groupInfo.Scheme, 81 | UnsafeConvertor: runtime.UnsafeObjectConvertor(groupInfo.Scheme), 82 | Typer: groupInfo.Scheme, 83 | Namer: runtime.Namer(meta.NewAccessor()), 84 | }, 85 | 86 | ResourceLister: provider.NewCustomMetricResourceLister(s.customMetricsProvider), 87 | Handlers: &specificapi.CMHandlers{}, 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /pkg/apiserver/emapis.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apiserver 18 | 19 | import ( 20 | "k8s.io/apimachinery/pkg/api/meta" 21 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 22 | "k8s.io/apimachinery/pkg/runtime" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | genericapi "k8s.io/apiserver/pkg/endpoints" 25 | "k8s.io/apiserver/pkg/endpoints/discovery" 26 | genericapiserver "k8s.io/apiserver/pkg/server" 27 | "k8s.io/metrics/pkg/apis/external_metrics" 28 | 29 | specificapi "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/installer" 30 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 31 | metricstorage "sigs.k8s.io/custom-metrics-apiserver/pkg/registry/external_metrics" 32 | ) 33 | 34 | // InstallExternalMetricsAPI registers the api server in Kube Aggregator 35 | func (s *CustomMetricsAdapterServer) InstallExternalMetricsAPI() error { 36 | groupInfo := genericapiserver.NewDefaultAPIGroupInfo(external_metrics.GroupName, Scheme, metav1.ParameterCodec, Codecs) 37 | 38 | mainGroupVer := groupInfo.PrioritizedVersions[0] 39 | preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{ 40 | GroupVersion: mainGroupVer.String(), 41 | Version: mainGroupVer.Version, 42 | } 43 | groupVersion := metav1.GroupVersionForDiscovery{ 44 | GroupVersion: mainGroupVer.String(), 45 | Version: mainGroupVer.Version, 46 | } 47 | apiGroup := metav1.APIGroup{ 48 | Name: mainGroupVer.Group, 49 | Versions: []metav1.GroupVersionForDiscovery{groupVersion}, 50 | PreferredVersion: preferredVersionForDiscovery, 51 | } 52 | 53 | emAPI := s.emAPI(&groupInfo, mainGroupVer) 54 | if err := emAPI.InstallREST(s.GenericAPIServer.Handler.GoRestfulContainer); err != nil { 55 | return err 56 | } 57 | 58 | s.GenericAPIServer.DiscoveryGroupManager.AddGroup(apiGroup) 59 | s.GenericAPIServer.Handler.GoRestfulContainer.Add(discovery.NewAPIGroupHandler(s.GenericAPIServer.Serializer, apiGroup).WebService()) 60 | 61 | return nil 62 | } 63 | 64 | func (s *CustomMetricsAdapterServer) emAPI(groupInfo *genericapiserver.APIGroupInfo, groupVersion schema.GroupVersion) *specificapi.MetricsAPIGroupVersion { 65 | resourceStorage := metricstorage.NewREST(s.externalMetricsProvider) 66 | 67 | return &specificapi.MetricsAPIGroupVersion{ 68 | DynamicStorage: resourceStorage, 69 | APIGroupVersion: &genericapi.APIGroupVersion{ 70 | Root: genericapiserver.APIGroupPrefix, 71 | GroupVersion: groupVersion, 72 | MetaGroupVersion: groupInfo.MetaGroupVersion, 73 | 74 | ParameterCodec: groupInfo.ParameterCodec, 75 | Serializer: groupInfo.NegotiatedSerializer, 76 | Creater: groupInfo.Scheme, 77 | Convertor: groupInfo.Scheme, 78 | UnsafeConvertor: runtime.UnsafeObjectConvertor(groupInfo.Scheme), 79 | Typer: groupInfo.Scheme, 80 | Namer: runtime.Namer(meta.NewAccessor()), 81 | }, 82 | ResourceLister: provider.NewExternalMetricResourceLister(s.externalMetricsProvider), 83 | Handlers: &specificapi.EMHandlers{}, 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /pkg/apiserver/endpoints/handlers/get.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package handlers 18 | 19 | import ( 20 | "net/http" 21 | "net/url" 22 | "strings" 23 | "time" 24 | 25 | "k8s.io/apimachinery/pkg/api/errors" 26 | metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" 27 | metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme" 28 | "k8s.io/apimachinery/pkg/fields" 29 | "k8s.io/apimachinery/pkg/runtime" 30 | "k8s.io/apiserver/pkg/endpoints/handlers" 31 | "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" 32 | "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" 33 | "k8s.io/apiserver/pkg/endpoints/request" 34 | utiltrace "k8s.io/utils/trace" 35 | 36 | cm_rest "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/registry/rest" 37 | ) 38 | 39 | func ListResourceWithOptions(r cm_rest.ListerWithOptions, scope handlers.RequestScope) http.HandlerFunc { 40 | return func(w http.ResponseWriter, req *http.Request) { 41 | // For performance tracking purposes. 42 | trace := utiltrace.New("List " + req.URL.Path) 43 | 44 | requestInfo, ok := request.RequestInfoFrom(req.Context()) 45 | if !ok { 46 | err := errors.NewBadRequest("missing requestInfo") 47 | writeError(&scope, err, w, req) 48 | return 49 | } 50 | 51 | // handle metrics describing namespaces 52 | if requestInfo.Namespace != "" && requestInfo.Resource == "metrics" { 53 | requestInfo.Subresource = requestInfo.Name 54 | requestInfo.Name = requestInfo.Namespace 55 | requestInfo.Resource = "namespaces" 56 | requestInfo.Namespace = "" 57 | requestInfo.Parts = append([]string{"namespaces", requestInfo.Name}, requestInfo.Parts[1:]...) 58 | } 59 | 60 | // handle invalid requests, e.g. /namespaces/name/foo 61 | if len(requestInfo.Parts) < 3 { 62 | err := errors.NewBadRequest("invalid request path") 63 | writeError(&scope, err, w, req) 64 | return 65 | } 66 | 67 | namespace, err := scope.Namer.Namespace(req) 68 | if err != nil { 69 | writeError(&scope, err, w, req) 70 | return 71 | } 72 | 73 | // Watches for single objects are routed to this function. 74 | // Treat a name parameter the same as a field selector entry. 75 | hasName := true 76 | _, name, err := scope.Namer.Name(req) 77 | if err != nil { 78 | hasName = false 79 | } 80 | 81 | ctx := req.Context() 82 | ctx = request.WithNamespace(ctx, namespace) 83 | 84 | opts := metainternalversion.ListOptions{} 85 | if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, &opts); err != nil { 86 | err = errors.NewBadRequest(err.Error()) 87 | writeError(&scope, err, w, req) 88 | return 89 | } 90 | 91 | // transform fields 92 | // TODO: DecodeParametersInto should do this. 93 | if opts.FieldSelector != nil { 94 | fn := func(label, value string) (newLabel, newValue string, err error) { 95 | return scope.Convertor.ConvertFieldLabel(scope.Kind, label, value) 96 | } 97 | if opts.FieldSelector, err = opts.FieldSelector.Transform(fn); err != nil { 98 | // TODO: allow bad request to set field causes based on query parameters 99 | err = errors.NewBadRequest(err.Error()) 100 | writeError(&scope, err, w, req) 101 | return 102 | } 103 | } 104 | 105 | if hasName { 106 | // metadata.name is the canonical internal name. 107 | // SelectionPredicate will notice that this is a request for 108 | // a single object and optimize the storage query accordingly. 109 | nameSelector := fields.OneTermEqualSelector("metadata.name", name) 110 | 111 | // Note that fieldSelector setting explicitly the "metadata.name" 112 | // will result in reaching this branch (as the value of that field 113 | // is propagated to requestInfo as the name parameter. 114 | // That said, the allowed field selectors in this branch are: 115 | // nil, fields.Everything and field selector matching metadata.name 116 | // for our name. 117 | if opts.FieldSelector != nil && !opts.FieldSelector.Empty() { 118 | selectedName, ok := opts.FieldSelector.RequiresExactMatch("metadata.name") 119 | if !ok || name != selectedName { 120 | writeError(&scope, errors.NewBadRequest("fieldSelector metadata.name doesn't match requested name"), w, req) 121 | return 122 | } 123 | } else { 124 | opts.FieldSelector = nameSelector 125 | } 126 | } 127 | 128 | // Log only long List requests (ignore Watch). 129 | defer trace.LogIfLong(500 * time.Millisecond) 130 | trace.Step("About to List from storage") 131 | extraOpts, hasSubpath, subpathKey := r.NewListOptions() 132 | if err := getRequestOptions(req, scope, extraOpts, hasSubpath, subpathKey, false); err != nil { 133 | err = errors.NewBadRequest(err.Error()) 134 | writeError(&scope, err, w, req) 135 | return 136 | } 137 | result, err := r.List(ctx, &opts, extraOpts) 138 | if err != nil { 139 | writeError(&scope, err, w, req) 140 | return 141 | } 142 | trace.Step("Listing from storage done") 143 | 144 | responsewriters.WriteObjectNegotiated(scope.Serializer, negotiation.DefaultEndpointRestrictions, scope.Kind.GroupVersion(), w, req, http.StatusOK, result, false) 145 | trace.Step("Writing http response done") 146 | } 147 | } 148 | 149 | // getRequestOptions parses out options and can include path information. The path information shouldn't include the subresource. 150 | func getRequestOptions(req *http.Request, scope handlers.RequestScope, into runtime.Object, hasSubpath bool, subpathKey string, isSubresource bool) error { 151 | if into == nil { 152 | return nil 153 | } 154 | 155 | query := req.URL.Query() 156 | if hasSubpath { 157 | newQuery := make(url.Values) 158 | for k, v := range query { 159 | newQuery[k] = v 160 | } 161 | 162 | ctx := req.Context() 163 | requestInfo, _ := request.RequestInfoFrom(ctx) 164 | startingIndex := 2 165 | if isSubresource { 166 | startingIndex = 3 167 | } 168 | 169 | p := strings.Join(requestInfo.Parts[startingIndex:], "/") 170 | 171 | // ensure non-empty subpaths correctly reflect a leading slash 172 | if len(p) > 0 && !strings.HasPrefix(p, "/") { 173 | p = "/" + p 174 | } 175 | 176 | // ensure subpaths correctly reflect the presence of a trailing slash on the original request 177 | if strings.HasSuffix(requestInfo.Path, "/") && !strings.HasSuffix(p, "/") { 178 | p += "/" 179 | } 180 | 181 | newQuery[subpathKey] = []string{p} 182 | query = newQuery 183 | } 184 | return scope.ParameterCodec.DecodeParameters(query, scope.Kind.GroupVersion(), into) 185 | } 186 | -------------------------------------------------------------------------------- /pkg/apiserver/endpoints/handlers/rest.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package handlers 18 | 19 | import ( 20 | "net/http" 21 | 22 | "k8s.io/apiserver/pkg/endpoints/handlers" 23 | "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" 24 | ) 25 | 26 | func writeError(scope *handlers.RequestScope, err error, w http.ResponseWriter, req *http.Request) { 27 | responsewriters.ErrorNegotiated(err, scope.Serializer, scope.Kind.GroupVersion(), w, req) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/apiserver/installer/cmhandlers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package installer 18 | 19 | import ( 20 | "net/http" 21 | 22 | "github.com/emicklei/go-restful/v3" 23 | 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apiserver/pkg/endpoints/handlers" 26 | "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" 27 | "k8s.io/apiserver/pkg/endpoints/metrics" 28 | 29 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/registry/rest" 30 | ) 31 | 32 | type CMHandlers struct{} 33 | 34 | // registerResourceHandlers registers the resource handlers for custom metrics. 35 | // Compared to the normal installer, this plays fast and loose a bit, but should still 36 | // follow the API conventions. 37 | func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restful.WebService) error { 38 | optionsExternalVersion := a.group.GroupVersion 39 | if a.group.OptionsExternalVersion != nil { 40 | optionsExternalVersion = *a.group.OptionsExternalVersion 41 | } 42 | 43 | fqKindToRegister, err := a.getResourceKind(a.group.DynamicStorage) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | kind := fqKindToRegister.Kind 49 | 50 | lister := a.group.DynamicStorage.(rest.ListerWithOptions) 51 | list := lister.NewList() 52 | listGVKs, _, err := a.group.Typer.ObjectKinds(list) 53 | if err != nil { 54 | return err 55 | } 56 | versionedListPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listGVKs[0].Kind)) 57 | if err != nil { 58 | return err 59 | } 60 | versionedList := indirectArbitraryPointer(versionedListPtr) 61 | 62 | versionedListOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ListOptions")) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | listOptions, _, _ := lister.NewListOptions() 68 | listOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(listOptions) 69 | if err != nil { 70 | return err 71 | } 72 | listOptionsInternalKind := listOptionsInternalKinds[0] 73 | versionedListExtraOptions, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listOptionsInternalKind.Kind)) 74 | if err != nil { 75 | versionedListExtraOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(listOptionsInternalKind.Kind)) 76 | if err != nil { 77 | return err 78 | } 79 | } 80 | 81 | nameParam := ws.PathParameter("name", "name of the described resource").DataType("string") 82 | resourceParam := ws.PathParameter("resource", "the name of the resource").DataType("string") 83 | subresourceParam := ws.PathParameter("subresource", "the name of the subresource").DataType("string") 84 | 85 | // metrics describing non-namespaced objects (e.g. nodes) 86 | rootScopedParams := []*restful.Parameter{ 87 | resourceParam, 88 | nameParam, 89 | subresourceParam, 90 | } 91 | rootScopedPath := "{resource}/{name}/{subresource}" 92 | 93 | // metrics describing namespaced objects (e.g. pods) 94 | namespaceParam := ws.PathParameter("namespace", "object name and auth scope, such as for teams and projects").DataType("string") 95 | namespacedParams := []*restful.Parameter{ 96 | namespaceParam, 97 | resourceParam, 98 | nameParam, 99 | subresourceParam, 100 | } 101 | namespacedPath := "namespaces/{namespace}/{resource}/{name}/{subresource}" 102 | 103 | namespaceSpecificPath := "namespaces/{namespace}/metrics/{name}" 104 | namespaceSpecificParams := []*restful.Parameter{ 105 | namespaceParam, 106 | nameParam, 107 | } 108 | 109 | mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer) 110 | allMediaTypes := append(mediaTypes, streamMediaTypes...) //nolint: gocritic 111 | ws.Produces(allMediaTypes...) 112 | 113 | reqScope := handlers.RequestScope{ 114 | Serializer: a.group.Serializer, 115 | ParameterCodec: a.group.ParameterCodec, 116 | Creater: a.group.Creater, 117 | Convertor: a.group.Convertor, 118 | Typer: a.group.Typer, 119 | UnsafeConvertor: a.group.UnsafeConvertor, 120 | 121 | // TODO: support TableConvertor? 122 | 123 | // TODO: This seems wrong for cross-group subresources. It makes an assumption that a subresource and its parent are in the same group version. Revisit this. 124 | Resource: a.group.GroupVersion.WithResource("*"), 125 | Subresource: "*", 126 | Kind: fqKindToRegister, 127 | 128 | MetaGroupVersion: metav1.SchemeGroupVersion, 129 | } 130 | if a.group.MetaGroupVersion != nil { 131 | reqScope.MetaGroupVersion = *a.group.MetaGroupVersion 132 | } 133 | 134 | // we need one path for namespaced resources, one for non-namespaced resources 135 | doc := "list custom metrics describing an object or objects" 136 | reqScope.Namer = MetricsNaming{ 137 | handlers.ContextBasedNaming{ 138 | Namer: a.group.Namer, 139 | ClusterScoped: true, 140 | }, 141 | } 142 | 143 | rootScopedHandler := metrics.InstrumentRouteFunc( 144 | "LIST", 145 | a.group.GroupVersion.Group, 146 | a.group.GroupVersion.Version, 147 | reqScope.Resource.Resource, 148 | reqScope.Subresource, 149 | "cluster", 150 | "custom-metrics", 151 | false, 152 | "", 153 | restfulListResourceWithOptions(lister, reqScope), 154 | ) 155 | 156 | // install the root-scoped route 157 | rootScopedRoute := ws.GET(rootScopedPath).To(rootScopedHandler). 158 | Doc(doc). 159 | Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). 160 | Operation("list"+kind). 161 | Produces(allMediaTypes...). 162 | Returns(http.StatusOK, "OK", versionedList). 163 | Writes(versionedList) 164 | if err := addObjectParams(ws, rootScopedRoute, versionedListOptions); err != nil { 165 | return err 166 | } 167 | if err := addObjectParams(ws, rootScopedRoute, versionedListExtraOptions); err != nil { 168 | return err 169 | } 170 | addParams(rootScopedRoute, rootScopedParams) 171 | ws.Route(rootScopedRoute) 172 | 173 | // install the namespace-scoped route 174 | reqScope.Namer = MetricsNaming{ 175 | handlers.ContextBasedNaming{ 176 | Namer: a.group.Namer, 177 | ClusterScoped: false, 178 | }, 179 | } 180 | namespacedHandler := metrics.InstrumentRouteFunc( 181 | "LIST", 182 | a.group.GroupVersion.Group, 183 | a.group.GroupVersion.Version, 184 | reqScope.Resource.Resource, 185 | reqScope.Subresource, 186 | "resource", 187 | "custom-metrics", 188 | false, 189 | "", 190 | restfulListResourceWithOptions(lister, reqScope), 191 | ) 192 | 193 | namespacedRoute := ws.GET(namespacedPath).To(namespacedHandler). 194 | Doc(doc). 195 | Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). 196 | Operation("listNamespaced"+kind). 197 | Produces(allMediaTypes...). 198 | Returns(http.StatusOK, "OK", versionedList). 199 | Writes(versionedList) 200 | if err := addObjectParams(ws, namespacedRoute, versionedListOptions); err != nil { 201 | return err 202 | } 203 | if err := addObjectParams(ws, namespacedRoute, versionedListExtraOptions); err != nil { 204 | return err 205 | } 206 | addParams(namespacedRoute, namespacedParams) 207 | ws.Route(namespacedRoute) 208 | 209 | // install the special route for metrics describing namespaces (last b/c we modify the context func) 210 | reqScope.Namer = MetricsNaming{ 211 | handlers.ContextBasedNaming{ 212 | Namer: a.group.Namer, 213 | ClusterScoped: false, 214 | }, 215 | } 216 | 217 | namespaceSpecificHandler := metrics.InstrumentRouteFunc( 218 | "LIST", 219 | a.group.GroupVersion.Group, 220 | a.group.GroupVersion.Version, 221 | reqScope.Resource.Resource, 222 | reqScope.Subresource, 223 | "resource", 224 | "custom-metrics", 225 | false, 226 | "", 227 | restfulListResourceWithOptions(lister, reqScope), 228 | ) 229 | 230 | namespaceSpecificRoute := ws.GET(namespaceSpecificPath).To(namespaceSpecificHandler). 231 | Doc(doc). 232 | Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). 233 | Operation("read"+kind+"ForNamespace"). 234 | Produces(allMediaTypes...). 235 | Returns(http.StatusOK, "OK", versionedList). 236 | Writes(versionedList) 237 | if err := addObjectParams(ws, namespaceSpecificRoute, versionedListOptions); err != nil { 238 | return err 239 | } 240 | if err := addObjectParams(ws, namespaceSpecificRoute, versionedListExtraOptions); err != nil { 241 | return err 242 | } 243 | addParams(namespaceSpecificRoute, namespaceSpecificParams) 244 | ws.Route(namespaceSpecificRoute) 245 | 246 | return nil 247 | } 248 | -------------------------------------------------------------------------------- /pkg/apiserver/installer/conversion.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package installer 18 | 19 | import ( 20 | "net/url" 21 | 22 | "k8s.io/apimachinery/pkg/conversion" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | cmv1beta1 "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1" 25 | cmv1beta2 "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2" 26 | ) 27 | 28 | func ConvertURLValuesToV1beta1MetricListOptions(in *url.Values, out *cmv1beta1.MetricListOptions, s conversion.Scope) error { 29 | if values, ok := map[string][]string(*in)["labelSelector"]; ok && len(values) > 0 { 30 | if err := runtime.Convert_Slice_string_To_string(&values, &out.LabelSelector, s); err != nil { 31 | return err 32 | } 33 | } else { 34 | out.LabelSelector = "" 35 | } 36 | if values, ok := map[string][]string(*in)["metricLabelSelector"]; ok && len(values) > 0 { 37 | if err := runtime.Convert_Slice_string_To_string(&values, &out.MetricLabelSelector, s); err != nil { 38 | return err 39 | } 40 | } else { 41 | out.MetricLabelSelector = "" 42 | } 43 | return nil 44 | } 45 | 46 | func ConvertURLValuesToV1beta2MetricListOptions(in *url.Values, out *cmv1beta2.MetricListOptions, s conversion.Scope) error { 47 | if values, ok := map[string][]string(*in)["labelSelector"]; ok && len(values) > 0 { 48 | if err := runtime.Convert_Slice_string_To_string(&values, &out.LabelSelector, s); err != nil { 49 | return err 50 | } 51 | } else { 52 | out.LabelSelector = "" 53 | } 54 | if values, ok := map[string][]string(*in)["metricLabelSelector"]; ok && len(values) > 0 { 55 | if err := runtime.Convert_Slice_string_To_string(&values, &out.MetricLabelSelector, s); err != nil { 56 | return err 57 | } 58 | } else { 59 | out.MetricLabelSelector = "" 60 | } 61 | return nil 62 | } 63 | 64 | // RegisterConversions adds conversion functions to the given scheme. 65 | func RegisterConversions(s *runtime.Scheme) error { 66 | if err := s.AddConversionFunc((*url.Values)(nil), (*cmv1beta1.MetricListOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { 67 | return ConvertURLValuesToV1beta1MetricListOptions(a.(*url.Values), b.(*cmv1beta1.MetricListOptions), scope) 68 | }); err != nil { 69 | return err 70 | } 71 | return s.AddConversionFunc((*url.Values)(nil), (*cmv1beta2.MetricListOptions)(nil), func(a, b interface{}, scope conversion.Scope) error { 72 | return ConvertURLValuesToV1beta2MetricListOptions(a.(*url.Values), b.(*cmv1beta2.MetricListOptions), scope) 73 | }) 74 | } 75 | -------------------------------------------------------------------------------- /pkg/apiserver/installer/emhandlers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package installer 18 | 19 | import ( 20 | "net/http" 21 | 22 | "github.com/emicklei/go-restful/v3" 23 | 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apiserver/pkg/endpoints/handlers" 26 | "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" 27 | "k8s.io/apiserver/pkg/endpoints/metrics" 28 | "k8s.io/apiserver/pkg/registry/rest" 29 | ) 30 | 31 | type EMHandlers struct{} 32 | 33 | // registerResourceHandlers registers the resource handlers for external metrics. 34 | // The implementation is based on corresponding registerResourceHandlers for Custom Metrics API 35 | func (ch *EMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restful.WebService) error { 36 | optionsExternalVersion := a.group.GroupVersion 37 | if a.group.OptionsExternalVersion != nil { 38 | optionsExternalVersion = *a.group.OptionsExternalVersion 39 | } 40 | 41 | fqKindToRegister, err := a.getResourceKind(a.group.DynamicStorage) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | kind := fqKindToRegister.Kind 47 | 48 | lister := a.group.DynamicStorage.(rest.Lister) 49 | list := lister.NewList() 50 | listGVKs, _, err := a.group.Typer.ObjectKinds(list) 51 | if err != nil { 52 | return err 53 | } 54 | versionedListPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listGVKs[0].Kind)) 55 | if err != nil { 56 | return err 57 | } 58 | versionedList := indirectArbitraryPointer(versionedListPtr) 59 | 60 | versionedListOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ListOptions")) 61 | if err != nil { 62 | return err 63 | } 64 | 65 | namespaceParam := ws.PathParameter("namespace", "object name and auth scope, such as for teams and projects").DataType("string") 66 | nameParam := ws.PathParameter("name", "name of the described resource").DataType("string") 67 | 68 | externalMetricParams := []*restful.Parameter{ 69 | namespaceParam, 70 | nameParam, 71 | } 72 | externalMetricPath := "namespaces" + "/{namespace}/{resource}" 73 | 74 | mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer) 75 | allMediaTypes := append(mediaTypes, streamMediaTypes...) //nolint: gocritic 76 | ws.Produces(allMediaTypes...) 77 | 78 | reqScope := handlers.RequestScope{ 79 | Serializer: a.group.Serializer, 80 | ParameterCodec: a.group.ParameterCodec, 81 | Creater: a.group.Creater, 82 | Convertor: a.group.Convertor, 83 | Typer: a.group.Typer, 84 | UnsafeConvertor: a.group.UnsafeConvertor, 85 | 86 | // TODO: support TableConvertor? 87 | 88 | // TODO: This seems wrong for cross-group subresources. It makes an assumption that a subresource and its parent are in the same group version. Revisit this. 89 | Resource: a.group.GroupVersion.WithResource("*"), 90 | Subresource: "*", 91 | Kind: fqKindToRegister, 92 | 93 | MetaGroupVersion: metav1.SchemeGroupVersion, 94 | } 95 | if a.group.MetaGroupVersion != nil { 96 | reqScope.MetaGroupVersion = *a.group.MetaGroupVersion 97 | } 98 | 99 | doc := "list external metrics" 100 | reqScope.Namer = MetricsNaming{ 101 | handlers.ContextBasedNaming{ 102 | Namer: a.group.Namer, 103 | ClusterScoped: false, 104 | }, 105 | } 106 | 107 | externalMetricHandler := metrics.InstrumentRouteFunc( 108 | "LIST", 109 | a.group.GroupVersion.Group, 110 | a.group.GroupVersion.Version, 111 | reqScope.Resource.Resource, 112 | reqScope.Subresource, 113 | "cluster", 114 | "external-metrics", 115 | false, 116 | "", 117 | restfulListResource(lister, nil, reqScope, false, a.minRequestTimeout), 118 | ) 119 | 120 | externalMetricRoute := ws.GET(externalMetricPath).To(externalMetricHandler). 121 | Doc(doc). 122 | Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). 123 | Operation("list"+kind). 124 | Produces(allMediaTypes...). 125 | Returns(http.StatusOK, "OK", versionedList). 126 | Writes(versionedList) 127 | if err := addObjectParams(ws, externalMetricRoute, versionedListOptions); err != nil { 128 | return err 129 | } 130 | addParams(externalMetricRoute, externalMetricParams) 131 | ws.Route(externalMetricRoute) 132 | 133 | return nil 134 | } 135 | -------------------------------------------------------------------------------- /pkg/apiserver/installer/installer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package installer 18 | 19 | import ( 20 | "fmt" 21 | gpath "path" 22 | "reflect" 23 | "strings" 24 | "time" 25 | 26 | "github.com/emicklei/go-restful/v3" 27 | 28 | "k8s.io/apimachinery/pkg/conversion" 29 | "k8s.io/apimachinery/pkg/runtime/schema" 30 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 31 | "k8s.io/apiserver/pkg/endpoints" 32 | "k8s.io/apiserver/pkg/endpoints/discovery" 33 | "k8s.io/apiserver/pkg/endpoints/handlers" 34 | "k8s.io/apiserver/pkg/endpoints/handlers/negotiation" 35 | "k8s.io/apiserver/pkg/registry/rest" 36 | 37 | cm_handlers "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/endpoints/handlers" 38 | cm_rest "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/registry/rest" 39 | ) 40 | 41 | // NB: the contents of this file should mostly be a subset of the functionality 42 | // in "k8s.io/apiserver/pkg/endpoints". It would be nice to eventual figure out 43 | // a way to not have to recreate/copy a bunch of the structure from the normal API 44 | // installer, so that this trivially tracks changes to the main installer. 45 | 46 | // MetricsAPIGroupVersion is similar to "k8s.io/apiserver/pkg/endpoints".APIGroupVersion, 47 | // except that it installs the metrics REST handlers, which use wildcard resources 48 | // and subresources. 49 | // 50 | // This basically only serves the limitted use case required by the metrics API server -- 51 | // the only verb accepted is GET (and perhaps WATCH in the future). 52 | type MetricsAPIGroupVersion struct { 53 | DynamicStorage rest.Storage 54 | 55 | *endpoints.APIGroupVersion 56 | 57 | ResourceLister discovery.APIResourceLister 58 | 59 | Handlers apiHandlers 60 | } 61 | 62 | type apiHandlers interface { 63 | registerResourceHandlers(a *MetricsAPIInstaller, ws *restful.WebService) error 64 | } 65 | 66 | // InstallDynamicREST registers the dynamic REST handlers into a restful Container. 67 | // It is expected that the provided path root prefix will serve all operations. Root MUST 68 | // NOT end in a slash. It should mirror InstallREST in the plain APIGroupVersion. 69 | func (g *MetricsAPIGroupVersion) InstallREST(container *restful.Container) error { 70 | installer := g.newDynamicInstaller() 71 | ws := installer.NewWebService() 72 | 73 | registrationErrors := installer.Install(ws) 74 | lister := g.ResourceLister 75 | if lister == nil { 76 | return fmt.Errorf("must provide a dynamic lister for dynamic API groups") 77 | } 78 | versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, lister) 79 | versionDiscoveryHandler.AddToWebService(ws) 80 | container.Add(ws) 81 | return utilerrors.NewAggregate(registrationErrors) 82 | } 83 | 84 | // newDynamicInstaller is a helper to create the installer. It mirrors 85 | // newInstaller in APIGroupVersion. 86 | func (g *MetricsAPIGroupVersion) newDynamicInstaller() *MetricsAPIInstaller { 87 | prefix := gpath.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version) 88 | installer := &MetricsAPIInstaller{ 89 | group: g, 90 | prefix: prefix, 91 | minRequestTimeout: g.MinRequestTimeout, 92 | handlers: g.Handlers, 93 | } 94 | 95 | return installer 96 | } 97 | 98 | // MetricsAPIInstaller is a specialized API installer for the metrics API. 99 | // It is intended to be fully compliant with the Kubernetes API server conventions, 100 | // but serves wildcard resource/subresource routes instead of hard-coded resources 101 | // and subresources. 102 | type MetricsAPIInstaller struct { 103 | group *MetricsAPIGroupVersion 104 | prefix string // Path prefix where API resources are to be registered. 105 | minRequestTimeout time.Duration 106 | handlers apiHandlers 107 | 108 | // TODO: do we want to embed a normal API installer here so we can serve normal 109 | // endpoints side by side with dynamic ones (from the same API group)? 110 | } 111 | 112 | // Install installs handlers for External Metrics API resources. 113 | func (a *MetricsAPIInstaller) Install(ws *restful.WebService) (errors []error) { 114 | errors = make([]error, 0) 115 | 116 | err := a.handlers.registerResourceHandlers(a, ws) 117 | if err != nil { 118 | errors = append(errors, fmt.Errorf("error in registering custom metrics resource: %v", err)) 119 | } 120 | 121 | return errors 122 | } 123 | 124 | // NewWebService creates a new restful webservice with the api installer's prefix and version. 125 | func (a *MetricsAPIInstaller) NewWebService() *restful.WebService { 126 | ws := new(restful.WebService) 127 | ws.Path(a.prefix) 128 | // a.prefix contains "prefix/group/version" 129 | ws.Doc("API at " + a.prefix) 130 | // Backwards compatibility, we accepted objects with empty content-type at V1. 131 | // If we stop using go-restful, we can default empty content-type to application/json on an 132 | // endpoint by endpoint basis 133 | ws.Consumes("*/*") 134 | mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer) 135 | ws.Produces(append(mediaTypes, streamMediaTypes...)...) 136 | ws.ApiVersion(a.group.GroupVersion.String()) 137 | 138 | return ws 139 | } 140 | 141 | // This magic incantation returns *ptrToObject for an arbitrary pointer 142 | func indirectArbitraryPointer(ptrToObject interface{}) interface{} { 143 | return reflect.Indirect(reflect.ValueOf(ptrToObject)).Interface() 144 | } 145 | 146 | // getResourceKind returns the external group version kind registered for the given storage object. 147 | func (a *MetricsAPIInstaller) getResourceKind(storage rest.Storage) (schema.GroupVersionKind, error) { 148 | object := storage.New() 149 | fqKinds, _, err := a.group.Typer.ObjectKinds(object) 150 | if err != nil { 151 | return schema.GroupVersionKind{}, err 152 | } 153 | 154 | // a given go type can have multiple potential fully qualified kinds. Find the one that corresponds with the group 155 | // we're trying to register here 156 | fqKindToRegister := schema.GroupVersionKind{} 157 | for _, fqKind := range fqKinds { 158 | if fqKind.Group == a.group.GroupVersion.Group { 159 | fqKindToRegister = a.group.GroupVersion.WithKind(fqKind.Kind) 160 | break 161 | } 162 | 163 | // TODO: keep rid of extensions api group dependency here 164 | // This keeps it doing what it was doing before, but it doesn't feel right. 165 | if fqKind.Group == "extensions" && fqKind.Kind == "ThirdPartyResourceData" { 166 | fqKindToRegister = a.group.GroupVersion.WithKind(fqKind.Kind) 167 | } 168 | } 169 | if fqKindToRegister.Empty() { 170 | return schema.GroupVersionKind{}, fmt.Errorf("unable to locate fully qualified kind for %v: found %v when registering for %v", reflect.TypeOf(object), fqKinds, a.group.GroupVersion) 171 | } 172 | return fqKindToRegister, nil 173 | } 174 | 175 | func addParams(route *restful.RouteBuilder, params []*restful.Parameter) { 176 | for _, param := range params { 177 | route.Param(param) 178 | } 179 | } 180 | 181 | // addObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route. 182 | // The object must be a pointer to a struct; only fields at the top level of the struct that are not 183 | // themselves interfaces or structs are used; only fields with a json tag that is non empty (the standard 184 | // Go JSON behavior for omitting a field) become query parameters. The name of the query parameter is 185 | // the JSON field name. If a description struct tag is set on the field, that description is used on the 186 | // query parameter. In essence, it converts a standard JSON top level object into a query param schema. 187 | func addObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error { 188 | sv, err := conversion.EnforcePtr(obj) 189 | if err != nil { 190 | return err 191 | } 192 | st := sv.Type() 193 | if st.Kind() == reflect.Struct { 194 | for i := 0; i < st.NumField(); i++ { 195 | name := st.Field(i).Name 196 | sf, ok := st.FieldByName(name) 197 | if !ok { 198 | continue 199 | } 200 | switch sf.Type.Kind() { 201 | case reflect.Interface, reflect.Struct: 202 | case reflect.Ptr: 203 | // TODO: This is a hack to let metav1.Time through. This needs to be fixed in a more generic way eventually. bug #36191 204 | if (sf.Type.Elem().Kind() == reflect.Interface || sf.Type.Elem().Kind() == reflect.Struct) && strings.TrimPrefix(sf.Type.String(), "*") != "metav1.Time" { 205 | continue 206 | } 207 | fallthrough 208 | default: 209 | jsonTag := sf.Tag.Get("json") 210 | if len(jsonTag) == 0 { 211 | continue 212 | } 213 | jsonName := strings.SplitN(jsonTag, ",", 2)[0] 214 | if len(jsonName) == 0 { 215 | continue 216 | } 217 | 218 | var desc string 219 | if docable, ok := obj.(documentable); ok { 220 | desc = docable.SwaggerDoc()[jsonName] 221 | } 222 | 223 | if route.ParameterNamed(jsonName) == nil { 224 | route.Param(ws.QueryParameter(jsonName, desc).DataType(typeToJSON(sf.Type.String()))) 225 | } 226 | } 227 | } 228 | } 229 | return nil 230 | } 231 | 232 | // TODO: this is incomplete, expand as needed. 233 | // Convert the name of a golang type to the name of a JSON type 234 | func typeToJSON(typeName string) string { 235 | switch typeName { 236 | case "bool", "*bool": 237 | return "boolean" 238 | case "uint8", "*uint8", "int", "*int", "int32", "*int32", "int64", "*int64", "uint32", "*uint32", "uint64", "*uint64": 239 | return "integer" 240 | case "float64", "*float64", "float32", "*float32": 241 | return "number" 242 | case "metav1.Time", "*metav1.Time": 243 | return "string" 244 | case "byte", "*byte": 245 | return "string" 246 | case "v1.DeletionPropagation", "*v1.DeletionPropagation", "v1.ResourceVersionMatch": 247 | return "string" 248 | 249 | // TODO: Fix these when go-restful supports a way to specify an array query param: 250 | // https://github.com/emicklei/go-restful/issues/225 251 | case "[]string", "[]*string": 252 | return "string" 253 | case "[]int32", "[]*int32": 254 | return "integer" 255 | 256 | default: 257 | return typeName 258 | } 259 | } 260 | 261 | // An interface to see if an object supports swagger documentation as a method 262 | type documentable interface { 263 | SwaggerDoc() map[string]string 264 | } 265 | 266 | // MetricsNaming is similar to handlers.ContextBasedNaming, except that it handles 267 | // polymorphism over subresources. 268 | type MetricsNaming struct { 269 | handlers.ContextBasedNaming 270 | } 271 | 272 | func restfulListResource(r rest.Lister, rw rest.Watcher, scope handlers.RequestScope, forceWatch bool, minRequestTimeout time.Duration) restful.RouteFunction { 273 | return func(req *restful.Request, res *restful.Response) { 274 | handlers.ListResource(r, rw, &scope, forceWatch, minRequestTimeout)(res.ResponseWriter, req.Request) 275 | } 276 | } 277 | 278 | func restfulListResourceWithOptions(r cm_rest.ListerWithOptions, scope handlers.RequestScope) restful.RouteFunction { 279 | return func(req *restful.Request, res *restful.Response) { 280 | cm_handlers.ListResourceWithOptions(r, scope)(res.ResponseWriter, req.Request) 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /pkg/apiserver/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package metrics provides metrics and instrumentation functions for the 18 | // metrics API server. 19 | package metrics 20 | 21 | import ( 22 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 23 | "k8s.io/component-base/metrics" 24 | "k8s.io/utils/clock" 25 | ) 26 | 27 | var ( 28 | metricFreshness = metrics.NewHistogramVec(&metrics.HistogramOpts{ 29 | Namespace: "metrics_apiserver", 30 | Name: "metric_freshness_seconds", 31 | Help: "Freshness of metrics exported", 32 | StabilityLevel: metrics.ALPHA, 33 | Buckets: metrics.ExponentialBuckets(1, 1.364, 20), 34 | }, []string{"group"}) 35 | ) 36 | 37 | // RegisterMetrics registers API server metrics, given a registration function. 38 | func RegisterMetrics(registrationFunc func(metrics.Registerable) error) error { 39 | return registrationFunc(metricFreshness) 40 | } 41 | 42 | // FreshnessObserver captures individual observations of the timestamp of 43 | // metrics. 44 | type FreshnessObserver interface { 45 | Observe(timestamp metav1.Time) 46 | } 47 | 48 | // NewFreshnessObserver creates a FreshnessObserver for a given metrics API group. 49 | func NewFreshnessObserver(apiGroup string) FreshnessObserver { 50 | return &freshnessObserver{ 51 | apiGroup: apiGroup, 52 | clock: clock.RealClock{}, 53 | } 54 | } 55 | 56 | type freshnessObserver struct { 57 | apiGroup string 58 | clock clock.PassiveClock 59 | } 60 | 61 | func (o *freshnessObserver) Observe(timestamp metav1.Time) { 62 | metricFreshness.WithLabelValues(o.apiGroup). 63 | Observe(o.clock.Since(timestamp.Time).Seconds()) 64 | } 65 | -------------------------------------------------------------------------------- /pkg/apiserver/metrics/metrics_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "strings" 21 | "testing" 22 | "time" 23 | 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/component-base/metrics/testutil" 26 | "k8s.io/metrics/pkg/apis/custom_metrics" 27 | "k8s.io/metrics/pkg/apis/external_metrics" 28 | clocktesting "k8s.io/utils/clock/testing" 29 | ) 30 | 31 | func TestFreshness(t *testing.T) { 32 | now := time.Now() 33 | 34 | metricFreshness.Create(nil) 35 | metricFreshness.Reset() 36 | 37 | externalMetricsList := external_metrics.ExternalMetricValueList{ 38 | Items: []external_metrics.ExternalMetricValue{ 39 | {Timestamp: metav1.NewTime(now.Add(-10 * time.Second))}, 40 | {Timestamp: metav1.NewTime(now.Add(-10 * time.Second))}, 41 | {Timestamp: metav1.NewTime(now.Add(-2 * time.Second))}, 42 | }, 43 | } 44 | externalObserver := NewFreshnessObserver("external.metrics.k8s.io") 45 | externalObserver.(*freshnessObserver).clock = clocktesting.NewFakeClock(now) 46 | for _, m := range externalMetricsList.Items { 47 | externalObserver.Observe(m.Timestamp) 48 | } 49 | 50 | customMetricsList := custom_metrics.MetricValueList{ 51 | Items: []custom_metrics.MetricValue{ 52 | {Timestamp: metav1.NewTime(now.Add(-5 * time.Second))}, 53 | {Timestamp: metav1.NewTime(now.Add(-10 * time.Second))}, 54 | {Timestamp: metav1.NewTime(now.Add(-25 * time.Second))}, 55 | }, 56 | } 57 | customObserver := NewFreshnessObserver("custom.metrics.k8s.io") 58 | customObserver.(*freshnessObserver).clock = clocktesting.NewFakeClock(now) 59 | for _, m := range customMetricsList.Items { 60 | customObserver.Observe(m.Timestamp) 61 | } 62 | 63 | err := testutil.CollectAndCompare(metricFreshness, strings.NewReader(` 64 | # HELP metrics_apiserver_metric_freshness_seconds [ALPHA] Freshness of metrics exported 65 | # TYPE metrics_apiserver_metric_freshness_seconds histogram 66 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="1"} 0 67 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="1.364"} 0 68 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="1.8604960000000004"} 0 69 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="2.5377165440000007"} 0 70 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="3.4614453660160014"} 0 71 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="4.721411479245826"} 0 72 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="6.440005257691307"} 1 73 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="8.784167171490942"} 1 74 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="11.981604021913647"} 2 75 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="16.342907885890217"} 2 76 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="22.291726356354257"} 2 77 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="30.405914750067208"} 3 78 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="41.47366771909167"} 3 79 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="56.57008276884105"} 3 80 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="77.16159289669919"} 3 81 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="105.2484127110977"} 3 82 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="143.55883493793726"} 3 83 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="195.81425085534644"} 3 84 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="267.09063816669254"} 3 85 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="364.31163045936864"} 3 86 | metrics_apiserver_metric_freshness_seconds_bucket{group="custom.metrics.k8s.io",le="+Inf"} 3 87 | metrics_apiserver_metric_freshness_seconds_sum{group="custom.metrics.k8s.io"} 40 88 | metrics_apiserver_metric_freshness_seconds_count{group="custom.metrics.k8s.io"} 3 89 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="1"} 0 90 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="1.364"} 0 91 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="1.8604960000000004"} 0 92 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="2.5377165440000007"} 1 93 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="3.4614453660160014"} 1 94 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="4.721411479245826"} 1 95 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="6.440005257691307"} 1 96 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="8.784167171490942"} 1 97 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="11.981604021913647"} 3 98 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="16.342907885890217"} 3 99 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="22.291726356354257"} 3 100 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="30.405914750067208"} 3 101 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="41.47366771909167"} 3 102 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="56.57008276884105"} 3 103 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="77.16159289669919"} 3 104 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="105.2484127110977"} 3 105 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="143.55883493793726"} 3 106 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="195.81425085534644"} 3 107 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="267.09063816669254"} 3 108 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="364.31163045936864"} 3 109 | metrics_apiserver_metric_freshness_seconds_bucket{group="external.metrics.k8s.io",le="+Inf"} 3 110 | metrics_apiserver_metric_freshness_seconds_sum{group="external.metrics.k8s.io"} 22 111 | metrics_apiserver_metric_freshness_seconds_count{group="external.metrics.k8s.io"} 3 112 | `)) 113 | if err != nil { 114 | t.Errorf("unexpected error: %v", err) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /pkg/apiserver/registry/rest/rest.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package rest 18 | 19 | import ( 20 | "context" 21 | 22 | metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" 23 | "k8s.io/apimachinery/pkg/runtime" 24 | ) 25 | 26 | // ListerWithOptions is an object that can retrieve resources that match the provided field 27 | // and label criteria and takes additional options on the list request. 28 | type ListerWithOptions interface { 29 | // NewList returns an empty object that can be used with the List call. 30 | // This object must be a pointer type for use with Codec.DecodeInto([]byte, runtime.Object) 31 | NewList() runtime.Object 32 | 33 | // List selects resources in the storage which match to the selector. 'options' can be nil. 34 | // The extraOptions object passed to it is of the same type returned by the NewListOptions 35 | // method. 36 | List(ctx context.Context, options *metainternalversion.ListOptions, extraOptions runtime.Object) (runtime.Object, error) 37 | 38 | // NewListOptions returns an empty options object that will be used to pass extra options 39 | // to the List method. It may return a bool and a string, if true, the 40 | // value of the request path below the list will be included as the named 41 | // string in the serialization of the runtime object. E.g., returning "path" 42 | // will convert the trailing request scheme value to "path" in the map[string][]string 43 | // passed to the converter. 44 | NewListOptions() (runtime.Object, bool, string) 45 | } 46 | -------------------------------------------------------------------------------- /pkg/cmd/builder.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "sync" 23 | "time" 24 | 25 | "github.com/spf13/pflag" 26 | 27 | apimeta "k8s.io/apimachinery/pkg/api/meta" 28 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 29 | openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" 30 | genericapiserver "k8s.io/apiserver/pkg/server" 31 | "k8s.io/client-go/discovery" 32 | "k8s.io/client-go/dynamic" 33 | "k8s.io/client-go/informers" 34 | "k8s.io/client-go/kubernetes" 35 | "k8s.io/client-go/rest" 36 | "k8s.io/client-go/tools/clientcmd" 37 | openapicommon "k8s.io/kube-openapi/pkg/common" 38 | 39 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver" 40 | "sigs.k8s.io/custom-metrics-apiserver/pkg/cmd/options" 41 | "sigs.k8s.io/custom-metrics-apiserver/pkg/dynamicmapper" 42 | generatedcore "sigs.k8s.io/custom-metrics-apiserver/pkg/generated/openapi/core" 43 | generatedcustommetrics "sigs.k8s.io/custom-metrics-apiserver/pkg/generated/openapi/custommetrics" 44 | generatedexternalmetrics "sigs.k8s.io/custom-metrics-apiserver/pkg/generated/openapi/externalmetrics" 45 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 46 | ) 47 | 48 | // AdapterBase provides a base set of functionality for any custom metrics adapter. 49 | // Embed it in a struct containing your options, then: 50 | // 51 | // - Use Flags() to add flags, then call Flags().Parse(os.Argv) 52 | // - Use DynamicClient and RESTMapper to fetch handles to common utilities 53 | // - Use WithCustomMetrics(provider) and WithExternalMetrics(provider) to install metrics providers 54 | // - Use Run(stopChannel) to start the server 55 | // 56 | // All methods on this struct are idempotent except for Run -- they'll perform any 57 | // initialization on the first call, then return the existing object on later calls. 58 | // Methods on this struct are not safe to call from multiple goroutines without 59 | // external synchronization. 60 | type AdapterBase struct { 61 | *options.CustomMetricsAdapterServerOptions 62 | 63 | // Name is the name of the API server. It defaults to custom-metrics-adapter 64 | Name string 65 | 66 | // RemoteKubeConfigFile specifies the kubeconfig to use to construct 67 | // the dynamic client and RESTMapper. It's set from a flag. 68 | RemoteKubeConfigFile string 69 | // DiscoveryInterval specifies the interval at which to recheck discovery 70 | // information for the discovery RESTMapper. It's set from a flag. 71 | DiscoveryInterval time.Duration 72 | // ClientQPS specifies the maximum QPS for the client-side throttle. It's set from a flag. 73 | ClientQPS float32 74 | // ClientBurst specifies the maximum QPS burst for client-side throttle. It's set from a flag. 75 | ClientBurst int 76 | 77 | // FlagSet is the flagset to add flags to. 78 | // It defaults to the normal CommandLine flags 79 | // if not explicitly set. 80 | FlagSet *pflag.FlagSet 81 | 82 | // OpenAPIConfig 83 | OpenAPIConfig *openapicommon.Config 84 | 85 | // OpenAPIV3Config 86 | OpenAPIV3Config *openapicommon.OpenAPIV3Config 87 | 88 | // flagOnce controls initialization of the flags. 89 | flagOnce sync.Once 90 | 91 | clientConfig *rest.Config 92 | discoveryClient discovery.DiscoveryInterface 93 | restMapper apimeta.RESTMapper 94 | dynamicClient dynamic.Interface 95 | informers informers.SharedInformerFactory 96 | 97 | config *apiserver.Config 98 | server *apiserver.CustomMetricsAdapterServer 99 | 100 | cmProvider provider.CustomMetricsProvider 101 | emProvider provider.ExternalMetricsProvider 102 | } 103 | 104 | // InstallFlags installs the minimum required set of flags into the flagset. 105 | func (b *AdapterBase) InstallFlags() { 106 | b.initFlagSet() 107 | b.flagOnce.Do(func() { 108 | if b.CustomMetricsAdapterServerOptions == nil { 109 | b.CustomMetricsAdapterServerOptions = options.NewCustomMetricsAdapterServerOptions() 110 | } 111 | 112 | b.CustomMetricsAdapterServerOptions.AddFlags(b.FlagSet) 113 | 114 | b.FlagSet.StringVar(&b.RemoteKubeConfigFile, "lister-kubeconfig", b.RemoteKubeConfigFile, 115 | "kubeconfig file pointing at the 'core' kubernetes server with enough rights to list "+ 116 | "any described objects") 117 | b.FlagSet.DurationVar(&b.DiscoveryInterval, "discovery-interval", b.DiscoveryInterval, 118 | "Interval at which to refresh API discovery information") 119 | b.FlagSet.Float32Var(&b.ClientQPS, "client-qps", rest.DefaultQPS, "Maximum QPS for client-side throttle") 120 | b.FlagSet.IntVar(&b.ClientBurst, "client-burst", rest.DefaultBurst, "Maximum QPS burst for client-side throttle") 121 | }) 122 | } 123 | 124 | // initFlagSet populates the flagset to the CommandLine flags if it's not already set. 125 | func (b *AdapterBase) initFlagSet() { 126 | if b.FlagSet == nil { 127 | // default to the normal commandline flags 128 | b.FlagSet = pflag.CommandLine 129 | } 130 | } 131 | 132 | // Flags returns the flagset used by this adapter. 133 | // It will initialize the flagset with the minimum required set 134 | // of flags as well. 135 | func (b *AdapterBase) Flags() *pflag.FlagSet { 136 | b.initFlagSet() 137 | b.InstallFlags() 138 | 139 | return b.FlagSet 140 | } 141 | 142 | // ClientConfig returns the REST client configuration used to construct 143 | // clients for the clients and RESTMapper, and may be used for other 144 | // purposes as well. If you need to mutate it, be sure to copy it with 145 | // rest.CopyConfig first. 146 | func (b *AdapterBase) ClientConfig() (*rest.Config, error) { 147 | if b.clientConfig == nil { 148 | var clientConfig *rest.Config 149 | var err error 150 | if len(b.RemoteKubeConfigFile) > 0 { 151 | loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: b.RemoteKubeConfigFile} 152 | loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{}) 153 | 154 | clientConfig, err = loader.ClientConfig() 155 | } else { 156 | clientConfig, err = rest.InClusterConfig() 157 | } 158 | if err != nil { 159 | return nil, fmt.Errorf("unable to construct lister client config to initialize provider: %v", err) 160 | } 161 | b.clientConfig = clientConfig 162 | } 163 | 164 | if b.ClientQPS > 0 { 165 | b.clientConfig.QPS = b.ClientQPS 166 | } 167 | if b.ClientBurst > 0 { 168 | b.clientConfig.Burst = b.ClientBurst 169 | } 170 | return b.clientConfig, nil 171 | } 172 | 173 | // DiscoveryClient returns a DiscoveryInterface suitable to for discovering resources 174 | // available on the cluster. 175 | func (b *AdapterBase) DiscoveryClient() (discovery.DiscoveryInterface, error) { 176 | if b.discoveryClient == nil { 177 | clientConfig, err := b.ClientConfig() 178 | if err != nil { 179 | return nil, err 180 | } 181 | discoveryClient, err := discovery.NewDiscoveryClientForConfig(clientConfig) 182 | if err != nil { 183 | return nil, fmt.Errorf("unable to construct discovery client for dynamic client: %v", err) 184 | } 185 | b.discoveryClient = discoveryClient 186 | } 187 | return b.discoveryClient, nil 188 | } 189 | 190 | // RESTMapper returns a RESTMapper dynamically populated with discovery information. 191 | // The discovery information will be periodically repopulated according to DiscoveryInterval. 192 | func (b *AdapterBase) RESTMapper() (apimeta.RESTMapper, error) { 193 | if b.restMapper == nil { 194 | discoveryClient, err := b.DiscoveryClient() 195 | if err != nil { 196 | return nil, err 197 | } 198 | // NB: since we never actually look at the contents of 199 | // the objects we fetch (beyond ObjectMeta), unstructured should be fine 200 | dynamicMapper, err := dynamicmapper.NewRESTMapper(discoveryClient, b.DiscoveryInterval) 201 | if err != nil { 202 | return nil, fmt.Errorf("unable to construct dynamic discovery mapper: %v", err) 203 | } 204 | 205 | b.restMapper = dynamicMapper 206 | } 207 | return b.restMapper, nil 208 | } 209 | 210 | // DynamicClient returns a dynamic Kubernetes client capable of listing and fetching 211 | // any resources on the cluster. 212 | func (b *AdapterBase) DynamicClient() (dynamic.Interface, error) { 213 | if b.dynamicClient == nil { 214 | clientConfig, err := b.ClientConfig() 215 | if err != nil { 216 | return nil, err 217 | } 218 | dynClient, err := dynamic.NewForConfig(clientConfig) 219 | if err != nil { 220 | return nil, fmt.Errorf("unable to construct lister client to initialize provider: %v", err) 221 | } 222 | b.dynamicClient = dynClient 223 | } 224 | return b.dynamicClient, nil 225 | } 226 | 227 | // WithCustomMetrics populates the custom metrics provider for this adapter. 228 | func (b *AdapterBase) WithCustomMetrics(p provider.CustomMetricsProvider) { 229 | b.cmProvider = p 230 | } 231 | 232 | // WithExternalMetrics populates the external metrics provider for this adapter. 233 | func (b *AdapterBase) WithExternalMetrics(p provider.ExternalMetricsProvider) { 234 | b.emProvider = p 235 | } 236 | 237 | func mergeOpenAPIDefinitions(definitionsGetters []openapicommon.GetOpenAPIDefinitions) openapicommon.GetOpenAPIDefinitions { 238 | return func(ref openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition { 239 | defsMap := make(map[string]openapicommon.OpenAPIDefinition) 240 | for _, definitionsGetter := range definitionsGetters { 241 | definitions := definitionsGetter(ref) 242 | for k, v := range definitions { 243 | defsMap[k] = v 244 | } 245 | } 246 | return defsMap 247 | } 248 | } 249 | 250 | func (b *AdapterBase) getAPIDefinitions() openapicommon.GetOpenAPIDefinitions { 251 | definitionsGetters := []openapicommon.GetOpenAPIDefinitions{generatedcore.GetOpenAPIDefinitions} 252 | if b.cmProvider != nil { 253 | definitionsGetters = append(definitionsGetters, generatedcustommetrics.GetOpenAPIDefinitions) 254 | } 255 | if b.emProvider != nil { 256 | definitionsGetters = append(definitionsGetters, generatedexternalmetrics.GetOpenAPIDefinitions) 257 | } 258 | return mergeOpenAPIDefinitions(definitionsGetters) 259 | } 260 | 261 | func (b *AdapterBase) defaultOpenAPIConfig() *openapicommon.Config { 262 | openAPIConfig := genericapiserver.DefaultOpenAPIConfig(b.getAPIDefinitions(), openapinamer.NewDefinitionNamer(apiserver.Scheme)) 263 | openAPIConfig.Info.Title = b.Name 264 | openAPIConfig.Info.Version = "1.0.0" 265 | return openAPIConfig 266 | } 267 | 268 | func (b *AdapterBase) defaultOpenAPIV3Config() *openapicommon.OpenAPIV3Config { 269 | openAPIConfig := genericapiserver.DefaultOpenAPIV3Config(b.getAPIDefinitions(), openapinamer.NewDefinitionNamer(apiserver.Scheme)) 270 | openAPIConfig.Info.Title = b.Name 271 | openAPIConfig.Info.Version = "1.0.0" 272 | return openAPIConfig 273 | } 274 | 275 | // Config fetches the configuration used to ultimately create the custom metrics adapter's 276 | // API server. While this method is idempotent, it does "cement" values of some of the other 277 | // fields, so make sure to only call it just before `Server` or `Run`. 278 | // Normal users should not need to call this method -- it's for advanced use cases. 279 | func (b *AdapterBase) Config() (*apiserver.Config, error) { 280 | if b.config == nil { 281 | b.InstallFlags() // just to be sure 282 | 283 | if b.Name == "" { 284 | b.Name = "custom-metrics-adapter" 285 | } 286 | 287 | if b.OpenAPIConfig == nil { 288 | b.OpenAPIConfig = b.defaultOpenAPIConfig() 289 | } 290 | b.CustomMetricsAdapterServerOptions.OpenAPIConfig = b.OpenAPIConfig 291 | 292 | if b.OpenAPIV3Config == nil { 293 | b.OpenAPIV3Config = b.defaultOpenAPIV3Config() 294 | } 295 | b.CustomMetricsAdapterServerOptions.OpenAPIV3Config = b.OpenAPIV3Config 296 | 297 | if errList := b.CustomMetricsAdapterServerOptions.Validate(); len(errList) > 0 { 298 | return nil, utilerrors.NewAggregate(errList) 299 | } 300 | 301 | // let's initialize informers if they're not already 302 | _, err := b.Informers() 303 | if err != nil { 304 | return nil, err 305 | } 306 | 307 | serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) 308 | serverConfig.ClientConfig = b.clientConfig 309 | serverConfig.SharedInformerFactory = b.informers 310 | err = b.CustomMetricsAdapterServerOptions.ApplyTo(serverConfig) 311 | if err != nil { 312 | return nil, err 313 | } 314 | b.config = &apiserver.Config{ 315 | GenericConfig: &serverConfig.Config, 316 | } 317 | } 318 | 319 | return b.config, nil 320 | } 321 | 322 | // Server fetches API server object used to ultimately run the custom metrics adapter. 323 | // While this method is idempotent, it does "cement" values of some of the other 324 | // fields, so make sure to only call it just before `Run`. 325 | // Normal users should not need to call this method -- it's for advanced use cases. 326 | func (b *AdapterBase) Server() (*apiserver.CustomMetricsAdapterServer, error) { 327 | if b.server == nil { 328 | config, err := b.Config() 329 | if err != nil { 330 | return nil, err 331 | } 332 | 333 | // we add in the informers if they're not nil, but we don't try and 334 | // construct them if the user didn't ask for them 335 | server, err := config.Complete(b.informers).New(b.Name, b.cmProvider, b.emProvider) 336 | if err != nil { 337 | return nil, err 338 | } 339 | b.server = server 340 | } 341 | 342 | return b.server, nil 343 | } 344 | 345 | // Informers returns a SharedInformerFactory for constructing new informers. 346 | // The informers will be automatically started as part of starting the adapter. 347 | func (b *AdapterBase) Informers() (informers.SharedInformerFactory, error) { 348 | if b.informers == nil { 349 | clientConfig, err := b.ClientConfig() 350 | if err != nil { 351 | return nil, err 352 | } 353 | kubeClient, err := kubernetes.NewForConfig(clientConfig) 354 | if err != nil { 355 | return nil, err 356 | } 357 | b.informers = informers.NewSharedInformerFactory(kubeClient, 0) 358 | } 359 | 360 | return b.informers, nil 361 | } 362 | 363 | // Run runs this custom metrics adapter until the given stop channel is closed. 364 | func (b *AdapterBase) Run(ctx context.Context) error { 365 | server, err := b.Server() 366 | if err != nil { 367 | return err 368 | } 369 | 370 | return server.GenericAPIServer.PrepareRun().RunWithContext(ctx) 371 | } 372 | -------------------------------------------------------------------------------- /pkg/cmd/builder_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package cmd 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | 24 | "k8s.io/kube-openapi/pkg/builder" 25 | 26 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/fake" 27 | ) 28 | 29 | func TestDefaultOpenAPIConfig(t *testing.T) { 30 | t.Run("no metric", func(t *testing.T) { 31 | adapter := &AdapterBase{} 32 | config := adapter.defaultOpenAPIConfig() 33 | 34 | _, err1 := builder.BuildOpenAPIDefinitionsForResources(config, "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2.MetricValue") 35 | // Should err, because no provider is installed 36 | assert.Error(t, err1) 37 | _, err2 := builder.BuildOpenAPIDefinitionsForResources(config, "k8s.io/metrics/pkg/apis/external_metrics/v1beta1.ExternalMetricValue") 38 | assert.Error(t, err2) 39 | }) 40 | 41 | t.Run("custom and external metrics", func(t *testing.T) { 42 | adapter := &AdapterBase{} 43 | 44 | adapter.WithCustomMetrics(fake.NewProvider()) 45 | adapter.WithExternalMetrics(fake.NewProvider()) 46 | 47 | config := adapter.defaultOpenAPIConfig() 48 | 49 | _, err1 := builder.BuildOpenAPIDefinitionsForResources(config, "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2.MetricValue") 50 | // Should NOT err 51 | assert.NoError(t, err1) 52 | _, err2 := builder.BuildOpenAPIDefinitionsForResources(config, "k8s.io/metrics/pkg/apis/external_metrics/v1beta1.ExternalMetricValue") 53 | assert.NoError(t, err2) 54 | }) 55 | } 56 | -------------------------------------------------------------------------------- /pkg/cmd/options/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package options provides configuration options for the metrics API server. 18 | package options 19 | 20 | import ( 21 | "fmt" 22 | "net" 23 | 24 | "github.com/spf13/pflag" 25 | 26 | genericapiserver "k8s.io/apiserver/pkg/server" 27 | genericoptions "k8s.io/apiserver/pkg/server/options" 28 | "k8s.io/client-go/kubernetes" 29 | openapicommon "k8s.io/kube-openapi/pkg/common" 30 | ) 31 | 32 | // CustomMetricsAdapterServerOptions contains the of options used to configure 33 | // the metrics API server. 34 | // 35 | // It is based on a subset of [genericoptions.RecommendedOptions]. 36 | type CustomMetricsAdapterServerOptions struct { 37 | SecureServing *genericoptions.SecureServingOptionsWithLoopback 38 | Authentication *genericoptions.DelegatingAuthenticationOptions 39 | Authorization *genericoptions.DelegatingAuthorizationOptions 40 | Audit *genericoptions.AuditOptions 41 | Features *genericoptions.FeatureOptions 42 | 43 | OpenAPIConfig *openapicommon.Config 44 | OpenAPIV3Config *openapicommon.OpenAPIV3Config 45 | EnableMetrics bool 46 | } 47 | 48 | // NewCustomMetricsAdapterServerOptions creates a new instance of 49 | // CustomMetricsAdapterServerOptions with its default values. 50 | func NewCustomMetricsAdapterServerOptions() *CustomMetricsAdapterServerOptions { 51 | o := &CustomMetricsAdapterServerOptions{ 52 | SecureServing: genericoptions.NewSecureServingOptions().WithLoopback(), 53 | Authentication: genericoptions.NewDelegatingAuthenticationOptions(), 54 | Authorization: genericoptions.NewDelegatingAuthorizationOptions(), 55 | Audit: genericoptions.NewAuditOptions(), 56 | Features: genericoptions.NewFeatureOptions(), 57 | 58 | EnableMetrics: true, 59 | } 60 | 61 | // Explicitly disable Priority and Fairness since metric servers are not 62 | // meant to be queried directly by default. 63 | o.Features.EnablePriorityAndFairness = false 64 | 65 | return o 66 | } 67 | 68 | // Validate validates CustomMetricsAdapterServerOptions 69 | func (o CustomMetricsAdapterServerOptions) Validate() []error { 70 | errors := []error{} 71 | errors = append(errors, o.SecureServing.Validate()...) 72 | errors = append(errors, o.Authentication.Validate()...) 73 | errors = append(errors, o.Authorization.Validate()...) 74 | errors = append(errors, o.Audit.Validate()...) 75 | errors = append(errors, o.Features.Validate()...) 76 | return errors 77 | } 78 | 79 | // AddFlags adds the flags defined for the options, to the given flagset. 80 | func (o *CustomMetricsAdapterServerOptions) AddFlags(fs *pflag.FlagSet) { 81 | o.SecureServing.AddFlags(fs) 82 | o.Authentication.AddFlags(fs) 83 | o.Authorization.AddFlags(fs) 84 | o.Audit.AddFlags(fs) 85 | o.Features.AddFlags(fs) 86 | } 87 | 88 | // ApplyTo applies CustomMetricsAdapterServerOptions to the server configuration. 89 | func (o *CustomMetricsAdapterServerOptions) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error { 90 | // TODO have a "real" external address (have an AdvertiseAddress?) 91 | if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil { 92 | return fmt.Errorf("error creating self-signed certificates: %v", err) 93 | } 94 | 95 | if err := o.SecureServing.ApplyTo(&serverConfig.SecureServing, &serverConfig.LoopbackClientConfig); err != nil { 96 | return err 97 | } 98 | if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil { 99 | return err 100 | } 101 | if err := o.Authorization.ApplyTo(&serverConfig.Authorization); err != nil { 102 | return err 103 | } 104 | if err := o.Audit.ApplyTo(&serverConfig.Config); err != nil { 105 | return err 106 | } 107 | 108 | clientset, err := kubernetes.NewForConfig(serverConfig.ClientConfig) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | if err := o.Features.ApplyTo(&serverConfig.Config, clientset, serverConfig.SharedInformerFactory); err != nil { 114 | return err 115 | } 116 | 117 | // enable OpenAPI schemas 118 | if o.OpenAPIConfig != nil { 119 | serverConfig.OpenAPIConfig = o.OpenAPIConfig 120 | } 121 | if o.OpenAPIV3Config != nil { 122 | serverConfig.OpenAPIV3Config = o.OpenAPIV3Config 123 | } 124 | 125 | serverConfig.EnableMetrics = o.EnableMetrics 126 | 127 | return nil 128 | } 129 | -------------------------------------------------------------------------------- /pkg/cmd/options/options_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package options 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/spf13/pflag" 23 | "github.com/stretchr/testify/assert" 24 | 25 | utilerrors "k8s.io/apimachinery/pkg/util/errors" 26 | genericapiserver "k8s.io/apiserver/pkg/server" 27 | "k8s.io/client-go/informers" 28 | "k8s.io/client-go/rest" 29 | 30 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver" 31 | ) 32 | 33 | func TestValidate(t *testing.T) { 34 | cases := []struct { 35 | testName string 36 | args []string 37 | shouldErr bool 38 | }{ 39 | { 40 | testName: "only-secure-port", 41 | args: []string{"--secure-port=6443"}, // default is 443, which requires privileges 42 | shouldErr: false, 43 | }, 44 | { 45 | testName: "secure-port-0", 46 | args: []string{"--secure-port=0"}, // means: "don't serve HTTPS at all" 47 | shouldErr: false, 48 | }, 49 | { 50 | testName: "invalid-secure-port", 51 | args: []string{"--secure-port=-1"}, 52 | shouldErr: true, 53 | }, 54 | { 55 | testName: "empty-header", 56 | args: []string{"--secure-port=6443", "--requestheader-username-headers=\" \""}, 57 | shouldErr: true, 58 | }, 59 | { 60 | testName: "invalid-audit-log-format", 61 | args: []string{"--secure-port=6443", "--audit-log-path=file", "--audit-log-format=txt"}, 62 | shouldErr: true, 63 | }, 64 | } 65 | 66 | for _, c := range cases { 67 | t.Run(c.testName, func(t *testing.T) { 68 | o := NewCustomMetricsAdapterServerOptions() 69 | 70 | flagSet := pflag.NewFlagSet("", pflag.PanicOnError) 71 | o.AddFlags(flagSet) 72 | err := flagSet.Parse(c.args) 73 | assert.NoErrorf(t, err, "Error while parsing flags") 74 | 75 | errList := o.Validate() 76 | err = utilerrors.NewAggregate(errList) 77 | if c.shouldErr { 78 | assert.Errorf(t, err, "Expected error while validating options") 79 | } else { 80 | assert.NoErrorf(t, err, "Error while validating options") 81 | } 82 | }) 83 | } 84 | } 85 | 86 | func TestApplyTo(t *testing.T) { 87 | cases := []struct { 88 | testName string 89 | args []string 90 | shouldErr bool 91 | }{ 92 | { 93 | testName: "only-secure-port", 94 | args: []string{"--secure-port=6443"}, // default is 443, which requires privileges 95 | }, 96 | { 97 | testName: "secure-port-0", 98 | args: []string{"--secure-port=0"}, // means: "don't serve HTTPS at all" 99 | shouldErr: false, 100 | }, 101 | } 102 | 103 | for _, c := range cases { 104 | t.Run(c.testName, func(t *testing.T) { 105 | o := NewCustomMetricsAdapterServerOptions() 106 | 107 | // Unit tests have no Kubernetes cluster access 108 | o.Authentication.RemoteKubeConfigFileOptional = true 109 | o.Authorization.RemoteKubeConfigFileOptional = true 110 | 111 | flagSet := pflag.NewFlagSet("", pflag.PanicOnError) 112 | o.AddFlags(flagSet) 113 | err := flagSet.Parse(c.args) 114 | assert.NoErrorf(t, err, "Error while parsing flags") 115 | 116 | serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) 117 | serverConfig.ClientConfig = &rest.Config{} 118 | serverConfig.SharedInformerFactory = informers.NewSharedInformerFactory(nil, 0) 119 | err = o.ApplyTo(serverConfig) 120 | 121 | defer func() { 122 | // Close the listener, if any 123 | if serverConfig.SecureServing != nil && serverConfig.SecureServing.Listener != nil { 124 | err := serverConfig.SecureServing.Listener.Close() 125 | assert.NoError(t, err) 126 | } 127 | }() 128 | 129 | assert.NoErrorf(t, err, "Error while applying options") 130 | }) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /pkg/dynamicmapper/mapper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package dynamicmapper 16 | 17 | import ( 18 | "fmt" 19 | "sync" 20 | "time" 21 | 22 | "k8s.io/apimachinery/pkg/api/meta" 23 | "k8s.io/apimachinery/pkg/runtime/schema" 24 | "k8s.io/apimachinery/pkg/util/wait" 25 | "k8s.io/client-go/discovery" 26 | "k8s.io/client-go/restmapper" 27 | "k8s.io/klog/v2" 28 | ) 29 | 30 | // RengeneratingDiscoveryRESTMapper is a RESTMapper which Regenerates its cache of mappings periodically. 31 | // It functions by recreating a normal discovery RESTMapper at the specified interval. 32 | // We don't refresh automatically on cache misses, since we get called on every label, plenty of which will 33 | // be unrelated to Kubernetes resources. 34 | type RegeneratingDiscoveryRESTMapper struct { 35 | discoveryClient discovery.DiscoveryInterface 36 | 37 | refreshInterval time.Duration 38 | 39 | mu sync.RWMutex 40 | 41 | delegate meta.RESTMapper 42 | } 43 | 44 | func NewRESTMapper(discoveryClient discovery.DiscoveryInterface, refreshInterval time.Duration) (*RegeneratingDiscoveryRESTMapper, error) { 45 | mapper := &RegeneratingDiscoveryRESTMapper{ 46 | discoveryClient: discoveryClient, 47 | refreshInterval: refreshInterval, 48 | } 49 | if err := mapper.RegenerateMappings(); err != nil { 50 | return nil, fmt.Errorf("unable to populate initial set of REST mappings: %v", err) 51 | } 52 | 53 | return mapper, nil 54 | } 55 | 56 | // RunUtil runs the mapping refresher until the given stop channel is closed. 57 | func (m *RegeneratingDiscoveryRESTMapper) RunUntil(stop <-chan struct{}) { 58 | go wait.Until(func() { 59 | if err := m.RegenerateMappings(); err != nil { 60 | klog.Errorf("error regenerating REST mappings from discovery: %v", err) 61 | } 62 | }, m.refreshInterval, stop) 63 | } 64 | 65 | func (m *RegeneratingDiscoveryRESTMapper) RegenerateMappings() error { 66 | resources, err := restmapper.GetAPIGroupResources(m.discoveryClient) 67 | if err != nil { 68 | return err 69 | } 70 | newDelegate := restmapper.NewDiscoveryRESTMapper(resources) 71 | 72 | // don't lock until we're ready to replace 73 | m.mu.Lock() 74 | defer m.mu.Unlock() 75 | m.delegate = newDelegate 76 | 77 | return nil 78 | } 79 | 80 | func (m *RegeneratingDiscoveryRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { 81 | m.mu.RLock() 82 | defer m.mu.RUnlock() 83 | 84 | return m.delegate.KindFor(resource) 85 | } 86 | 87 | func (m *RegeneratingDiscoveryRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { 88 | m.mu.RLock() 89 | defer m.mu.RUnlock() 90 | 91 | return m.delegate.KindsFor(resource) 92 | } 93 | 94 | func (m *RegeneratingDiscoveryRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { 95 | m.mu.RLock() 96 | defer m.mu.RUnlock() 97 | 98 | return m.delegate.ResourceFor(input) 99 | } 100 | 101 | func (m *RegeneratingDiscoveryRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { 102 | m.mu.RLock() 103 | defer m.mu.RUnlock() 104 | 105 | return m.delegate.ResourcesFor(input) 106 | } 107 | 108 | func (m *RegeneratingDiscoveryRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { 109 | m.mu.RLock() 110 | defer m.mu.RUnlock() 111 | 112 | return m.delegate.RESTMapping(gk, versions...) 113 | } 114 | 115 | func (m *RegeneratingDiscoveryRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { 116 | m.mu.RLock() 117 | defer m.mu.RUnlock() 118 | 119 | return m.delegate.RESTMappings(gk, versions...) 120 | } 121 | 122 | func (m *RegeneratingDiscoveryRESTMapper) ResourceSingularizer(resource string) (singular string, err error) { 123 | m.mu.RLock() 124 | defer m.mu.RUnlock() 125 | 126 | return m.delegate.ResourceSingularizer(resource) 127 | } 128 | -------------------------------------------------------------------------------- /pkg/dynamicmapper/mapper_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package dynamicmapper 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | 26 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 27 | "k8s.io/apimachinery/pkg/runtime/schema" 28 | "k8s.io/client-go/discovery/fake" 29 | core "k8s.io/client-go/testing" 30 | ) 31 | 32 | const testingMapperRefreshInterval = 1 * time.Second 33 | 34 | func setupMapper(t *testing.T, stopChan <-chan struct{}) (*RegeneratingDiscoveryRESTMapper, *fake.FakeDiscovery) { 35 | fakeDiscovery := &fake.FakeDiscovery{Fake: &core.Fake{}} 36 | mapper, err := NewRESTMapper(fakeDiscovery, testingMapperRefreshInterval) 37 | require.NoError(t, err, "constructing the rest mapper shouldn't have produced an error") 38 | 39 | fakeDiscovery.Resources = []*metav1.APIResourceList{ 40 | { 41 | GroupVersion: "v1", 42 | APIResources: []metav1.APIResource{ 43 | {Name: "pods", Namespaced: true, Kind: "Pod"}, 44 | }, 45 | }, 46 | } 47 | 48 | if stopChan != nil { 49 | mapper.RunUntil(stopChan) 50 | } 51 | 52 | return mapper, fakeDiscovery 53 | } 54 | 55 | func TestRegeneratingUpdatesMapper(t *testing.T) { 56 | mapper, fakeDiscovery := setupMapper(t, nil) 57 | require.NoError(t, mapper.RegenerateMappings(), "regenerating the mappings the first time should not have yielded an error") 58 | 59 | // add first, to ensure we don't update before regen 60 | fakeDiscovery.Resources[0].APIResources = append(fakeDiscovery.Resources[0].APIResources, metav1.APIResource{ 61 | Name: "services", Namespaced: true, Kind: "Service", 62 | }) 63 | fakeDiscovery.Resources = append(fakeDiscovery.Resources, &metav1.APIResourceList{ 64 | GroupVersion: "wardle/v1alpha1", 65 | APIResources: []metav1.APIResource{ 66 | {Name: "flunders", Namespaced: true, Kind: "Flunder"}, 67 | }, 68 | }) 69 | 70 | // fetch before regen 71 | podsGVK, err := mapper.KindFor(schema.GroupVersionResource{Resource: "pods"}) 72 | require.NoError(t, err, "should have been able to fetch the kind for 'pods' the first time") 73 | assert.Equal(t, schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, podsGVK, "should have correctly fetched the kind for 'pods' the first time") 74 | _, err = mapper.KindFor(schema.GroupVersionResource{Resource: "services"}) 75 | assert.Error(t, err, "should not have been able to fetch the kind for 'services' the first time") 76 | _, err = mapper.KindFor(schema.GroupVersionResource{Resource: "flunders", Group: "wardle"}) 77 | assert.Error(t, err, "should not have been able to fetch the kind for 'flunders.wardle' the first time") 78 | 79 | // regen and check again 80 | require.NoError(t, mapper.RegenerateMappings(), "regenerating the mappings the second time should not have yielded an error") 81 | 82 | podsGVK, err = mapper.KindFor(schema.GroupVersionResource{Resource: "pods"}) 83 | if assert.NoError(t, err, "should have been able to fetch the kind for 'pods' the second time") { 84 | assert.Equal(t, schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, podsGVK, "should have correctly fetched the kind for 'pods' the first time") 85 | } 86 | servicesGVK, err := mapper.KindFor(schema.GroupVersionResource{Resource: "services"}) 87 | if assert.NoError(t, err, "should have been able to fetch the kind for 'services' the second time") { 88 | assert.Equal(t, schema.GroupVersionKind{Version: "v1", Kind: "Service"}, servicesGVK, "should have correctly fetched the kind for 'services' the second time") 89 | } 90 | flundersGVK, err := mapper.KindFor(schema.GroupVersionResource{Resource: "flunders", Group: "wardle"}) 91 | if assert.NoError(t, err, "should have been able to fetch the kind for 'flunders.wardle' the second time") { 92 | assert.Equal(t, schema.GroupVersionKind{Version: "v1alpha1", Kind: "Flunder", Group: "wardle"}, flundersGVK, "should have correctly fetched the kind for 'flunders.wardle' the second time") 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /pkg/generated/openapi-gen.go: -------------------------------------------------------------------------------- 1 | //go:build codegen 2 | // +build codegen 3 | 4 | /* 5 | Copyright 2018 The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Package is only a stub to ensure k8s.io/kube-openapi/cmd/openapi-gen is vendored 21 | // so the same version of kube-openapi is used to generate and render the openapi spec 22 | package main 23 | 24 | import ( 25 | _ "k8s.io/kube-openapi/cmd/openapi-gen" 26 | ) 27 | -------------------------------------------------------------------------------- /pkg/generated/openapi/core/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // package core contains autogenerated openapi schema definitions 16 | // for the core k8s.io API. 17 | package core 18 | -------------------------------------------------------------------------------- /pkg/generated/openapi/custommetrics/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // package custommetrics contains autogenerated openapi schema definitions 16 | // for the k8s.io/metrics/pkg/apis/custom_metrics API. 17 | package custommetrics 18 | -------------------------------------------------------------------------------- /pkg/generated/openapi/externalmetrics/doc.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Kubernetes Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // package externalmetrics contains autogenerated openapi schema definitions 16 | // for the k8s.io/metrics/pkg/apis/external_metrics API. 17 | package externalmetrics 18 | -------------------------------------------------------------------------------- /pkg/generated/openapi/externalmetrics/zz_generated.openapi.go: -------------------------------------------------------------------------------- 1 | //go:build !ignore_autogenerated 2 | // +build !ignore_autogenerated 3 | 4 | /* 5 | Copyright The Kubernetes Authors. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | */ 19 | 20 | // Code generated by openapi-gen. DO NOT EDIT. 21 | 22 | package externalmetrics 23 | 24 | import ( 25 | common "k8s.io/kube-openapi/pkg/common" 26 | spec "k8s.io/kube-openapi/pkg/validation/spec" 27 | ) 28 | 29 | func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { 30 | return map[string]common.OpenAPIDefinition{ 31 | "k8s.io/metrics/pkg/apis/external_metrics/v1beta1.ExternalMetricValue": schema_pkg_apis_external_metrics_v1beta1_ExternalMetricValue(ref), 32 | "k8s.io/metrics/pkg/apis/external_metrics/v1beta1.ExternalMetricValueList": schema_pkg_apis_external_metrics_v1beta1_ExternalMetricValueList(ref), 33 | } 34 | } 35 | 36 | func schema_pkg_apis_external_metrics_v1beta1_ExternalMetricValue(ref common.ReferenceCallback) common.OpenAPIDefinition { 37 | return common.OpenAPIDefinition{ 38 | Schema: spec.Schema{ 39 | SchemaProps: spec.SchemaProps{ 40 | Description: "ExternalMetricValue is a metric value for external metric A single metric value is identified by metric name and a set of string labels. For one metric there can be multiple values with different sets of labels.", 41 | Type: []string{"object"}, 42 | Properties: map[string]spec.Schema{ 43 | "kind": { 44 | SchemaProps: spec.SchemaProps{ 45 | Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", 46 | Type: []string{"string"}, 47 | Format: "", 48 | }, 49 | }, 50 | "apiVersion": { 51 | SchemaProps: spec.SchemaProps{ 52 | Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", 53 | Type: []string{"string"}, 54 | Format: "", 55 | }, 56 | }, 57 | "metricName": { 58 | SchemaProps: spec.SchemaProps{ 59 | Description: "the name of the metric", 60 | Default: "", 61 | Type: []string{"string"}, 62 | Format: "", 63 | }, 64 | }, 65 | "metricLabels": { 66 | SchemaProps: spec.SchemaProps{ 67 | Description: "a set of labels that identify a single time series for the metric", 68 | Type: []string{"object"}, 69 | AdditionalProperties: &spec.SchemaOrBool{ 70 | Allows: true, 71 | Schema: &spec.Schema{ 72 | SchemaProps: spec.SchemaProps{ 73 | Default: "", 74 | Type: []string{"string"}, 75 | Format: "", 76 | }, 77 | }, 78 | }, 79 | }, 80 | }, 81 | "timestamp": { 82 | SchemaProps: spec.SchemaProps{ 83 | Description: "indicates the time at which the metrics were produced", 84 | Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Time"), 85 | }, 86 | }, 87 | "window": { 88 | SchemaProps: spec.SchemaProps{ 89 | Description: "indicates the window ([Timestamp-Window, Timestamp]) from which these metrics were calculated, when returning rate metrics calculated from cumulative metrics (or zero for non-calculated instantaneous metrics).", 90 | Type: []string{"integer"}, 91 | Format: "int64", 92 | }, 93 | }, 94 | "value": { 95 | SchemaProps: spec.SchemaProps{ 96 | Description: "the value of the metric", 97 | Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), 98 | }, 99 | }, 100 | }, 101 | Required: []string{"metricName", "metricLabels", "timestamp", "value"}, 102 | }, 103 | }, 104 | Dependencies: []string{ 105 | "k8s.io/apimachinery/pkg/api/resource.Quantity", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, 106 | } 107 | } 108 | 109 | func schema_pkg_apis_external_metrics_v1beta1_ExternalMetricValueList(ref common.ReferenceCallback) common.OpenAPIDefinition { 110 | return common.OpenAPIDefinition{ 111 | Schema: spec.Schema{ 112 | SchemaProps: spec.SchemaProps{ 113 | Description: "ExternalMetricValueList is a list of values for a given metric for some set labels", 114 | Type: []string{"object"}, 115 | Properties: map[string]spec.Schema{ 116 | "kind": { 117 | SchemaProps: spec.SchemaProps{ 118 | Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", 119 | Type: []string{"string"}, 120 | Format: "", 121 | }, 122 | }, 123 | "apiVersion": { 124 | SchemaProps: spec.SchemaProps{ 125 | Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", 126 | Type: []string{"string"}, 127 | Format: "", 128 | }, 129 | }, 130 | "metadata": { 131 | SchemaProps: spec.SchemaProps{ 132 | Default: map[string]interface{}{}, 133 | Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), 134 | }, 135 | }, 136 | "items": { 137 | SchemaProps: spec.SchemaProps{ 138 | Description: "value of the metric matching a given set of labels", 139 | Type: []string{"array"}, 140 | Items: &spec.SchemaOrArray{ 141 | Schema: &spec.Schema{ 142 | SchemaProps: spec.SchemaProps{ 143 | Default: map[string]interface{}{}, 144 | Ref: ref("k8s.io/metrics/pkg/apis/external_metrics/v1beta1.ExternalMetricValue"), 145 | }, 146 | }, 147 | }, 148 | }, 149 | }, 150 | }, 151 | Required: []string{"items"}, 152 | }, 153 | }, 154 | Dependencies: []string{ 155 | "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta", "k8s.io/metrics/pkg/apis/external_metrics/v1beta1.ExternalMetricValue"}, 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /pkg/provider/defaults/default_metric_providers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package defaults provides a default implementation of metrics providers. 18 | package defaults 19 | 20 | import ( 21 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 22 | ) 23 | 24 | type DefaultExternalMetricsProvider struct{} 25 | 26 | func (em DefaultExternalMetricsProvider) ListAllExternalMetrics() []provider.ExternalMetricInfo { 27 | return []provider.ExternalMetricInfo{ 28 | { 29 | Metric: "externalmetrics", 30 | }, 31 | } 32 | } 33 | 34 | type DefaultCustomMetricsProvider struct{} 35 | 36 | func (cm DefaultCustomMetricsProvider) ListAllMetrics() []provider.CustomMetricInfo { 37 | return []provider.CustomMetricInfo{ 38 | { 39 | Metric: "custommetrics", 40 | }, 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pkg/provider/errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "fmt" 21 | "net/http" 22 | 23 | apierr "k8s.io/apimachinery/pkg/api/errors" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | 26 | "k8s.io/apimachinery/pkg/labels" 27 | "k8s.io/apimachinery/pkg/runtime/schema" 28 | ) 29 | 30 | // NewMetricNotFoundError returns a StatusError indicating the given metric could not be found. 31 | // It is similar to NewNotFound, but more specialized 32 | func NewMetricNotFoundError(resource schema.GroupResource, metricName string) *apierr.StatusError { 33 | return &apierr.StatusError{ErrStatus: metav1.Status{ 34 | Status: metav1.StatusFailure, 35 | Code: int32(http.StatusNotFound), 36 | Reason: metav1.StatusReasonNotFound, 37 | Message: fmt.Sprintf("the server could not find the metric %s for %s", metricName, resource.String()), 38 | }} 39 | } 40 | 41 | // NewMetricNotFoundForError returns a StatusError indicating the given metric could not be found for 42 | // the given named object. It is similar to NewNotFound, but more specialized 43 | func NewMetricNotFoundForError(resource schema.GroupResource, metricName string, resourceName string) *apierr.StatusError { 44 | return &apierr.StatusError{ErrStatus: metav1.Status{ 45 | Status: metav1.StatusFailure, 46 | Code: int32(http.StatusNotFound), 47 | Reason: metav1.StatusReasonNotFound, 48 | Message: fmt.Sprintf("the server could not find the metric %s for %s %s", metricName, resource.String(), resourceName), 49 | }} 50 | } 51 | 52 | // NewMetricNotFoundForSelectorError returns a StatusError indicating the given metric could not be found for 53 | // the given named object and selector. It is similar to NewNotFound, but more specialized 54 | func NewMetricNotFoundForSelectorError(resource schema.GroupResource, metricName string, resourceName string, selector labels.Selector) *apierr.StatusError { 55 | return &apierr.StatusError{ErrStatus: metav1.Status{ 56 | Status: metav1.StatusFailure, 57 | Code: int32(http.StatusNotFound), 58 | Reason: metav1.StatusReasonNotFound, 59 | Message: fmt.Sprintf("the server could not find the metric %s for %s %s with selector %s", metricName, resource.String(), resourceName, selector.String()), 60 | }} 61 | } 62 | -------------------------------------------------------------------------------- /pkg/provider/fake/fake.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // Package fake provides a fake implementation of metrics providers. 18 | package fake 19 | 20 | import ( 21 | "context" 22 | 23 | "k8s.io/apimachinery/pkg/labels" 24 | "k8s.io/apimachinery/pkg/types" 25 | "k8s.io/metrics/pkg/apis/custom_metrics" 26 | "k8s.io/metrics/pkg/apis/external_metrics" 27 | 28 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 29 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/defaults" 30 | ) 31 | 32 | type fakeProvider struct { 33 | defaults.DefaultCustomMetricsProvider 34 | defaults.DefaultExternalMetricsProvider 35 | } 36 | 37 | func (*fakeProvider) GetMetricByName(_ context.Context, _ types.NamespacedName, _ provider.CustomMetricInfo, _ labels.Selector) (*custom_metrics.MetricValue, error) { 38 | return &custom_metrics.MetricValue{}, nil 39 | } 40 | 41 | func (*fakeProvider) GetMetricBySelector(_ context.Context, _ string, _ labels.Selector, _ provider.CustomMetricInfo, _ labels.Selector) (*custom_metrics.MetricValueList, error) { 42 | return &custom_metrics.MetricValueList{}, nil 43 | } 44 | 45 | func (*fakeProvider) GetExternalMetric(_ context.Context, _ string, _ labels.Selector, _ provider.ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) { 46 | return &external_metrics.ExternalMetricValueList{}, nil 47 | } 48 | 49 | // NewProvider creates a fake implementation of MetricsProvider. 50 | func NewProvider() provider.MetricsProvider { 51 | return &fakeProvider{} 52 | } 53 | -------------------------------------------------------------------------------- /pkg/provider/helpers/helpers.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package helpers 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | apimeta "k8s.io/apimachinery/pkg/api/meta" 24 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 25 | "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 26 | "k8s.io/apimachinery/pkg/labels" 27 | "k8s.io/apimachinery/pkg/runtime" 28 | "k8s.io/apimachinery/pkg/runtime/schema" 29 | "k8s.io/apimachinery/pkg/types" 30 | "k8s.io/client-go/dynamic" 31 | "k8s.io/metrics/pkg/apis/custom_metrics" 32 | 33 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 34 | ) 35 | 36 | // ResourceFor attempts to resolve a single qualified resource for the given metric. 37 | // You can use this to resolve a particular piece of CustomMetricInfo to the underlying 38 | // resource that it describes, so that you can list matching objects in the cluster. 39 | func ResourceFor(mapper apimeta.RESTMapper, info provider.CustomMetricInfo) (schema.GroupVersionResource, error) { 40 | fullResources, err := mapper.ResourcesFor(info.GroupResource.WithVersion("")) 41 | if err == nil && len(fullResources) == 0 { 42 | err = fmt.Errorf("no fully versioned resources known for group-resource %v", info.GroupResource) 43 | } 44 | if err != nil { 45 | return schema.GroupVersionResource{}, fmt.Errorf("unable to find preferred version to list matching resource names: %v", err) 46 | } 47 | 48 | return fullResources[0], nil 49 | } 50 | 51 | // ReferenceFor returns a new ObjectReference for the given group-resource and name. 52 | // The group-resource is converted into a group-version-kind using the given RESTMapper. 53 | // You can use this to easily construct an object reference for use in the DescribedObject 54 | // field of CustomMetricInfo. 55 | func ReferenceFor(mapper apimeta.RESTMapper, name types.NamespacedName, info provider.CustomMetricInfo) (custom_metrics.ObjectReference, error) { 56 | kind, err := mapper.KindFor(info.GroupResource.WithVersion("")) 57 | if err != nil { 58 | return custom_metrics.ObjectReference{}, err 59 | } 60 | 61 | // NB: return straight value, not a reference, so that the object can easily 62 | // be copied for use multiple times with a different name. 63 | return custom_metrics.ObjectReference{ 64 | APIVersion: kind.Group + "/" + kind.Version, 65 | Kind: kind.Kind, 66 | Name: name.Name, 67 | Namespace: name.Namespace, 68 | }, nil 69 | } 70 | 71 | // ListObjectNames uses the given dynamic client to list the names of all objects 72 | // of the given resource matching the given selector. Namespace may be empty 73 | // if the metric is for a root-scoped resource. 74 | func ListObjectNames(mapper apimeta.RESTMapper, client dynamic.Interface, namespace string, selector labels.Selector, info provider.CustomMetricInfo) ([]string, error) { 75 | res, err := ResourceFor(mapper, info) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | var resClient dynamic.ResourceInterface 81 | if info.Namespaced { 82 | resClient = client.Resource(res).Namespace(namespace) 83 | } else { 84 | resClient = client.Resource(res) 85 | } 86 | 87 | matchingObjectsRaw, err := resClient.List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | if !apimeta.IsListType(matchingObjectsRaw) { 93 | return nil, fmt.Errorf("result of label selector list operation was not a list") 94 | } 95 | 96 | var names []string 97 | err = apimeta.EachListItem(matchingObjectsRaw, func(item runtime.Object) error { 98 | objName := item.(*unstructured.Unstructured).GetName() 99 | names = append(names, objName) 100 | return nil 101 | }) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | return names, nil 107 | } 108 | -------------------------------------------------------------------------------- /pkg/provider/interfaces.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | apimeta "k8s.io/apimachinery/pkg/api/meta" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/apimachinery/pkg/runtime/schema" 26 | "k8s.io/apimachinery/pkg/types" 27 | "k8s.io/metrics/pkg/apis/custom_metrics" 28 | "k8s.io/metrics/pkg/apis/external_metrics" 29 | ) 30 | 31 | // CustomMetricInfo describes a metric for a particular 32 | // fully-qualified group resource. 33 | type CustomMetricInfo struct { 34 | GroupResource schema.GroupResource 35 | Namespaced bool 36 | Metric string 37 | } 38 | 39 | // ExternalMetricInfo describes a metric. 40 | type ExternalMetricInfo struct { 41 | Metric string 42 | } 43 | 44 | func (i CustomMetricInfo) String() string { 45 | if i.Namespaced { 46 | return fmt.Sprintf("%s/%s(namespaced)", i.GroupResource.String(), i.Metric) 47 | } 48 | return fmt.Sprintf("%s/%s", i.GroupResource.String(), i.Metric) 49 | } 50 | 51 | // Normalized returns a copy of the current MetricInfo with the GroupResource resolved using the 52 | // provided REST mapper, to ensure consistent pluralization, etc, for use when looking up or comparing 53 | // the MetricInfo. It also returns the singular form of the GroupResource associated with the given 54 | // MetricInfo. 55 | func (i CustomMetricInfo) Normalized(mapper apimeta.RESTMapper) (normalizedInfo CustomMetricInfo, singluarResource string, err error) { 56 | normalizedGroupRes, err := mapper.ResourceFor(i.GroupResource.WithVersion("")) 57 | if err != nil { 58 | return i, "", err 59 | } 60 | i.GroupResource = normalizedGroupRes.GroupResource() 61 | 62 | singularResource, err := mapper.ResourceSingularizer(i.GroupResource.Resource) 63 | if err != nil { 64 | return i, "", err 65 | } 66 | 67 | return i, singularResource, nil 68 | } 69 | 70 | // CustomMetricsProvider is a source of custom metrics 71 | // which is able to supply a list of available metrics, 72 | // as well as metric values themselves on demand. 73 | // 74 | // Note that group-resources are provided as GroupResources, 75 | // not GroupKinds. This is to allow flexibility on the part 76 | // of the implementor: implementors do not necessarily need 77 | // to be aware of all existing kinds and their corresponding 78 | // REST mappings in order to perform queries. 79 | // 80 | // For queries that use label selectors, it is up to the 81 | // implementor to decide how to make use of the label selector -- 82 | // they may wish to query the main Kubernetes API server, or may 83 | // wish to simply make use of stored information in their TSDB. 84 | type CustomMetricsProvider interface { 85 | // GetMetricByName fetches a particular metric for a particular object. 86 | // The namespace will be empty if the metric is root-scoped. 87 | GetMetricByName(ctx context.Context, name types.NamespacedName, info CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) 88 | 89 | // GetMetricBySelector fetches a particular metric for a set of objects matching 90 | // the given label selector. The namespace will be empty if the metric is root-scoped. 91 | GetMetricBySelector(ctx context.Context, namespace string, selector labels.Selector, info CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) 92 | 93 | // ListAllMetrics provides a list of all available metrics at 94 | // the current time. Note that this is not allowed to return 95 | // an error, so it is recommended that implementors use the 96 | // default implementation provided by DefaultCustomMetricsProvider. 97 | ListAllMetrics() []CustomMetricInfo 98 | } 99 | 100 | // ExternalMetricsProvider is a source of external metrics. 101 | // Metric is normally identified by a name and a set of labels/tags. It is up to a specific 102 | // implementation how to translate metricSelector to a filter for metric values. 103 | // Namespace can be used by the implemetation for metric identification, access control or ignored. 104 | type ExternalMetricsProvider interface { 105 | GetExternalMetric(ctx context.Context, namespace string, metricSelector labels.Selector, info ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) 106 | 107 | // ListAllExternalMetrics provides a list of all available 108 | // external metrics at the current time. 109 | // Note that this is not allowed to return an error, so it is 110 | // recommended that implementors use the default implementation 111 | // provided by DefaultExternalMetricsProvider. 112 | ListAllExternalMetrics() []ExternalMetricInfo 113 | } 114 | 115 | type MetricsProvider interface { 116 | CustomMetricsProvider 117 | ExternalMetricsProvider 118 | } 119 | -------------------------------------------------------------------------------- /pkg/provider/interfaces_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "testing" 21 | 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | 25 | corev1 "k8s.io/api/core/v1" 26 | apimeta "k8s.io/apimachinery/pkg/api/meta" 27 | "k8s.io/apimachinery/pkg/runtime/schema" 28 | ) 29 | 30 | // restMapper creates a RESTMapper with just the types we need for 31 | // these tests. 32 | func restMapper() apimeta.RESTMapper { 33 | mapper := apimeta.NewDefaultRESTMapper([]schema.GroupVersion{corev1.SchemeGroupVersion}) 34 | mapper.Add(corev1.SchemeGroupVersion.WithKind("Pod"), apimeta.RESTScopeNamespace) 35 | 36 | return mapper 37 | } 38 | 39 | func TestNormalizeMetricInfoProducesSingularForm(t *testing.T) { 40 | pluralInfo := CustomMetricInfo{ 41 | GroupResource: schema.GroupResource{Resource: "pods"}, 42 | Namespaced: true, 43 | Metric: "cpu_usage", 44 | } 45 | 46 | _, singularRes, err := pluralInfo.Normalized(restMapper()) 47 | require.NoError(t, err, "should not have returned an error while normalizing the plural MetricInfo") 48 | assert.Equal(t, "pod", singularRes, "should have produced a singular resource from the pural metric info") 49 | } 50 | 51 | func TestNormalizeMetricInfoDealsWithPluralization(t *testing.T) { 52 | singularInfo := CustomMetricInfo{ 53 | GroupResource: schema.GroupResource{Resource: "pod"}, 54 | Namespaced: true, 55 | Metric: "cpu_usage", 56 | } 57 | 58 | pluralInfo := CustomMetricInfo{ 59 | GroupResource: schema.GroupResource{Resource: "pods"}, 60 | Namespaced: true, 61 | Metric: "cpu_usage", 62 | } 63 | 64 | singularNormalized, singularRes, err := singularInfo.Normalized(restMapper()) 65 | require.NoError(t, err, "should not have returned an error while normalizing the singular MetricInfo") 66 | pluralNormalized, pluralSingularRes, err := pluralInfo.Normalized(restMapper()) 67 | require.NoError(t, err, "should not have returned an error while normalizing the plural MetricInfo") 68 | 69 | assert.Equal(t, singularRes, pluralSingularRes, "the plural and singular MetricInfo should have the same singularized resource") 70 | assert.Equal(t, singularNormalized, pluralNormalized, "the plural and singular MetricInfo should have the same normailzed form") 71 | } 72 | -------------------------------------------------------------------------------- /pkg/provider/resource_lister.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 21 | "k8s.io/apiserver/pkg/endpoints/discovery" 22 | ) 23 | 24 | type customMetricsResourceLister struct { 25 | provider CustomMetricsProvider 26 | } 27 | 28 | type externalMetricsResourceLister struct { 29 | provider ExternalMetricsProvider 30 | } 31 | 32 | // NewCustomMetricResourceLister creates APIResourceLister for provided CustomMetricsProvider. 33 | func NewCustomMetricResourceLister(provider CustomMetricsProvider) discovery.APIResourceLister { 34 | return &customMetricsResourceLister{ 35 | provider: provider, 36 | } 37 | } 38 | 39 | func (l *customMetricsResourceLister) ListAPIResources() []metav1.APIResource { 40 | metrics := l.provider.ListAllMetrics() 41 | resources := make([]metav1.APIResource, len(metrics)) 42 | 43 | for i, metric := range metrics { 44 | resources[i] = metav1.APIResource{ 45 | Name: metric.GroupResource.String() + "/" + metric.Metric, 46 | Namespaced: metric.Namespaced, 47 | Kind: "MetricValueList", 48 | Verbs: metav1.Verbs{"get"}, // TODO: support "watch" 49 | } 50 | } 51 | 52 | return resources 53 | } 54 | 55 | // NewExternalMetricResourceLister creates APIResourceLister for provided CustomMetricsProvider. 56 | func NewExternalMetricResourceLister(provider ExternalMetricsProvider) discovery.APIResourceLister { 57 | return &externalMetricsResourceLister{ 58 | provider: provider, 59 | } 60 | } 61 | 62 | // ListAPIResources lists all supported custom metrics. 63 | func (l *externalMetricsResourceLister) ListAPIResources() []metav1.APIResource { 64 | metrics := l.provider.ListAllExternalMetrics() 65 | resources := make([]metav1.APIResource, len(metrics)) 66 | 67 | for i, metric := range metrics { 68 | resources[i] = metav1.APIResource{ 69 | Name: metric.Metric, 70 | Namespaced: true, 71 | Kind: "ExternalMetricValueList", 72 | Verbs: metav1.Verbs{"get"}, 73 | } 74 | } 75 | 76 | return resources 77 | } 78 | -------------------------------------------------------------------------------- /pkg/registry/custom_metrics/reststorage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apiserver 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apimachinery/pkg/runtime/schema" 27 | "k8s.io/apimachinery/pkg/types" 28 | "k8s.io/apiserver/pkg/endpoints/request" 29 | "k8s.io/apiserver/pkg/registry/rest" 30 | "k8s.io/metrics/pkg/apis/custom_metrics" 31 | 32 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/metrics" 33 | cm_rest "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/registry/rest" 34 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 35 | ) 36 | 37 | type REST struct { 38 | cmProvider provider.CustomMetricsProvider 39 | freshnessObserver metrics.FreshnessObserver 40 | } 41 | 42 | var _ rest.Storage = &REST{} 43 | var _ cm_rest.ListerWithOptions = &REST{} 44 | 45 | func NewREST(cmProvider provider.CustomMetricsProvider) *REST { 46 | freshnessObserver := metrics.NewFreshnessObserver(custom_metrics.GroupName) 47 | return &REST{ 48 | cmProvider: cmProvider, 49 | freshnessObserver: freshnessObserver, 50 | } 51 | } 52 | 53 | // Implement Storage 54 | 55 | func (r *REST) New() runtime.Object { 56 | return &custom_metrics.MetricValue{} 57 | } 58 | 59 | func (r *REST) Destroy() { 60 | } 61 | 62 | // Implement ListerWithOptions 63 | 64 | func (r *REST) NewList() runtime.Object { 65 | return &custom_metrics.MetricValueList{} 66 | } 67 | 68 | func (r *REST) NewListOptions() (runtime.Object, bool, string) { 69 | return &custom_metrics.MetricListOptions{}, true, "metricName" 70 | } 71 | 72 | func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions, metricOpts runtime.Object) (runtime.Object, error) { 73 | metricOptions, ok := metricOpts.(*custom_metrics.MetricListOptions) 74 | if !ok { 75 | return nil, fmt.Errorf("invalid options object: %#v", options) 76 | } 77 | 78 | // populate the label selector, defaulting to all 79 | selector := labels.Everything() 80 | if options != nil && options.LabelSelector != nil { 81 | selector = options.LabelSelector 82 | } 83 | 84 | metricLabelSelector := labels.Everything() 85 | if metricOptions != nil && len(metricOptions.MetricLabelSelector) > 0 { 86 | sel, err := labels.Parse(metricOptions.MetricLabelSelector) 87 | if err != nil { 88 | return nil, err 89 | } 90 | metricLabelSelector = sel 91 | } 92 | 93 | // grab the name, if present, from the field selector list options 94 | // (this is how the list handler logic injects it) 95 | // (otherwise we'd have to write a custom list handler) 96 | name := "*" 97 | if options != nil && options.FieldSelector != nil { 98 | if nameMatch, required := options.FieldSelector.RequiresExactMatch("metadata.name"); required { 99 | name = nameMatch 100 | } 101 | } 102 | 103 | namespace := request.NamespaceValue(ctx) 104 | 105 | requestInfo, ok := request.RequestInfoFrom(ctx) 106 | if !ok { 107 | return nil, fmt.Errorf("unable to get resource and metric name from request") 108 | } 109 | 110 | resourceRaw := requestInfo.Resource 111 | metricName := requestInfo.Subresource 112 | 113 | groupResource := schema.ParseGroupResource(resourceRaw) 114 | 115 | var res *custom_metrics.MetricValueList 116 | var err error 117 | 118 | // handle namespaced and root metrics 119 | if name == "*" { 120 | res, err = r.handleWildcardOp(ctx, namespace, groupResource, selector, metricName, metricLabelSelector) 121 | } else { 122 | res, err = r.handleIndividualOp(ctx, namespace, groupResource, name, metricName, metricLabelSelector) 123 | } 124 | 125 | if err != nil { 126 | return nil, err 127 | } 128 | 129 | for _, m := range res.Items { 130 | r.freshnessObserver.Observe(m.Timestamp) 131 | } 132 | 133 | return res, nil 134 | } 135 | 136 | func (r *REST) handleIndividualOp(ctx context.Context, namespace string, groupResource schema.GroupResource, name string, metricName string, metricLabelSelector labels.Selector) (*custom_metrics.MetricValueList, error) { 137 | singleRes, err := r.cmProvider.GetMetricByName(ctx, types.NamespacedName{Namespace: namespace, Name: name}, provider.CustomMetricInfo{ 138 | GroupResource: groupResource, 139 | Metric: metricName, 140 | Namespaced: namespace != "", 141 | }, metricLabelSelector) 142 | if err != nil { 143 | return nil, err 144 | } 145 | 146 | return &custom_metrics.MetricValueList{ 147 | Items: []custom_metrics.MetricValue{*singleRes}, 148 | }, nil 149 | } 150 | 151 | func (r *REST) handleWildcardOp(ctx context.Context, namespace string, groupResource schema.GroupResource, selector labels.Selector, metricName string, metricLabelSelector labels.Selector) (*custom_metrics.MetricValueList, error) { 152 | return r.cmProvider.GetMetricBySelector(ctx, namespace, selector, provider.CustomMetricInfo{ 153 | GroupResource: groupResource, 154 | Metric: metricName, 155 | Namespaced: namespace != "", 156 | }, metricLabelSelector) 157 | } 158 | -------------------------------------------------------------------------------- /pkg/registry/external_metrics/reststorage.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package apiserver 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | 23 | metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" 24 | "k8s.io/apimachinery/pkg/labels" 25 | "k8s.io/apimachinery/pkg/runtime" 26 | "k8s.io/apiserver/pkg/endpoints/request" 27 | "k8s.io/apiserver/pkg/registry/rest" 28 | "k8s.io/metrics/pkg/apis/external_metrics" 29 | 30 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/metrics" 31 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 32 | ) 33 | 34 | // REST is a wrapper for CustomMetricsProvider that provides implementation for Storage and Lister 35 | // interfaces. 36 | type REST struct { 37 | emProvider provider.ExternalMetricsProvider 38 | freshnessObserver metrics.FreshnessObserver 39 | rest.TableConvertor 40 | } 41 | 42 | var _ rest.Storage = &REST{} 43 | var _ rest.Lister = &REST{} 44 | 45 | // NewREST returns new REST object for provided CustomMetricsProvider. 46 | func NewREST(emProvider provider.ExternalMetricsProvider) *REST { 47 | freshnessObserver := metrics.NewFreshnessObserver(external_metrics.GroupName) 48 | return &REST{ 49 | emProvider: emProvider, 50 | freshnessObserver: freshnessObserver, 51 | } 52 | } 53 | 54 | // Implement Storage 55 | 56 | // New returns empty MetricValue. 57 | func (r *REST) New() runtime.Object { 58 | return &external_metrics.ExternalMetricValue{} 59 | } 60 | 61 | func (r *REST) Destroy() { 62 | } 63 | 64 | // Implement Lister 65 | 66 | // NewList returns empty MetricValueList. 67 | func (r *REST) NewList() runtime.Object { 68 | return &external_metrics.ExternalMetricValueList{} 69 | } 70 | 71 | // List selects resources in the storage which match to the selector. 72 | func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { 73 | // populate the label selector, defaulting to all 74 | metricSelector := labels.Everything() 75 | if options != nil && options.LabelSelector != nil { 76 | metricSelector = options.LabelSelector 77 | } 78 | 79 | namespace := request.NamespaceValue(ctx) 80 | 81 | requestInfo, ok := request.RequestInfoFrom(ctx) 82 | if !ok { 83 | return nil, fmt.Errorf("unable to get resource and metric name from request") 84 | } 85 | metricName := requestInfo.Resource 86 | 87 | res, err := r.emProvider.GetExternalMetric(ctx, namespace, metricSelector, provider.ExternalMetricInfo{Metric: metricName}) 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | for _, m := range res.Items { 93 | r.freshnessObserver.Observe(m.Timestamp) 94 | } 95 | 96 | return res, nil 97 | } 98 | -------------------------------------------------------------------------------- /test-adapter-deploy/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM BASEIMAGE 2 | COPY adapter / 3 | ENTRYPOINT ["/adapter"] 4 | -------------------------------------------------------------------------------- /test-adapter-deploy/README.md: -------------------------------------------------------------------------------- 1 | # Sample Deployment Files 2 | 3 | These files can be used to deploy the sample adapter container. You can 4 | build that with `make test-adapter-container`. The Dockerfile describes the 5 | container itself, while the [manifest](./testing-adapter.yaml) can be used 6 | to deploy that container as a provider of the custom metrics and external 7 | metrics APIs on the cluster. 8 | -------------------------------------------------------------------------------- /test-adapter-deploy/testing-adapter.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: custom-metrics 5 | --- 6 | apiVersion: rbac.authorization.k8s.io/v1 7 | kind: ClusterRoleBinding 8 | metadata: 9 | name: custom-metrics:system:auth-delegator 10 | roleRef: 11 | apiGroup: rbac.authorization.k8s.io 12 | kind: ClusterRole 13 | name: system:auth-delegator 14 | subjects: 15 | - kind: ServiceAccount 16 | name: custom-metrics-apiserver 17 | namespace: custom-metrics 18 | --- 19 | apiVersion: rbac.authorization.k8s.io/v1 20 | kind: RoleBinding 21 | metadata: 22 | name: custom-metrics-auth-reader 23 | namespace: kube-system 24 | roleRef: 25 | apiGroup: rbac.authorization.k8s.io 26 | kind: Role 27 | name: extension-apiserver-authentication-reader 28 | subjects: 29 | - kind: ServiceAccount 30 | name: custom-metrics-apiserver 31 | namespace: custom-metrics 32 | --- 33 | apiVersion: apps/v1 34 | kind: Deployment 35 | metadata: 36 | labels: 37 | app: custom-metrics-apiserver 38 | name: custom-metrics-apiserver 39 | namespace: custom-metrics 40 | spec: 41 | replicas: 1 42 | selector: 43 | matchLabels: 44 | app: custom-metrics-apiserver 45 | template: 46 | metadata: 47 | labels: 48 | app: custom-metrics-apiserver 49 | name: custom-metrics-apiserver 50 | spec: 51 | serviceAccountName: custom-metrics-apiserver 52 | containers: 53 | - name: custom-metrics-apiserver 54 | image: REGISTRY/k8s-test-metrics-adapter-amd64:latest 55 | imagePullPolicy: IfNotPresent 56 | args: 57 | - --secure-port=6443 58 | - --cert-dir=/var/run/serving-cert 59 | - --v=10 60 | ports: 61 | - containerPort: 6443 62 | name: https 63 | - containerPort: 8080 64 | name: http 65 | volumeMounts: 66 | - mountPath: /tmp 67 | name: temp-vol 68 | readOnly: false 69 | - mountPath: /var/run/serving-cert 70 | name: volume-serving-cert 71 | readOnly: false 72 | volumes: 73 | - name: temp-vol 74 | emptyDir: {} 75 | - name: volume-serving-cert 76 | emptyDir: {} 77 | --- 78 | apiVersion: rbac.authorization.k8s.io/v1 79 | kind: ClusterRoleBinding 80 | metadata: 81 | name: custom-metrics-resource-reader 82 | roleRef: 83 | apiGroup: rbac.authorization.k8s.io 84 | kind: ClusterRole 85 | name: custom-metrics-resource-reader 86 | subjects: 87 | - kind: ServiceAccount 88 | name: custom-metrics-apiserver 89 | namespace: custom-metrics 90 | --- 91 | kind: ServiceAccount 92 | apiVersion: v1 93 | metadata: 94 | name: custom-metrics-apiserver 95 | namespace: custom-metrics 96 | --- 97 | apiVersion: v1 98 | kind: Service 99 | metadata: 100 | name: custom-metrics-apiserver 101 | namespace: custom-metrics 102 | spec: 103 | ports: 104 | - name: https 105 | port: 443 106 | targetPort: 6443 107 | - name: http 108 | port: 80 109 | targetPort: 8080 110 | selector: 111 | app: custom-metrics-apiserver 112 | --- 113 | apiVersion: apiregistration.k8s.io/v1 114 | kind: APIService 115 | metadata: 116 | name: v1beta1.custom.metrics.k8s.io 117 | spec: 118 | service: 119 | name: custom-metrics-apiserver 120 | namespace: custom-metrics 121 | group: custom.metrics.k8s.io 122 | version: v1beta1 123 | insecureSkipTLSVerify: true 124 | groupPriorityMinimum: 100 125 | versionPriority: 100 126 | --- 127 | apiVersion: apiregistration.k8s.io/v1 128 | kind: APIService 129 | metadata: 130 | name: v1beta2.custom.metrics.k8s.io 131 | spec: 132 | service: 133 | name: custom-metrics-apiserver 134 | namespace: custom-metrics 135 | group: custom.metrics.k8s.io 136 | version: v1beta2 137 | insecureSkipTLSVerify: true 138 | groupPriorityMinimum: 100 139 | versionPriority: 200 140 | --- 141 | apiVersion: apiregistration.k8s.io/v1 142 | kind: APIService 143 | metadata: 144 | name: v1beta1.external.metrics.k8s.io 145 | spec: 146 | service: 147 | name: custom-metrics-apiserver 148 | namespace: custom-metrics 149 | group: external.metrics.k8s.io 150 | version: v1beta1 151 | insecureSkipTLSVerify: true 152 | groupPriorityMinimum: 100 153 | versionPriority: 100 154 | --- 155 | apiVersion: rbac.authorization.k8s.io/v1 156 | kind: ClusterRole 157 | metadata: 158 | name: custom-metrics-server-resources 159 | rules: 160 | - apiGroups: 161 | - custom.metrics.k8s.io 162 | resources: ["*"] 163 | verbs: ["*"] 164 | --- 165 | apiVersion: rbac.authorization.k8s.io/v1 166 | kind: ClusterRole 167 | metadata: 168 | name: custom-metrics-resource-reader 169 | rules: 170 | - apiGroups: 171 | - "" 172 | resources: 173 | - namespaces 174 | - pods 175 | - services 176 | verbs: 177 | - get 178 | - list 179 | --- 180 | apiVersion: rbac.authorization.k8s.io/v1 181 | kind: ClusterRoleBinding 182 | metadata: 183 | name: hpa-controller-custom-metrics 184 | roleRef: 185 | apiGroup: rbac.authorization.k8s.io 186 | kind: ClusterRole 187 | name: custom-metrics-server-resources 188 | subjects: 189 | - kind: ServiceAccount 190 | name: horizontal-pod-autoscaler 191 | namespace: kube-system 192 | -------------------------------------------------------------------------------- /test-adapter/README.md: -------------------------------------------------------------------------------- 1 | # Sample API Server 2 | 3 | This is a sample of how to write a custom metrics API server using this 4 | library. See the [README](/README.md) and [getting started 5 | guide](/docs/getting-started.md) for more information. 6 | 7 | The main entrypoint lives in [main.go](main.go), while the provider 8 | implementations live in [provider/provider.go](provider/provider.go). 9 | -------------------------------------------------------------------------------- /test-adapter/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "context" 21 | "net/http" 22 | "os" 23 | "time" 24 | 25 | "github.com/emicklei/go-restful/v3" 26 | "k8s.io/component-base/logs" 27 | "k8s.io/component-base/metrics/legacyregistry" 28 | "k8s.io/klog/v2" 29 | 30 | "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver/metrics" 31 | basecmd "sigs.k8s.io/custom-metrics-apiserver/pkg/cmd" 32 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 33 | fakeprov "sigs.k8s.io/custom-metrics-apiserver/test-adapter/provider" 34 | ) 35 | 36 | type SampleAdapter struct { 37 | basecmd.AdapterBase 38 | 39 | // Message is printed on successful startup 40 | Message string 41 | } 42 | 43 | func (a *SampleAdapter) makeProviderOrDie() (provider.MetricsProvider, *restful.WebService) { 44 | client, err := a.DynamicClient() 45 | if err != nil { 46 | klog.Fatalf("unable to construct dynamic client: %v", err) 47 | } 48 | 49 | mapper, err := a.RESTMapper() 50 | if err != nil { 51 | klog.Fatalf("unable to construct discovery REST mapper: %v", err) 52 | } 53 | 54 | return fakeprov.NewFakeProvider(client, mapper) 55 | } 56 | 57 | func main() { 58 | logs.InitLogs() 59 | defer logs.FlushLogs() 60 | 61 | cmd := &SampleAdapter{} 62 | cmd.Name = "test-adapter" 63 | 64 | cmd.Flags().StringVar(&cmd.Message, "msg", "starting adapter...", "startup message") 65 | logs.AddFlags(cmd.Flags()) 66 | if err := cmd.Flags().Parse(os.Args); err != nil { 67 | klog.Fatalf("unable to parse flags: %v", err) 68 | } 69 | 70 | testProvider, webService := cmd.makeProviderOrDie() 71 | cmd.WithCustomMetrics(testProvider) 72 | cmd.WithExternalMetrics(testProvider) 73 | 74 | if err := metrics.RegisterMetrics(legacyregistry.Register); err != nil { 75 | klog.Fatalf("unable to register metrics: %v", err) 76 | } 77 | 78 | klog.Infof("%s", cmd.Message) 79 | // Set up POST endpoint for writing fake metric values 80 | restful.DefaultContainer.Add(webService) 81 | go func() { 82 | // Open port for POSTing fake metrics 83 | server := &http.Server{ 84 | Addr: ":8080", 85 | ReadHeaderTimeout: 3 * time.Second, 86 | } 87 | klog.Fatal(server.ListenAndServe()) 88 | }() 89 | if err := cmd.Run(context.Background()); err != nil { 90 | klog.Fatalf("unable to run custom metrics adapter: %v", err) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /test-adapter/provider/provider.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2017 The Kubernetes Authors. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package provider 18 | 19 | import ( 20 | "context" 21 | "net/http" 22 | "sync" 23 | 24 | "github.com/emicklei/go-restful/v3" 25 | apierr "k8s.io/apimachinery/pkg/api/errors" 26 | apimeta "k8s.io/apimachinery/pkg/api/meta" 27 | "k8s.io/apimachinery/pkg/api/resource" 28 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 29 | "k8s.io/apimachinery/pkg/labels" 30 | "k8s.io/apimachinery/pkg/runtime/schema" 31 | "k8s.io/apimachinery/pkg/types" 32 | "k8s.io/client-go/dynamic" 33 | "k8s.io/klog/v2" 34 | "k8s.io/metrics/pkg/apis/custom_metrics" 35 | "k8s.io/metrics/pkg/apis/external_metrics" 36 | 37 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider" 38 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/defaults" 39 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/helpers" 40 | ) 41 | 42 | // CustomMetricResource wraps provider.CustomMetricInfo in a struct which stores the Name and Namespace of the resource 43 | // So that we can accurately store and retrieve the metric as if this were an actual metrics server. 44 | type CustomMetricResource struct { 45 | provider.CustomMetricInfo 46 | types.NamespacedName 47 | } 48 | 49 | // externalMetric provides examples for metrics which would otherwise be reported from an external source 50 | // TODO (damemi): add dynamic external metrics instead of just hardcoded examples 51 | type externalMetric struct { 52 | info provider.ExternalMetricInfo 53 | labels map[string]string 54 | value external_metrics.ExternalMetricValue 55 | } 56 | 57 | var ( 58 | testingExternalMetrics = []externalMetric{ 59 | { 60 | info: provider.ExternalMetricInfo{ 61 | Metric: "my-external-metric", 62 | }, 63 | labels: map[string]string{"foo": "bar"}, 64 | value: external_metrics.ExternalMetricValue{ 65 | MetricName: "my-external-metric", 66 | MetricLabels: map[string]string{ 67 | "foo": "bar", 68 | }, 69 | Value: *resource.NewQuantity(42, resource.DecimalSI), 70 | }, 71 | }, 72 | { 73 | info: provider.ExternalMetricInfo{ 74 | Metric: "my-external-metric", 75 | }, 76 | labels: map[string]string{"foo": "baz"}, 77 | value: external_metrics.ExternalMetricValue{ 78 | MetricName: "my-external-metric", 79 | MetricLabels: map[string]string{ 80 | "foo": "baz", 81 | }, 82 | Value: *resource.NewQuantity(43, resource.DecimalSI), 83 | }, 84 | }, 85 | { 86 | info: provider.ExternalMetricInfo{ 87 | Metric: "other-external-metric", 88 | }, 89 | labels: map[string]string{}, 90 | value: external_metrics.ExternalMetricValue{ 91 | MetricName: "other-external-metric", 92 | MetricLabels: map[string]string{}, 93 | Value: *resource.NewQuantity(44, resource.DecimalSI), 94 | }, 95 | }, 96 | } 97 | ) 98 | 99 | type metricValue struct { 100 | labels labels.Set 101 | value resource.Quantity 102 | timestamp metav1.Time 103 | } 104 | 105 | var _ provider.MetricsProvider = &testingProvider{} 106 | 107 | // testingProvider is a sample implementation of provider.MetricsProvider which stores a map of fake metrics 108 | type testingProvider struct { 109 | defaults.DefaultCustomMetricsProvider 110 | defaults.DefaultExternalMetricsProvider 111 | client dynamic.Interface 112 | mapper apimeta.RESTMapper 113 | 114 | valuesLock sync.RWMutex 115 | values map[CustomMetricResource]metricValue 116 | externalMetrics []externalMetric 117 | } 118 | 119 | // NewFakeProvider returns an instance of testingProvider, along with its restful.WebService that opens endpoints to post new fake metrics 120 | func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper) (provider.MetricsProvider, *restful.WebService) { 121 | provider := &testingProvider{ 122 | client: client, 123 | mapper: mapper, 124 | values: make(map[CustomMetricResource]metricValue), 125 | externalMetrics: testingExternalMetrics, 126 | } 127 | return provider, provider.webService() 128 | } 129 | 130 | // webService creates a restful.WebService with routes set up for receiving fake metrics 131 | // These writing routes have been set up to be identical to the format of routes which metrics are read from. 132 | // There are 3 metric types available: namespaced, root-scoped, and namespaces. 133 | // (Note: Namespaces, we're assuming, are themselves namespaced resources, but for consistency with how metrics are retreived they have a separate route) 134 | func (p *testingProvider) webService() *restful.WebService { 135 | ws := new(restful.WebService) 136 | 137 | ws.Path("/write-metrics") 138 | 139 | // Namespaced resources 140 | ws.Route(ws.POST("/namespaces/{namespace}/{resourceType}/{name}/{metric}").To(p.updateMetric). 141 | Param(ws.BodyParameter("value", "value to set metric").DataType("integer").DefaultValue("0"))) 142 | 143 | // Root-scoped resources 144 | ws.Route(ws.POST("/{resourceType}/{name}/{metric}").To(p.updateMetric). 145 | Param(ws.BodyParameter("value", "value to set metric").DataType("integer").DefaultValue("0"))) 146 | 147 | // Namespaces, where {resourceType} == "namespaces" to match API 148 | ws.Route(ws.POST("/{resourceType}/{name}/metrics/{metric}").To(p.updateMetric). 149 | Param(ws.BodyParameter("value", "value to set metric").DataType("integer").DefaultValue("0"))) 150 | return ws 151 | } 152 | 153 | // updateMetric writes the metric provided by a restful request and stores it in memory 154 | func (p *testingProvider) updateMetric(request *restful.Request, response *restful.Response) { 155 | p.valuesLock.Lock() 156 | defer p.valuesLock.Unlock() 157 | 158 | namespace := request.PathParameter("namespace") 159 | resourceType := request.PathParameter("resourceType") 160 | namespaced := false 161 | if len(namespace) > 0 || resourceType == "namespaces" { 162 | namespaced = true 163 | } 164 | name := request.PathParameter("name") 165 | metricName := request.PathParameter("metric") 166 | 167 | value := new(resource.Quantity) 168 | err := request.ReadEntity(value) 169 | if err != nil { 170 | if err := response.WriteErrorString(http.StatusBadRequest, err.Error()); err != nil { 171 | klog.Errorf("Error writing error: %s", err) 172 | } 173 | return 174 | } 175 | 176 | groupResource := schema.ParseGroupResource(resourceType) 177 | 178 | metricLabels := labels.Set{} 179 | sel := request.QueryParameter("labels") 180 | if len(sel) > 0 { 181 | metricLabels, err = labels.ConvertSelectorToLabelsMap(sel) 182 | if err != nil { 183 | if err := response.WriteErrorString(http.StatusBadRequest, err.Error()); err != nil { 184 | klog.Errorf("Error writing error: %s", err) 185 | } 186 | return 187 | } 188 | } 189 | 190 | info := provider.CustomMetricInfo{ 191 | GroupResource: groupResource, 192 | Metric: metricName, 193 | Namespaced: namespaced, 194 | } 195 | 196 | info, _, err = info.Normalized(p.mapper) 197 | if err != nil { 198 | klog.Errorf("Error normalizing info: %s", err) 199 | } 200 | namespacedName := types.NamespacedName{ 201 | Name: name, 202 | Namespace: namespace, 203 | } 204 | 205 | metricInfo := CustomMetricResource{ 206 | CustomMetricInfo: info, 207 | NamespacedName: namespacedName, 208 | } 209 | p.values[metricInfo] = metricValue{ 210 | labels: metricLabels, 211 | value: *value, 212 | timestamp: metav1.Now(), 213 | } 214 | } 215 | 216 | // valueFor is a helper function to get just the value of a specific metric 217 | func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.NamespacedName, metricSelector labels.Selector) (metricValue, error) { 218 | info, _, err := info.Normalized(p.mapper) 219 | if err != nil { 220 | return metricValue{}, err 221 | } 222 | metricInfo := CustomMetricResource{ 223 | CustomMetricInfo: info, 224 | NamespacedName: name, 225 | } 226 | 227 | value, found := p.values[metricInfo] 228 | if !found { 229 | return metricValue{}, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name) 230 | } 231 | 232 | if !metricSelector.Matches(value.labels) { 233 | return metricValue{}, provider.NewMetricNotFoundForSelectorError(info.GroupResource, info.Metric, name.Name, metricSelector) 234 | } 235 | 236 | return value, nil 237 | } 238 | 239 | // metricFor is a helper function which formats a value, metric, and object info into a MetricValue which can be returned by the metrics API 240 | func (p *testingProvider) metricFor(value metricValue, name types.NamespacedName, _ labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) { 241 | objRef, err := helpers.ReferenceFor(p.mapper, name, info) 242 | if err != nil { 243 | return nil, err 244 | } 245 | 246 | metric := &custom_metrics.MetricValue{ 247 | DescribedObject: objRef, 248 | Metric: custom_metrics.MetricIdentifier{ 249 | Name: info.Metric, 250 | }, 251 | Timestamp: value.timestamp, 252 | Value: value.value, 253 | } 254 | 255 | if len(metricSelector.String()) > 0 { 256 | sel, err := metav1.ParseToLabelSelector(metricSelector.String()) 257 | if err != nil { 258 | return nil, err 259 | } 260 | metric.Metric.Selector = sel 261 | } 262 | 263 | return metric, nil 264 | } 265 | 266 | // metricsFor is a wrapper used by GetMetricBySelector to format several metrics which match a resource selector 267 | func (p *testingProvider) metricsFor(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) { 268 | names, err := helpers.ListObjectNames(p.mapper, p.client, namespace, selector, info) 269 | if err != nil { 270 | return nil, err 271 | } 272 | 273 | res := make([]custom_metrics.MetricValue, 0, len(names)) 274 | for _, name := range names { 275 | namespacedName := types.NamespacedName{Name: name, Namespace: namespace} 276 | value, err := p.valueFor(info, namespacedName, metricSelector) 277 | if err != nil { 278 | if apierr.IsNotFound(err) { 279 | continue 280 | } 281 | return nil, err 282 | } 283 | 284 | metric, err := p.metricFor(value, namespacedName, selector, info, metricSelector) 285 | if err != nil { 286 | return nil, err 287 | } 288 | res = append(res, *metric) 289 | } 290 | 291 | return &custom_metrics.MetricValueList{ 292 | Items: res, 293 | }, nil 294 | } 295 | 296 | func (p *testingProvider) GetMetricByName(_ context.Context, name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) { 297 | p.valuesLock.RLock() 298 | defer p.valuesLock.RUnlock() 299 | 300 | value, err := p.valueFor(info, name, metricSelector) 301 | if err != nil { 302 | return nil, err 303 | } 304 | return p.metricFor(value, name, labels.Everything(), info, metricSelector) 305 | } 306 | 307 | func (p *testingProvider) GetMetricBySelector(_ context.Context, namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) { 308 | p.valuesLock.RLock() 309 | defer p.valuesLock.RUnlock() 310 | 311 | return p.metricsFor(namespace, selector, info, metricSelector) 312 | } 313 | 314 | func (p *testingProvider) GetExternalMetric(_ context.Context, _ string, metricSelector labels.Selector, info provider.ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) { 315 | p.valuesLock.RLock() 316 | defer p.valuesLock.RUnlock() 317 | 318 | matchingMetrics := []external_metrics.ExternalMetricValue{} 319 | for _, metric := range p.externalMetrics { 320 | if metric.info.Metric == info.Metric && 321 | metricSelector.Matches(labels.Set(metric.labels)) { 322 | metricValue := metric.value 323 | metricValue.Timestamp = metav1.Now() 324 | matchingMetrics = append(matchingMetrics, metricValue) 325 | } 326 | } 327 | return &external_metrics.ExternalMetricValueList{ 328 | Items: matchingMetrics, 329 | }, nil 330 | } 331 | --------------------------------------------------------------------------------