├── .github
└── ISSUE_TEMPLATE
│ └── bug_report.md
├── .gitignore
├── .golangci.yml
├── CONTRIBUTING.md
├── Dockerfile
├── LICENSE
├── Makefile
├── NOTICE
├── OWNERS
├── README.md
├── RELEASE.md
├── SECURITY.md
├── SECURITY_CONTACTS
├── VERSION
├── cloudbuild.yaml
├── cmd
├── adapter
│ ├── adapter.go
│ ├── adapter_test.go
│ └── testdata
│ │ ├── ca-error.pem
│ │ ├── ca.pem
│ │ ├── kubeconfig
│ │ ├── kubeconfig-error
│ │ ├── tlscert-error.crt
│ │ ├── tlscert.crt
│ │ └── tlskey.key
└── config-gen
│ ├── main.go
│ └── utils
│ └── default.go
├── code-of-conduct.md
├── deploy
├── README.md
└── manifests
│ ├── api-service.yaml
│ ├── cluster-role-aggregated-metrics-reader.yaml
│ ├── cluster-role-binding-delegator.yaml
│ ├── cluster-role-binding-hpa-custom-metrics.yaml
│ ├── cluster-role-binding.yaml
│ ├── cluster-role-metrics-server-resources.yaml
│ ├── cluster-role.yaml
│ ├── config-map.yaml
│ ├── deployment.yaml
│ ├── network-policy.yaml
│ ├── pod-disruption-budget.yaml
│ ├── role-binding-auth-reader.yaml
│ ├── service-account.yaml
│ └── service.yaml
├── docs
├── config-walkthrough.md
├── config.md
├── externalmetrics.md
├── sample-config.yaml
└── walkthrough.md
├── go.mod
├── go.sum
├── hack
├── boilerplate.go.txt
└── tools.go
├── pkg
├── api
│ ├── generated
│ │ └── openapi
│ │ │ └── zz_generated.openapi.go
│ └── openapi-gen.go
├── client
│ ├── api.go
│ ├── fake
│ │ └── client.go
│ ├── helpers.go
│ ├── interfaces.go
│ ├── metrics
│ │ └── metrics.go
│ └── types.go
├── config
│ ├── config.go
│ └── loader.go
├── custom-provider
│ ├── provider.go
│ ├── provider_suite_test.go
│ ├── provider_test.go
│ ├── series_registry.go
│ └── series_registry_test.go
├── external-provider
│ ├── basic_metric_lister.go
│ ├── external_series_registry.go
│ ├── metric_converter.go
│ ├── periodic_metric_lister.go
│ ├── periodic_metric_lister_test.go
│ └── provider.go
├── naming
│ ├── errors.go
│ ├── lbl_res.go
│ ├── metric_namer.go
│ ├── metrics_query.go
│ ├── metrics_query_test.go
│ ├── regex_matcher_test.go
│ └── resource_converter.go
└── resourceprovider
│ ├── provider.go
│ ├── provider_suite_test.go
│ └── provider_test.go
└── test
├── README.md
├── e2e
└── e2e_test.go
├── prometheus-manifests
├── cluster-role-binding.yaml
├── cluster-role.yaml
├── prometheus.yaml
├── service-account.yaml
├── service-monitor-kubelet.yaml
└── service.yaml
└── run-e2e-tests.sh
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: Bug report
3 | about: Report a bug encountered while running prometheus-adapter
4 | title: ''
5 | labels: kind/bug
6 | assignees: ''
7 |
8 | ---
9 |
10 |
14 |
15 | **What happened?**:
16 |
17 | **What did you expect to happen?**:
18 |
19 | **Please provide the prometheus-adapter config**:
20 |
21 | prometheus-adapter config
22 |
23 |
24 |
25 |
26 |
27 | **Please provide the HPA resource used for autoscaling**:
28 |
29 | HPA yaml
30 |
31 |
32 |
33 |
34 |
35 | **Please provide the HPA status**:
36 |
37 | **Please provide the prometheus-adapter logs with -v=6 around the time the issue happened**:
38 |
39 | prometheus-adapter logs
40 |
41 |
42 |
43 |
44 |
45 | **Anything else we need to know?**:
46 |
47 | **Environment**:
48 | - prometheus-adapter version:
49 | - prometheus version:
50 | - Kubernetes version (use `kubectl version`):
51 | - Cloud provider or hardware configuration:
52 | - Other info:
53 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.swp
2 | *~
3 | /vendor
4 | /adapter
5 | .e2e
6 |
--------------------------------------------------------------------------------
/.golangci.yml:
--------------------------------------------------------------------------------
1 | run:
2 | deadline: 5m
3 |
4 | linters:
5 | disable-all: true
6 | enable:
7 | - bodyclose
8 | - dogsled
9 | - dupl
10 | - errcheck
11 | - exportloopref
12 | - gocritic
13 | - gocyclo
14 | - gofmt
15 | - goimports
16 | - gosec
17 | - goprintffuncname
18 | - gosimple
19 | - govet
20 | - ineffassign
21 | - misspell
22 | - nakedret
23 | - nolintlint
24 | - revive
25 | - staticcheck
26 | - stylecheck
27 | - typecheck
28 | - unconvert
29 | - unused
30 | - whitespace
31 |
32 | linters-settings:
33 | goimports:
34 | local-prefixes: sigs.k8s.io/prometheus-adapter
35 | revive:
36 | rules:
37 | - name: exported
38 | arguments:
39 | - disableStutteringCheck
40 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Contributing Guidelines
2 |
3 | Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://git.k8s.io/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt:
4 |
5 | _As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._
6 |
7 | ## Getting Started
8 |
9 | We have full documentation on how to get started contributing here:
10 |
11 |
14 |
15 | - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests
16 | - [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing)
17 | - [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet) - Common resources for existing developers
18 |
19 | ## Mentorship
20 |
21 | - [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers!
22 |
23 |
32 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | ARG ARCH
2 | ARG GO_VERSION
3 |
4 | FROM golang:${GO_VERSION} as build
5 |
6 | WORKDIR /go/src/sigs.k8s.io/prometheus-adapter
7 | COPY go.mod .
8 | COPY go.sum .
9 | RUN go mod download
10 |
11 | COPY pkg pkg
12 | COPY cmd cmd
13 | COPY Makefile Makefile
14 |
15 | ARG ARCH
16 | RUN make prometheus-adapter
17 |
18 | FROM gcr.io/distroless/static:latest-$ARCH
19 |
20 | COPY --from=build /go/src/sigs.k8s.io/prometheus-adapter/adapter /
21 | USER 65534
22 | ENTRYPOINT ["/adapter"]
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "{}"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright {yyyy} {name of copyright owner}
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | REGISTRY?=gcr.io/k8s-staging-prometheus-adapter
2 | IMAGE=prometheus-adapter
3 | ARCH?=$(shell go env GOARCH)
4 | ALL_ARCH=amd64 arm arm64 ppc64le s390x
5 | GOPATH:=$(shell go env GOPATH)
6 |
7 | VERSION=$(shell cat VERSION)
8 | TAG_PREFIX=v
9 | TAG?=$(TAG_PREFIX)$(VERSION)
10 |
11 | GO_VERSION?=1.22.5
12 | GOLANGCI_VERSION?=1.56.2
13 |
14 | .PHONY: all
15 | all: prometheus-adapter
16 |
17 | # Build
18 | # -----
19 |
20 | SRC_DEPS=$(shell find pkg cmd -type f -name "*.go")
21 |
22 | prometheus-adapter: $(SRC_DEPS)
23 | CGO_ENABLED=0 GOARCH=$(ARCH) go build sigs.k8s.io/prometheus-adapter/cmd/adapter
24 |
25 | .PHONY: container
26 | container:
27 | docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG) --build-arg ARCH=$(ARCH) --build-arg GO_VERSION=$(GO_VERSION) .
28 |
29 | # Container push
30 | # --------------
31 |
32 | PUSH_ARCH_TARGETS=$(addprefix push-,$(ALL_ARCH))
33 |
34 | .PHONY: push
35 | push: container
36 | docker push $(REGISTRY)/$(IMAGE)-$(ARCH):$(TAG)
37 |
38 | push-all: $(PUSH_ARCH_TARGETS) push-multi-arch;
39 |
40 | .PHONY: $(PUSH_ARCH_TARGETS)
41 | $(PUSH_ARCH_TARGETS): push-%:
42 | ARCH=$* $(MAKE) push
43 |
44 | .PHONY: push-multi-arch
45 | push-multi-arch: export DOCKER_CLI_EXPERIMENTAL = enabled
46 | push-multi-arch:
47 | docker manifest create --amend $(REGISTRY)/$(IMAGE):$(TAG) $(shell echo $(ALL_ARCH) | sed -e "s~[^ ]*~$(REGISTRY)/$(IMAGE)\-&:$(TAG)~g")
48 | @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} $(REGISTRY)/$(IMAGE):$(TAG) $(REGISTRY)/$(IMAGE)-$${arch}:$(TAG); done
49 | docker manifest push --purge $(REGISTRY)/$(IMAGE):$(TAG)
50 |
51 | # Test
52 | # ----
53 |
54 | .PHONY: test
55 | test:
56 | CGO_ENABLED=0 go test ./cmd/... ./pkg/...
57 |
58 | .PHONY: test-e2e
59 | test-e2e:
60 | ./test/run-e2e-tests.sh
61 |
62 |
63 | # Static analysis
64 | # ---------------
65 |
66 | .PHONY: verify
67 | verify: verify-lint verify-deps verify-generated
68 |
69 | .PHONY: update
70 | update: update-lint update-generated
71 |
72 | # Format and lint
73 | # ---------------
74 |
75 | HAS_GOLANGCI_VERSION:=$(shell $(GOPATH)/bin/golangci-lint version --format=short)
76 | .PHONY: golangci
77 | golangci:
78 | ifneq ($(HAS_GOLANGCI_VERSION), $(GOLANGCI_VERSION))
79 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v$(GOLANGCI_VERSION)
80 | endif
81 |
82 | .PHONY: verify-lint
83 | verify-lint: golangci
84 | $(GOPATH)/bin/golangci-lint run --modules-download-mode=readonly || (echo 'Run "make update-lint"' && exit 1)
85 |
86 | .PHONY: update-lint
87 | update-lint: golangci
88 | $(GOPATH)/bin/golangci-lint run --fix --modules-download-mode=readonly
89 |
90 |
91 | # Dependencies
92 | # ------------
93 |
94 | .PHONY: verify-deps
95 | verify-deps:
96 | go mod verify
97 | go mod tidy
98 | @git diff --exit-code -- go.mod go.sum
99 |
100 | # Generation
101 | # ----------
102 |
103 | generated_files=pkg/api/generated/openapi/zz_generated.openapi.go
104 |
105 | .PHONY: verify-generated
106 | verify-generated: update-generated
107 | @git diff --exit-code -- $(generated_files)
108 |
109 | .PHONY: update-generated
110 | update-generated:
111 | go install -mod=readonly k8s.io/kube-openapi/cmd/openapi-gen
112 | $(GOPATH)/bin/openapi-gen --logtostderr \
113 | --go-header-file ./hack/boilerplate.go.txt \
114 | --output-pkg ./pkg/api/generated/openapi \
115 | --output-file zz_generated.openapi.go \
116 | --output-dir ./pkg/api/generated/openapi \
117 | -r /dev/null \
118 | "k8s.io/metrics/pkg/apis/custom_metrics" "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1" "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2" "k8s.io/metrics/pkg/apis/external_metrics" "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" "k8s.io/metrics/pkg/apis/metrics" "k8s.io/metrics/pkg/apis/metrics/v1beta1" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/version" "k8s.io/api/core/v1"
119 |
--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
1 | When donating the k8s-prometheus-adapter project to the CNCF, we could not
2 | reach all the contributors to make them sign the CNCF CLA. As such, according
3 | to the CNCF rules to donate a repository, we must add a NOTICE referencing
4 | section 7 of the CLA with a list of developers who could not be reached.
5 |
6 | `7. Should You wish to submit work that is not Your original creation, You may
7 | submit it to the Foundation separately from any Contribution, identifying the
8 | complete details of its source and of any license or other restriction
9 | (including, but not limited to, related patents, trademarks, and license
10 | agreements) of which you are personally aware, and conspicuously marking the
11 | work as "Submitted on behalf of a third-party: [named here]".`
12 |
13 | Submitted on behalf of a third-party: Andrew "thisisamurray" Murray
14 | Submitted on behalf of a third-party: Duane "duane-ibm" D'Souza
15 | Submitted on behalf of a third-party: John "john-delivuk" Delivuk
16 | Submitted on behalf of a third-party: Richard "rrtaylor" Taylor
17 |
--------------------------------------------------------------------------------
/OWNERS:
--------------------------------------------------------------------------------
1 | # See the OWNERS docs at https://go.k8s.io/owners
2 | approvers:
3 | - dgrisonnet
4 | - logicalhan
5 | - dashpole
6 |
7 | reviewers:
8 | - dgrisonnet
9 | - olivierlemasle
10 | - logicalhan
11 | - dashpole
12 |
13 | emeritus_approvers:
14 | - brancz
15 | - directxman12
16 | - lilic
17 | - s-urbaniak
18 |
--------------------------------------------------------------------------------
/RELEASE.md:
--------------------------------------------------------------------------------
1 | # Release Process
2 |
3 | prometheus-adapter is released on an as-needed basis. The process is as follows:
4 |
5 | 1. An issue is proposing a new release with a changelog since the last release
6 | 1. At least one [OWNERS](OWNERS) must LGTM this release
7 | 1. A PR that bumps version hardcoded in code is created and merged
8 | 1. An OWNER creates a draft Github release
9 | 1. An OWNER creates a release tag using `git tag -s $VERSION`, inserts the changelog and pushes the tag with `git push $VERSION`. Then waits for [prow.k8s.io](https://prow.k8s.io) to build and push new images to [gcr.io/k8s-staging-prometheus-adapter](https://gcr.io/k8s-staging-prometheus-adapter)
10 | 1. A PR in [kubernetes/k8s.io](https://github.com/kubernetes/k8s.io/blob/main/k8s.gcr.io/images/k8s-staging-prometheus-adapter/images.yaml) is created to release images to `k8s.gcr.io`
11 | 1. An OWNER publishes the GitHub release
12 | 1. An announcement email is sent to `kubernetes-sig-instrumentation@googlegroups.com` with the subject `[ANNOUNCE] prometheus-adapter $VERSION is released`
13 | 1. The release issue is closed
14 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | # Security Policy
2 |
3 | ## Security Announcements
4 |
5 | Join the [kubernetes-security-announce] group for security and vulnerability announcements.
6 |
7 | You can also subscribe to an RSS feed of the above using [this link][kubernetes-security-announce-rss].
8 |
9 | ## Reporting a Vulnerability
10 |
11 | Instructions for reporting a vulnerability can be found on the
12 | [Kubernetes Security and Disclosure Information] page.
13 |
14 | ## Supported Versions
15 |
16 | Information about supported Kubernetes versions can be found on the
17 | [Kubernetes version and version skew support policy] page on the Kubernetes website.
18 |
19 | [kubernetes-security-announce]: https://groups.google.com/forum/#!forum/kubernetes-security-announce
20 | [kubernetes-security-announce-rss]: https://groups.google.com/forum/feed/kubernetes-security-announce/msgs/rss_v2_0.xml?num=50
21 | [Kubernetes version and version skew support policy]: https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions
22 | [Kubernetes Security and Disclosure Information]: https://kubernetes.io/docs/reference/issues-security/security/#report-a-vulnerability
23 |
--------------------------------------------------------------------------------
/SECURITY_CONTACTS:
--------------------------------------------------------------------------------
1 | # Defined below are the security contacts for this repo.
2 | #
3 | # They are the contact point for the Product Security Committee to reach out
4 | # to for triaging and handling of incoming issues.
5 | #
6 | # The below names agree to abide by the
7 | # [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy)
8 | # and will be removed and replaced if they violate that agreement.
9 | #
10 | # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
11 | # INSTRUCTIONS AT https://kubernetes.io/security/
12 |
13 | dgrisonnet
14 | s-urbaniak
15 |
--------------------------------------------------------------------------------
/VERSION:
--------------------------------------------------------------------------------
1 | 0.12.0
2 |
--------------------------------------------------------------------------------
/cloudbuild.yaml:
--------------------------------------------------------------------------------
1 | # See https://cloud.google.com/cloud-build/docs/build-config
2 | timeout: 3600s
3 | options:
4 | substitution_option: ALLOW_LOOSE
5 | steps:
6 | - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90'
7 | entrypoint: make
8 | env:
9 | - TAG=$_PULL_BASE_REF
10 | args:
11 | - push-all
12 |
--------------------------------------------------------------------------------
/cmd/adapter/adapter_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2016 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package main
18 |
19 | import (
20 | "net/http"
21 | "os"
22 | "path/filepath"
23 | "reflect"
24 | "testing"
25 | )
26 |
27 | const certsDir = "testdata"
28 |
29 | func TestMakeKubeconfigHTTPClient(t *testing.T) {
30 | tests := []struct {
31 | kubeconfigPath string
32 | inClusterAuth bool
33 | success bool
34 | }{
35 | {
36 | kubeconfigPath: filepath.Join(certsDir, "kubeconfig"),
37 | inClusterAuth: false,
38 | success: true,
39 | },
40 | {
41 | kubeconfigPath: filepath.Join(certsDir, "kubeconfig"),
42 | inClusterAuth: true,
43 | success: false,
44 | },
45 | {
46 | kubeconfigPath: filepath.Join(certsDir, "kubeconfig-error"),
47 | inClusterAuth: false,
48 | success: false,
49 | },
50 | {
51 | kubeconfigPath: "",
52 | inClusterAuth: false,
53 | success: true,
54 | },
55 | }
56 |
57 | os.Setenv("KUBERNETES_SERVICE_HOST", "prometheus")
58 | os.Setenv("KUBERNETES_SERVICE_PORT", "8080")
59 |
60 | for _, test := range tests {
61 | t.Logf("Running test for: inClusterAuth %v, kubeconfigPath %v", test.inClusterAuth, test.kubeconfigPath)
62 | kubeconfigHTTPClient, err := makeKubeconfigHTTPClient(test.inClusterAuth, test.kubeconfigPath)
63 | if test.success {
64 | if err != nil {
65 | t.Errorf("Error is %v, expected nil", err)
66 | continue
67 | }
68 | if kubeconfigHTTPClient.Transport == nil {
69 | if test.inClusterAuth || test.kubeconfigPath != "" {
70 | t.Error("HTTP client Transport is nil, expected http.RoundTripper")
71 | }
72 | }
73 | } else if err == nil {
74 | t.Errorf("Error is nil, expected %v", err)
75 | }
76 | }
77 | }
78 |
79 | func TestMakePrometheusCAClient(t *testing.T) {
80 | tests := []struct {
81 | caFilePath string
82 | tlsCertFilePath string
83 | tlsKeyFilePath string
84 | success bool
85 | tlsUsed bool
86 | }{
87 | {
88 | caFilePath: filepath.Join(certsDir, "ca.pem"),
89 | tlsCertFilePath: filepath.Join(certsDir, "tlscert.crt"),
90 | tlsKeyFilePath: filepath.Join(certsDir, "tlskey.key"),
91 | success: true,
92 | tlsUsed: true,
93 | },
94 | {
95 | caFilePath: filepath.Join(certsDir, "ca-error.pem"),
96 | tlsCertFilePath: filepath.Join(certsDir, "tlscert.crt"),
97 | tlsKeyFilePath: filepath.Join(certsDir, "tlskey.key"),
98 | success: false,
99 | tlsUsed: true,
100 | },
101 | {
102 | caFilePath: filepath.Join(certsDir, "ca.pem"),
103 | tlsCertFilePath: filepath.Join(certsDir, "tlscert-error.crt"),
104 | tlsKeyFilePath: filepath.Join(certsDir, "tlskey.key"),
105 | success: false,
106 | tlsUsed: true,
107 | },
108 | {
109 | caFilePath: filepath.Join(certsDir, "ca.pem"),
110 | tlsCertFilePath: "",
111 | tlsKeyFilePath: "",
112 | success: true,
113 | tlsUsed: false,
114 | },
115 | }
116 |
117 | for _, test := range tests {
118 | t.Logf("Running test for: caFilePath %v, tlsCertFilePath %v, tlsKeyFilePath %v", test.caFilePath, test.tlsCertFilePath, test.tlsKeyFilePath)
119 | prometheusCAClient, err := makePrometheusCAClient(test.caFilePath, test.tlsCertFilePath, test.tlsKeyFilePath)
120 | if test.success {
121 | if err != nil {
122 | t.Errorf("Error is %v, expected nil", err)
123 | continue
124 | }
125 | if prometheusCAClient.Transport.(*http.Transport).TLSClientConfig.RootCAs == nil {
126 | t.Error("RootCAs is nil, expected *x509.CertPool")
127 | continue
128 | }
129 | if test.tlsUsed {
130 | if prometheusCAClient.Transport.(*http.Transport).TLSClientConfig.Certificates == nil {
131 | t.Error("TLS certificates is nil, expected []tls.Certificate")
132 | continue
133 | }
134 | } else {
135 | if prometheusCAClient.Transport.(*http.Transport).TLSClientConfig.Certificates != nil {
136 | t.Errorf("TLS certificates is %+v, expected nil", prometheusCAClient.Transport.(*http.Transport).TLSClientConfig.Certificates)
137 | }
138 | }
139 | } else if err == nil {
140 | t.Errorf("Error is nil, expected %v", err)
141 | }
142 | }
143 | }
144 |
145 | func TestParseHeaderArgs(t *testing.T) {
146 | tests := []struct {
147 | args []string
148 | headers http.Header
149 | }{
150 | {
151 | headers: http.Header{},
152 | },
153 | {
154 | args: []string{"foo=bar"},
155 | headers: http.Header{
156 | "Foo": []string{"bar"},
157 | },
158 | },
159 | {
160 | args: []string{"foo"},
161 | headers: http.Header{
162 | "Foo": []string{""},
163 | },
164 | },
165 | {
166 | args: []string{"foo=bar", "foo=baz", "bux=baz=23"},
167 | headers: http.Header{
168 | "Foo": []string{"bar", "baz"},
169 | "Bux": []string{"baz=23"},
170 | },
171 | },
172 | }
173 |
174 | for _, test := range tests {
175 | got := parseHeaderArgs(test.args)
176 | if !reflect.DeepEqual(got, test.headers) {
177 | t.Errorf("Expected %#v but got %#v", test.headers, got)
178 | }
179 | }
180 | }
181 |
182 | func TestFlags(t *testing.T) {
183 | cmd := &PrometheusAdapter{
184 | PrometheusURL: "https://localhost",
185 | }
186 | cmd.addFlags()
187 |
188 | flags := cmd.FlagSet
189 | if flags == nil {
190 | t.Fatalf("FlagSet should not be nil")
191 | }
192 |
193 | expectedFlags := []struct {
194 | flag string
195 | defaultValue string
196 | }{
197 | {flag: "v", defaultValue: "0"}, // logging flag (klog)
198 | {flag: "prometheus-url", defaultValue: "https://localhost"}, // default is set in cmd
199 | }
200 |
201 | for _, e := range expectedFlags {
202 | flag := flags.Lookup(e.flag)
203 | if flag == nil {
204 | t.Errorf("Flag %q expected to be present, was absent", e.flag)
205 | continue
206 | }
207 | if flag.DefValue != e.defaultValue {
208 | t.Errorf("Expected default value %q for flag %q, got %q", e.defaultValue, e.flag, flag.DefValue)
209 | }
210 | }
211 | }
212 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/ca-error.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIDdjCCAl4CCQDdbOsYxSKoeDANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV
3 | UzENMAsGA1UEBwwEVGVzdDEaMBgGA1UECgwRUHJvbWV0aGV1c0FkYXB0ZXIxJDAi
4 | BgNVBAMMG2s4cy1wcm9tZXRoZXVzLWFkYXB0ZXIudGVzdDEdMBsGCSqGSIb3DQEJ
5 | ARYOdGVzdEB0ZXN0LnRlc3QwHhcNMjEwMjA5MTE0NzUwWhcNMjYwMjA4MTE0NzUw
6 | WjB9MQswCQYDVQQGEwJVUzENMAsGA1UEBwwEVGVzdDEaMBgGA1UECgwRUHJvbWV0
7 | aGV1c0FkYXB0ZXIxJDAiBgNVBAMMG2s4cy1wcm9tZXRoZXVzLWFkYXB0ZXIudGVz
8 | dDEdMBsGCSqGSIb3DQEJARYOdGVzdEB0ZXN0LnRlc3QwggEiMA0GCSqGSIb3DQEB
9 | AQUAA4IBDwAwggEKAoIBAQC24TDfTWLtYZPLDXqEjF7yn4K7oBOltX5Nngsk7LNd
10 | AQELBQADggEBAD/bbeAZuyvtuEwdJ+4wkhBsHYXQ4OPxff1f3t4buIQFtnilWTXE
11 | S60K3SEaQS8rOw8V9eHmzCsh3mPuVCoM7WsgKhp2mVhbGVZoBWBZ8kPQXqtsw+v4
12 | tqTuJXnFPiF4clXb6Wp96Rc7nxzRAfn/6uVbSWds4JwRToUVszVOxe+yu0I84vuB
13 | SHrRa077b1V+UT8otm+C5tC3jBZ0/IPRNWoT/rVcSoVLouX0fkbtxNF7c9v+PYg6
14 | 849A9T8cGKWKpKPGNEwBL9HYwtK6W0tTJr8A8pnAJ/UlniHA6u7SMHN+NoqBfi6M
15 | bqq9lQ4QhjFrN2B1z9r3ak+EzQX1711TQ8w=
16 | -----END CERTIFICATE-----
17 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/ca.pem:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIDLjCCAhYCCQDlnNCOw7JHFDANBgkqhkiG9w0BAQsFADBYMQswCQYDVQQGEwJV
3 | UzENMAsGA1UECgwEVGVzdDEbMBkGA1UEAwwScHJvbWV0aGV1cy1hZGFwdGVyMR0w
4 | GwYJKoZIhvcNAQkBFg50ZXN0QHRlc3QudGVzdDAgFw0yMTAyMjIyMDMxNTBaGA80
5 | NzU5MDEyMDIwMzE1MFowWDELMAkGA1UEBhMCVVMxDTALBgNVBAoMBFRlc3QxGzAZ
6 | BgNVBAMMEnByb21ldGhldXMtYWRhcHRlcjEdMBsGCSqGSIb3DQEJARYOdGVzdEB0
7 | ZXN0LnRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvn9HQlfhw
8 | qDH77+eEFU+N+ztqCtat54neVez+sFa4dfYxuvVYK+nc+oh4E7SS4u+17eKV+QFb
9 | ZhRhrOTNI+fmuO+xDPKyU1MuYUDfwasRfMDcUpssea2fO/SrsxHmX9oOam0kgefJ
10 | 8aSwI9TYw4N4kIpG+EGatogDlR2KXhrqsRfx5PUB4npFaCrdoglyvvAQig83Iq5L
11 | +bCknSe6NUMiqtL9CcuLzzRKB3DMOrvbB0tJdb4uv/gS26sx/Hp/1ri73/tv4I9z
12 | GLLoUUoff7vfvxrhiGR9i+qBOda7THbbmYBD54y+SR0dBa2uuDDX0JbgNNfXtjiG
13 | 52hvAnc1/wv7AgMBAAEwDQYJKoZIhvcNAQELBQADggEBACCysIzT9NKaniEvXtnx
14 | Yx/jRxpiEEUGl8kg83a95X4f13jdPpUSwcn3/iK5SAE/7ntGVM+ajtlXrHGxwjB7
15 | ER0w4WC6Ozypzoh/yI/VXs+DRJTJu8CBJOBRQEpzkK4r64HU8iN2c9lPp1+6b3Vy
16 | jfbf3yfnRUbJztSjOFDUeA2t3FThVddhqif/oxj65s5R8p9HEurcwhA3Q6lE53yx
17 | jgee8qV9HXAqa4V0qQQJ0tjcpajhQahDTtThRr+Z2H4TzQuwHa3dM7IIF6EPWsCo
18 | DtbUXEPL7zT3EBH7THOdvNsFlD/SFmT2RwiQ5606bRAHwAzzxjxjxFTMl7r4tX5W
19 | Ldc=
20 | -----END CERTIFICATE-----
21 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/kubeconfig:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Config
3 | clusters:
4 | - name: test
5 | cluster:
6 | certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURMakNDQWhZQ0NRRGxuTkNPdzdKSEZEQU5CZ2txaGtpRzl3MEJBUXNGQURCWU1Rc3dDUVlEVlFRR0V3SlYKVXpFTk1Bc0dBMVVFQ2d3RVZHVnpkREViTUJrR0ExVUVBd3dTY0hKdmJXVjBhR1YxY3kxaFpHRndkR1Z5TVIwdwpHd1lKS29aSWh2Y05BUWtCRmc1MFpYTjBRSFJsYzNRdWRHVnpkREFnRncweU1UQXlNakl5TURNeE5UQmFHQTgwCk56VTVNREV5TURJd016RTFNRm93V0RFTE1Ba0dBMVVFQmhNQ1ZWTXhEVEFMQmdOVkJBb01CRlJsYzNReEd6QVoKQmdOVkJBTU1FbkJ5YjIxbGRHaGxkWE10WVdSaGNIUmxjakVkTUJzR0NTcUdTSWIzRFFFSkFSWU9kR1Z6ZEVCMApaWE4wTG5SbGMzUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDdm45SFFsZmh3CnFESDc3K2VFRlUrTit6dHFDdGF0NTRuZVZleitzRmE0ZGZZeHV2VllLK25jK29oNEU3U1M0dSsxN2VLVitRRmIKWmhSaHJPVE5JK2ZtdU8reERQS3lVMU11WVVEZndhc1JmTURjVXBzc2VhMmZPL1Nyc3hIbVg5b09hbTBrZ2VmSgo4YVN3STlUWXc0TjRrSXBHK0VHYXRvZ0RsUjJLWGhycXNSZng1UFVCNG5wRmFDcmRvZ2x5dnZBUWlnODNJcTVMCitiQ2tuU2U2TlVNaXF0TDlDY3VMenpSS0IzRE1PcnZiQjB0SmRiNHV2L2dTMjZzeC9IcC8xcmk3My90djRJOXoKR0xMb1VVb2ZmN3ZmdnhyaGlHUjlpK3FCT2RhN1RIYmJtWUJENTR5K1NSMGRCYTJ1dUREWDBKYmdOTmZYdGppRwo1Mmh2QW5jMS93djdBZ01CQUFFd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDQ3lzSXpUOU5LYW5pRXZYdG54Cll4L2pSeHBpRUVVR2w4a2c4M2E5NVg0ZjEzamRQcFVTd2NuMy9pSzVTQUUvN250R1ZNK2FqdGxYckhHeHdqQjcKRVIwdzRXQzZPenlwem9oL3lJL1ZYcytEUkpUSnU4Q0JKT0JSUUVwemtLNHI2NEhVOGlOMmM5bFBwMSs2YjNWeQpqZmJmM3lmblJVYkp6dFNqT0ZEVWVBMnQzRlRoVmRkaHFpZi9veGo2NXM1UjhwOUhFdXJjd2hBM1E2bEU1M3l4CmpnZWU4cVY5SFhBcWE0VjBxUVFKMHRqY3BhamhRYWhEVHRUaFJyK1oySDRUelF1d0hhM2RNN0lJRjZFUFdzQ28KRHRiVVhFUEw3elQzRUJIN1RIT2R2TnNGbEQvU0ZtVDJSd2lRNTYwNmJSQUh3QXp6eGp4anhGVE1sN3I0dFg1VwpMZGM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
7 | server: test.test
8 | contexts:
9 | - name: test
10 | context:
11 | cluster: test
12 | user: test-user
13 | current-context: test
14 | users:
15 | - name: test-user
16 | user:
17 | token: abcde12345
18 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/kubeconfig-error:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Config
3 | clusters:
4 | - name: test
5 | cluster:
6 | certificate-authority-data: abcde12345
7 | server: test.test
8 | contexts:
9 | - name: test
10 | context:
11 | cluster: test
12 | user: test-user
13 | current-context: test
14 | users:
15 | - name: test-user
16 | user:
17 | token: abcde12345
18 |
19 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/tlscert-error.crt:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIFdjCCA14CCQC+svUhDVv51DANBgkqhkiG9w0BAQsFADB9MQswCQYDVQQGEwJV
3 | UzENMAsGA1UEBwwEVGVzdDEaMBgGA1UECgwRUHJvbWV0aGV1c0FkYXB0ZXIxJDAi
4 | BgNVBAMMG2s4cy1wcm9tZXRoZXVzLWFkYXB0ZXIudGVzdDEdMBsGCSqGSIb3DQEJ
5 | ARYOdGVzdEB0ZXN0LnRlc3QwHhcNMjEwMjA5MTE0NDMyWhcNMjIwMjA5MTE0NDMy
6 | WjB9MQswCQYDVQQGEwJVUzENMAsGA1UEBwwEVGVzdDEaMBgGA1UECgwRUHJvbWV0
7 | aGV1c0FkYXB0ZXIxJDAiBgNVBAMMG2s4cy1wcm9tZXRoZXVzLWFkYXB0ZXIudGVz
8 | dDEdMBsGCSqGSIb3DQEJARYOdGVzdEB0ZXN0LnRlc3QwggIiMA0GCSqGSIb3DQEB
9 | AQUAA4ICDwAwggIKAoICAQDtLqKuIJqRETLOUSMDBtUDmIBaD2pG9Qv+cOBhQbVS
10 | apZRWk8uKZKxqBOxgQ3UxY1szeVkx1Dphe3RN6ndmofiRc23ns1qncbDllgbtflk
11 | GFvLKGcVBa6Z/lZ6FCZDWn6K6mJb0a7jtkOMG6+J/5eJHfZ23u/GYL1RKxH+qPPc
12 | AwIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQC0oE4/Yel5GHeuKJ5+U9KEBsLCrBjj
13 | jUW8or7SIZ1Gl4J2ak3p3WabhZjqOe10rsIXCNTaC2rEMvkNiP5Om0aeE6bx14jV
14 | e9FIfJ7ytAL/PISaZXINgml05m4Su3bUaxCQpajBgqneNp4w57jfeFhcPt35j0H4
15 | bxGk/hnIY1MmRULSOFBstmxNZSDNsGlTcZoN3+0KtuqLg6vTNuuJIyx1zd9/QT8t
16 | RJ4fgrffJcPJscvq6hEdWmtcJhaDLWOEblsbFfN0J+zK07hHhqRavQrnwaBZgFWa
17 | OIqAo6NfZONhCFy9mWFxLvQky1NXr60y220+N1GkEiLRQES7+p1pcKgn0v+f2EfW
18 | uN6+LCppWX7FqtkB3OhZkHM6nbE/9GP5T76Kj30Fed/nHkTJ3QORRMQUTs4J6LNk
19 | BD1i14MZMCn3UBZh8wX+d63xJHtfMvfac7L655GwLEnWW8JM8h8DDfRYM7JuEURG
20 | pSbvoaygyvddT0FKRLcFGhfI7aBSWGcJH5rHdEcUQ+mnloD1RioQqTC+kxUSddJI
21 | QNjgYivl9kwW9cJV1jzmKd8GQfg+j1X+jR9icNT5cacvclwnL0Mim0w/ZLfWQYmJ
22 | q2ud+GS9+5RtPzWwHR60+Qs3dr8oQGh5wO12qUJ8d5MI+4YGWRjKRyYdio6g1Bhi
23 | 9WInD4va9cC7fw==
24 | -----END CERTIFICATE-----
25 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/tlscert.crt:
--------------------------------------------------------------------------------
1 | -----BEGIN CERTIFICATE-----
2 | MIIFLjCCAxYCCQDMlabDYYlDKzANBgkqhkiG9w0BAQsFADBYMQswCQYDVQQGEwJV
3 | UzENMAsGA1UECgwEVGVzdDEbMBkGA1UEAwwScHJvbWV0aGV1cy1hZGFwdGVyMR0w
4 | GwYJKoZIhvcNAQkBFg50ZXN0QHRlc3QudGVzdDAgFw0yMTAyMjIyMDMwMTNaGA80
5 | NzU5MDEyMDIwMzAxM1owWDELMAkGA1UEBhMCVVMxDTALBgNVBAoMBFRlc3QxGzAZ
6 | BgNVBAMMEnByb21ldGhldXMtYWRhcHRlcjEdMBsGCSqGSIb3DQEJARYOdGVzdEB0
7 | ZXN0LnRlc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDIJOO8apS3
8 | 84bssvnTVHp1VAiPg1tX+E6wPjayVPx+S4LgMKA4QM2kNKoPLQtvGV+lyqfhYp0H
9 | cdGzCPVQ6aBlbHuOhInusJaXjgOlNTalThgigzky0t1jFxqaNjFtXqv6ME1Zcb9H
10 | VrGMreEfNj8/Ijp7cfsBe7Jv2rBc06aidx9j66+oPTC98XcNnURUGO95UF8SjTQt
11 | oi10m17uA7z/JUUSBvDJAg5Z62myZPU2stz38cuthROyEyXRBWimHh7bD17rwqhc
12 | WRCfkFORWvwz9GMV5KfFCfWm2D2pm1f3ZWm5/FQbQrlxgxUagwDMoma+F6hQp84a
13 | /sYPqqkWDRUK0NGZzWwxjfra8r8H2xFab+5ZFVr9+FhMgy6eelZ1JJc860s35Qpk
14 | ZrSRH8RNMqLRG1cnDwHn9Md6joCZgLJhEW9L5xjpCVWkLXK59yA9ry5Jau9/2zDs
15 | wlRzYI4TNazbVa84KliEjt3nZ6DgQh3PRtxHDrqJIQSSYu1MtUmPArLtEDDP/BqD
16 | fGWCayc/SdxSWW9qU/aOq+D4KMKQXV44qc22f6rd/LKt/fcvDpbfcexXbeeNABQg
17 | x1rAnhA8L/rYc4WTTbTrb8jwhaUoqJve6XOsHPVbk/L4CS9ReP1UwvhMM1C+Ast6
18 | rr/a2bZkoMK+jmkA2QTwUsjvt/8G4dtVOwIDAQABMA0GCSqGSIb3DQEBCwUAA4IC
19 | AQAvrWslCR21l8XRGI+l4z6bpZ7+089KQRemYcHWOKZ/nDTYcydDWQMdMtDZS43d
20 | B2Wuu4UfOnK27YwuP4Ojf2hAzeaDBgv7xOcKRZ1K+zOCm+VqbtWV49c/Ow0Rc5KU
21 | N7rApohdpXeJBp5TB1qQJsKcBv3gveLAivCFTeD0LiLMVdxjRRl9ZbMXtD3PABDC
22 | KKFtE/n2MV/0wroMD9hHs9LckcNjHSrIFaQEy9cESn8q3kngFf3wvc2oM77yCOZ1
23 | 5y0AN+9ZXyMHHlMjye7GuW0Mpiwo1O4tW2brC0boqSmvSFNW9KRogKvu6Oij9Pm6
24 | jJpuUsM0KOnID8m9jJ+Xb+DGC9cgLGHRJc+zw74X2KMQnH4/pZDNbIGG7d8xEoPn
25 | RS/EbCoALmUbI2kqflVN88kN4ZUchsoHly5gIdidfo9yjeOihTF0xEEou/tzGW+K
26 | AYxwy9uIYhz4lmH894H5nqJWPY/aLxD4M9nFW0yxczCQ8tpjwVYmP3/dCKp1IUXy
27 | 0h9TjyBRPv9O3JrTLTYBPLisLNqiU+YOZM6wgqmZTPtTCxKMmNlhGWKa8prAhMdb
28 | GRxwkO6ylyL/j3J3HgcDHEC22/685L21HVFv8z/DMuj/eba4yyn1FBVXOU9hgLWS
29 | LVLoVFFp7RaGSIECcqTyXldoZZpZrA89XDVuqSvHDiCOrg==
30 | -----END CERTIFICATE-----
31 |
--------------------------------------------------------------------------------
/cmd/adapter/testdata/tlskey.key:
--------------------------------------------------------------------------------
1 | -----BEGIN PRIVATE KEY-----
2 | MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDIJOO8apS384bs
3 | svnTVHp1VAiPg1tX+E6wPjayVPx+S4LgMKA4QM2kNKoPLQtvGV+lyqfhYp0HcdGz
4 | CPVQ6aBlbHuOhInusJaXjgOlNTalThgigzky0t1jFxqaNjFtXqv6ME1Zcb9HVrGM
5 | reEfNj8/Ijp7cfsBe7Jv2rBc06aidx9j66+oPTC98XcNnURUGO95UF8SjTQtoi10
6 | m17uA7z/JUUSBvDJAg5Z62myZPU2stz38cuthROyEyXRBWimHh7bD17rwqhcWRCf
7 | kFORWvwz9GMV5KfFCfWm2D2pm1f3ZWm5/FQbQrlxgxUagwDMoma+F6hQp84a/sYP
8 | qqkWDRUK0NGZzWwxjfra8r8H2xFab+5ZFVr9+FhMgy6eelZ1JJc860s35QpkZrSR
9 | H8RNMqLRG1cnDwHn9Md6joCZgLJhEW9L5xjpCVWkLXK59yA9ry5Jau9/2zDswlRz
10 | YI4TNazbVa84KliEjt3nZ6DgQh3PRtxHDrqJIQSSYu1MtUmPArLtEDDP/BqDfGWC
11 | ayc/SdxSWW9qU/aOq+D4KMKQXV44qc22f6rd/LKt/fcvDpbfcexXbeeNABQgx1rA
12 | nhA8L/rYc4WTTbTrb8jwhaUoqJve6XOsHPVbk/L4CS9ReP1UwvhMM1C+Ast6rr/a
13 | 2bZkoMK+jmkA2QTwUsjvt/8G4dtVOwIDAQABAoICAQCFd9RG+exjH4uCnXfsbhGb
14 | 3IY47igj6frPnS1sjzAyKLkGOGcgHFcGgfhGVouhcxJNxW9e5hxBsq1c70RoyOOl
15 | v0pGKCyzeB90wce8jFf8tK9zlH64XdY1FlsvK6Sagt+84Ck01J3yPOX6IppV7h8P
16 | Qwws9j2lJ5A+919VB++/uCC+yZVCZEv03um9snq2ekp4ZBiCjpeVNumJMXOE1glb
17 | PMdq1iYMZcqcPFkoFhtQdsbUsfJZrL0Nq6c0VJ8M6Fk7TGzIW+9aZiqnvd98t2go
18 | XXkWSH148MNYmCvGx0lKOd7foF2WMFDqWbfhDiuiS0qoya3822qepff+ypgnlGHK
19 | vr+9pLsWT7TG8pBfbXj47a7TwYAXkRMi+78vFQwoaeiKdehJM1YXZg9vBVS8BV3r
20 | +0wYNE4WpdxUvX3aAnJO6ntRU6KCz3/D1+fxUT/w1rKX2Z1uTH5x2UxB6UUGDSF9
21 | HiJfDp6RRtXHbQMR6uowM6UYBn0dl9Aso21oc2K4Gpx5QlsZaPi9M6BBMbPUhFcx
22 | QH+w7fLmccwneJVGxjHkYOcLVLF7nuH5C2DsffrMubrgwuhSw2b8zy7ZpZ0eJ83D
23 | CjJN9EgqwbmH0Or5N91YyVdR0Zm4EtODAo615O1kEMCKasKjpolOx/t9cgtbdkiq
24 | pbLruOS+8jEG1erA7nYkQQKCAQEA4yba38hSkfIMUzfrlgF7AkXHbU4iINgKpHti
25 | A9QrvEL9W4VHRiA5UTezzblyfMck9w/Hhx74pQjjGLj76L+8ZssCFI8ingNo3ItL
26 | /AX3MN68NT4heiy8EvKRwRNWV05WEehZg9tTUKexIDRcDSr/9E+qG/cW5KOIQpYl
27 | RIsKW2RUNFd3TVCQVUIzwe/0n6LuO2b7Btow+nfJ7U3mWQmHGYu7+SYjVlvIoQ68
28 | jFGviGRineu/J7EiPND7qQzj78AtnXkULf+mjK2JdapRcn2EBNL34QepVCyjfXZf
29 | QWm/ykI9nVOKRy1F38OhRHKrBICfWhN2Bgyvw3PPhGcb8EdknwKCAQEA4Y/2bpiz
30 | S0H8LPUUsOLZWCadpp8yzvTerB/vuigkQiHM8w4QUwEpL2iXSF36MD8yV/c4ilVN
31 | 8m1p5prp1YtasTxsOWv7FDEs4oZfum1w3QsAvlrFRhctsACsZ1i4i3mvxQWJ955q
32 | zZxs5vhO5CL24rVoQYGVQj/uCSHlyK7ko9AA8XkejTlZMJ5h0Mip+oWNxz3M/VTa
33 | sJlYkQrbP0cWxCjKJLEmtVlVSCMeHoILGZzLcol6RVPbaAb57i27SRwY9YIFt1A+
34 | OMpHFs4fgDa4A1IlobBwhhd1dAw3CL5QJN+ylDnBYsm1bwBRHx/AKUjpRv+7ZXQb
35 | H9ngSivFHrXN5QKCAQBAqzUw9LUdO83qe0ck47MDiJ4oLlBlDVyqSz4yXNs+s8ux
36 | nJYYDuCCkNstvJgtkfyiIenqPBUJ1yfgR/nf34ZhtXYYKE/wsIPQFhBB5ejkDuWC
37 | OvgI8mdw9YItd7XjEThLzNx/P5fOpI823fE/BnjsMyn44DWyTiRi4KAnjXYbYsre
38 | Q/CBIGiW/UwC8K+yKw6r9ruMzd2X0Ta5yq3Dt4Sw7ylK22LAGU1bHPjs8eyJZhr1
39 | XsKDKFjY+55KGJNkFFBoPqpSFjByaI1z5FNfxwAo528Or8GzZyn8dBDWbKbfjFBC
40 | VCBP90GnXOiytfqeQ4gaeuPlAQOhH3168mfv1kN9AoIBABOZzgFYVaRBjKdfeLfS
41 | Tq7BVEvJY8HmN39fmxZjLJtukn/AhhygajLLdPH98KLGqxpHymsC9K4PYfd/GLjM
42 | zkm+hW0L/BqKF2tr39+0aO1camkgPCpWE0tLE7A7XnYIUgTd8VpKMt/BKxl7FGfw
43 | veF/gBrJJu5F3ep/PpeM0yOFDL/vFX+SLzTxXnClL1gsyOA6d5jACez0tmSMO/co
44 | t0q+fKpploKFy8pj+tcN1+cW3/sJBU4G9nb4vDk9UhwNTAHxlYuTdoS61yidKtGa
45 | b60iM1D0oyKT4Un/Ubz5xL8fjUYiKrLp8lE+Bs6clLdBtbvMtz0etMi0xy/K0+tS
46 | Qx0CggEBALfe2TUfAt9aMqpcidaFwgNFTr61wgOeoLWLt559OeRIeZWKAEB81bnz
47 | EJLxDF51Y2tLc/pEXrc0zJzzrFIfk/drYe0uD5RnJjRxE3+spwin6D32ZOZW3KSX
48 | 1zReW1On80o/LJU6nyDJrNJvay2eL9PyWi47nBdO7MRZi53im72BmmwxaAKXf40l
49 | StykjloyFdI+eyGyQUqcs4nFHd3WWmV+lLIDhGDlF5EBUgueCJz1xO54oPj1PKGl
50 | vDs7JXdJiS3HDf20GREGwvL1y1kewX+KqdO7aBZhLN3Rx/fZnS/UFC3xCtbikuG4
51 | LeU1NmvuCRmWmrgEkqiKs3jgjbEPVQI=
52 | -----END PRIVATE KEY-----
53 |
--------------------------------------------------------------------------------
/cmd/config-gen/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "time"
7 |
8 | "github.com/spf13/cobra"
9 | yaml "gopkg.in/yaml.v2"
10 |
11 | "sigs.k8s.io/prometheus-adapter/cmd/config-gen/utils"
12 | )
13 |
14 | func main() {
15 | var labelPrefix string
16 | var rateInterval time.Duration
17 |
18 | cmd := &cobra.Command{
19 | Short: "Generate a config matching the legacy discovery rules",
20 | Long: `Generate a config that produces the same functionality
21 | as the legacy discovery rules. This includes discovering metrics and associating
22 | resources according to the Kubernetes instrumention conventions and the cAdvisor
23 | conventions, and auto-converting cumulative metrics into rate metrics.`,
24 | RunE: func(c *cobra.Command, args []string) error {
25 | cfg := utils.DefaultConfig(rateInterval, labelPrefix)
26 | enc := yaml.NewEncoder(os.Stdout)
27 | if err := enc.Encode(cfg); err != nil {
28 | return err
29 | }
30 | return enc.Close()
31 | },
32 | }
33 |
34 | cmd.Flags().StringVar(&labelPrefix, "label-prefix", "",
35 | "Prefix to expect on labels referring to pod resources. For example, if the prefix is "+
36 | "'kube_', any series with the 'kube_pod' label would be considered a pod metric")
37 | cmd.Flags().DurationVar(&rateInterval, "rate-interval", 5*time.Minute,
38 | "Period of time used to calculate rate metrics from cumulative metrics")
39 |
40 | if err := cmd.Execute(); err != nil {
41 | fmt.Fprintf(os.Stderr, "Unable to generate config: %v\n", err)
42 | os.Exit(1)
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/cmd/config-gen/utils/default.go:
--------------------------------------------------------------------------------
1 | package utils
2 |
3 | import (
4 | "fmt"
5 | "time"
6 |
7 | pmodel "github.com/prometheus/common/model"
8 |
9 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
10 | "sigs.k8s.io/prometheus-adapter/pkg/config"
11 | )
12 |
13 | // DefaultConfig returns a configuration equivalent to the former
14 | // pre-advanced-config settings. This means that "normal" series labels
15 | // will be of the form `<<.Resource>>`, cadvisor series will be
16 | // of the form `container_`, and have the label `pod`. Any series ending
17 | // in total will be treated as a rate metric.
18 | func DefaultConfig(rateInterval time.Duration, labelPrefix string) *config.MetricsDiscoveryConfig {
19 | return &config.MetricsDiscoveryConfig{
20 | Rules: []config.DiscoveryRule{
21 | // container seconds rate metrics
22 | {
23 | SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod", ""))),
24 | Resources: config.ResourceMapping{
25 | Overrides: map[string]config.GroupResource{
26 | "namespace": {Resource: "namespace"},
27 | "pod": {Resource: "pod"},
28 | },
29 | },
30 | Name: config.NameMapping{Matches: "^container_(.*)_seconds_total$"},
31 | MetricsQuery: fmt.Sprintf(`sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[%s])) by (<<.GroupBy>>)`, pmodel.Duration(rateInterval).String()),
32 | },
33 |
34 | // container rate metrics
35 | {
36 | SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod", ""))),
37 | SeriesFilters: []config.RegexFilter{{IsNot: "^container_.*_seconds_total$"}},
38 | Resources: config.ResourceMapping{
39 | Overrides: map[string]config.GroupResource{
40 | "namespace": {Resource: "namespace"},
41 | "pod": {Resource: "pod"},
42 | },
43 | },
44 | Name: config.NameMapping{Matches: "^container_(.*)_total$"},
45 | MetricsQuery: fmt.Sprintf(`sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[%s])) by (<<.GroupBy>>)`, pmodel.Duration(rateInterval).String()),
46 | },
47 |
48 | // container non-cumulative metrics
49 | {
50 | SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod", ""))),
51 | SeriesFilters: []config.RegexFilter{{IsNot: "^container_.*_total$"}},
52 | Resources: config.ResourceMapping{
53 | Overrides: map[string]config.GroupResource{
54 | "namespace": {Resource: "namespace"},
55 | "pod": {Resource: "pod"},
56 | },
57 | },
58 | Name: config.NameMapping{Matches: "^container_(.*)$"},
59 | MetricsQuery: `sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>)`,
60 | },
61 |
62 | // normal non-cumulative metrics
63 | {
64 | SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))),
65 | SeriesFilters: []config.RegexFilter{{IsNot: ".*_total$"}},
66 | Resources: config.ResourceMapping{
67 | Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix),
68 | },
69 | MetricsQuery: "sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)",
70 | },
71 |
72 | // normal rate metrics
73 | {
74 | SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))),
75 | SeriesFilters: []config.RegexFilter{{IsNot: ".*_seconds_total"}},
76 | Name: config.NameMapping{Matches: "^(.*)_total$"},
77 | Resources: config.ResourceMapping{
78 | Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix),
79 | },
80 | MetricsQuery: fmt.Sprintf("sum(rate(<<.Series>>{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()),
81 | },
82 |
83 | // seconds rate metrics
84 | {
85 | SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))),
86 | Name: config.NameMapping{Matches: "^(.*)_seconds_total$"},
87 | Resources: config.ResourceMapping{
88 | Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix),
89 | },
90 | MetricsQuery: fmt.Sprintf("sum(rate(<<.Series>>{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()),
91 | },
92 | },
93 |
94 | ResourceRules: &config.ResourceRules{
95 | CPU: config.ResourceRule{
96 | ContainerQuery: fmt.Sprintf("sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()),
97 | NodeQuery: fmt.Sprintf("sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()),
98 | Resources: config.ResourceMapping{
99 | Overrides: map[string]config.GroupResource{
100 | "namespace": {Resource: "namespace"},
101 | "pod": {Resource: "pod"},
102 | "instance": {Resource: "node"},
103 | },
104 | },
105 | ContainerLabel: fmt.Sprintf("%scontainer", labelPrefix),
106 | },
107 | Memory: config.ResourceRule{
108 | ContainerQuery: "sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)",
109 | NodeQuery: "sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)",
110 | Resources: config.ResourceMapping{
111 | Overrides: map[string]config.GroupResource{
112 | "namespace": {Resource: "namespace"},
113 | "pod": {Resource: "pod"},
114 | "instance": {Resource: "node"},
115 | },
116 | },
117 | ContainerLabel: fmt.Sprintf("%scontainer", labelPrefix),
118 | },
119 | Window: pmodel.Duration(rateInterval),
120 | },
121 | }
122 | }
123 |
--------------------------------------------------------------------------------
/code-of-conduct.md:
--------------------------------------------------------------------------------
1 | # Kubernetes Community Code of Conduct
2 |
3 | Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md)
4 |
--------------------------------------------------------------------------------
/deploy/README.md:
--------------------------------------------------------------------------------
1 | Example Deployment
2 | ==================
3 |
4 | 1. Make sure you've built the included Dockerfile with `TAG=latest make container`. The image should be tagged as `registry.k8s.io/prometheus-adapter/staging-prometheus-adapter:latest`.
5 |
6 | 2. `kubectl create namespace monitoring` to ensure that the namespace that we're installing
7 | the custom metrics adapter in exists.
8 |
9 | 3. `kubectl create -f manifests/`, modifying the Deployment as necessary to
10 | point to your Prometheus server, and the ConfigMap to contain your desired
11 | metrics discovery configuration.
12 |
--------------------------------------------------------------------------------
/deploy/manifests/api-service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apiregistration.k8s.io/v1
2 | kind: APIService
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: v1beta1.metrics.k8s.io
9 | spec:
10 | group: metrics.k8s.io
11 | groupPriorityMinimum: 100
12 | insecureSkipTLSVerify: true
13 | service:
14 | name: prometheus-adapter
15 | namespace: monitoring
16 | version: v1beta1
17 | versionPriority: 100
18 |
--------------------------------------------------------------------------------
/deploy/manifests/cluster-role-aggregated-metrics-reader.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | rbac.authorization.k8s.io/aggregate-to-admin: "true"
9 | rbac.authorization.k8s.io/aggregate-to-edit: "true"
10 | rbac.authorization.k8s.io/aggregate-to-view: "true"
11 | name: system:aggregated-metrics-reader
12 | namespace: monitoring
13 | rules:
14 | - apiGroups:
15 | - metrics.k8s.io
16 | resources:
17 | - pods
18 | - nodes
19 | verbs:
20 | - get
21 | - list
22 | - watch
23 |
--------------------------------------------------------------------------------
/deploy/manifests/cluster-role-binding-delegator.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRoleBinding
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: resource-metrics:system:auth-delegator
9 | namespace: monitoring
10 | roleRef:
11 | apiGroup: rbac.authorization.k8s.io
12 | kind: ClusterRole
13 | name: system:auth-delegator
14 | subjects:
15 | - kind: ServiceAccount
16 | name: prometheus-adapter
17 | namespace: monitoring
18 |
--------------------------------------------------------------------------------
/deploy/manifests/cluster-role-binding-hpa-custom-metrics.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRoleBinding
3 | metadata:
4 | name: hpa-controller-custom-metrics
5 | labels:
6 | app.kubernetes.io/component: metrics-adapter
7 | app.kubernetes.io/name: prometheus-adapter
8 | roleRef:
9 | apiGroup: rbac.authorization.k8s.io
10 | kind: ClusterRole
11 | name: custom-metrics-server-resources
12 | subjects:
13 | - kind: ServiceAccount
14 | name: horizontal-pod-autoscaler
15 | namespace: kube-system
16 |
--------------------------------------------------------------------------------
/deploy/manifests/cluster-role-binding.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRoleBinding
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: prometheus-adapter
9 | namespace: monitoring
10 | roleRef:
11 | apiGroup: rbac.authorization.k8s.io
12 | kind: ClusterRole
13 | name: prometheus-adapter
14 | subjects:
15 | - kind: ServiceAccount
16 | name: prometheus-adapter
17 | namespace: monitoring
18 |
--------------------------------------------------------------------------------
/deploy/manifests/cluster-role-metrics-server-resources.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: resource-metrics-server-resources
9 | rules:
10 | - apiGroups:
11 | - metrics.k8s.io
12 | resources:
13 | - '*'
14 | verbs:
15 | - '*'
16 |
--------------------------------------------------------------------------------
/deploy/manifests/cluster-role.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: prometheus-adapter
9 | rules:
10 | - apiGroups:
11 | - ""
12 | resources:
13 | - nodes
14 | - namespaces
15 | - pods
16 | - services
17 | verbs:
18 | - get
19 | - list
20 | - watch
21 |
--------------------------------------------------------------------------------
/deploy/manifests/config-map.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | data:
3 | config.yaml: |-
4 | "resourceRules":
5 | "cpu":
6 | "containerLabel": "container"
7 | "containerQuery": |
8 | sum by (<<.GroupBy>>) (
9 | irate (
10 | container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="",pod!=""}[4m]
11 | )
12 | )
13 | "nodeQuery": |
14 | sum by (<<.GroupBy>>) (
15 | irate(
16 | node_cpu_usage_seconds_total{<<.LabelMatchers>>}[4m]
17 | )
18 | )
19 | "resources":
20 | "overrides":
21 | "namespace":
22 | "resource": "namespace"
23 | "node":
24 | "resource": "node"
25 | "pod":
26 | "resource": "pod"
27 | "memory":
28 | "containerLabel": "container"
29 | "containerQuery": |
30 | sum by (<<.GroupBy>>) (
31 | container_memory_working_set_bytes{<<.LabelMatchers>>,container!="",pod!=""}
32 | )
33 | "nodeQuery": |
34 | sum by (<<.GroupBy>>) (
35 | node_memory_working_set_bytes{<<.LabelMatchers>>}
36 | )
37 | "resources":
38 | "overrides":
39 | "node":
40 | "resource": "node"
41 | "namespace":
42 | "resource": "namespace"
43 | "pod":
44 | "resource": "pod"
45 | "window": "5m"
46 | kind: ConfigMap
47 | metadata:
48 | labels:
49 | app.kubernetes.io/component: metrics-adapter
50 | app.kubernetes.io/name: prometheus-adapter
51 | app.kubernetes.io/version: 0.12.0
52 | name: adapter-config
53 | namespace: monitoring
54 |
--------------------------------------------------------------------------------
/deploy/manifests/deployment.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: apps/v1
2 | kind: Deployment
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: prometheus-adapter
9 | namespace: monitoring
10 | spec:
11 | replicas: 2
12 | selector:
13 | matchLabels:
14 | app.kubernetes.io/component: metrics-adapter
15 | app.kubernetes.io/name: prometheus-adapter
16 | strategy:
17 | rollingUpdate:
18 | maxSurge: 1
19 | maxUnavailable: 1
20 | template:
21 | metadata:
22 | labels:
23 | app.kubernetes.io/component: metrics-adapter
24 | app.kubernetes.io/name: prometheus-adapter
25 | app.kubernetes.io/version: 0.12.0
26 | spec:
27 | automountServiceAccountToken: true
28 | containers:
29 | - args:
30 | - --cert-dir=/var/run/serving-cert
31 | - --config=/etc/adapter/config.yaml
32 | - --metrics-relist-interval=1m
33 | - --prometheus-url=https://prometheus.monitoring.svc:9090/
34 | - --secure-port=6443
35 | - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA
36 | image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0
37 | livenessProbe:
38 | failureThreshold: 5
39 | httpGet:
40 | path: /livez
41 | port: https
42 | scheme: HTTPS
43 | initialDelaySeconds: 30
44 | periodSeconds: 5
45 | name: prometheus-adapter
46 | ports:
47 | - containerPort: 6443
48 | name: https
49 | readinessProbe:
50 | failureThreshold: 5
51 | httpGet:
52 | path: /readyz
53 | port: https
54 | scheme: HTTPS
55 | initialDelaySeconds: 30
56 | periodSeconds: 5
57 | resources:
58 | requests:
59 | cpu: 102m
60 | memory: 180Mi
61 | securityContext:
62 | allowPrivilegeEscalation: false
63 | capabilities:
64 | drop:
65 | - ALL
66 | readOnlyRootFilesystem: true
67 | terminationMessagePolicy: FallbackToLogsOnError
68 | volumeMounts:
69 | - mountPath: /tmp
70 | name: tmpfs
71 | readOnly: false
72 | - mountPath: /var/run/serving-cert
73 | name: volume-serving-cert
74 | readOnly: false
75 | - mountPath: /etc/adapter
76 | name: config
77 | readOnly: false
78 | nodeSelector:
79 | kubernetes.io/os: linux
80 | securityContext: {}
81 | serviceAccountName: prometheus-adapter
82 | volumes:
83 | - emptyDir: {}
84 | name: tmpfs
85 | - emptyDir: {}
86 | name: volume-serving-cert
87 | - configMap:
88 | name: adapter-config
89 | name: config
90 |
--------------------------------------------------------------------------------
/deploy/manifests/network-policy.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: networking.k8s.io/v1
2 | kind: NetworkPolicy
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: prometheus-adapter
9 | namespace: monitoring
10 | spec:
11 | egress:
12 | - {}
13 | ingress:
14 | - {}
15 | podSelector:
16 | matchLabels:
17 | app.kubernetes.io/component: metrics-adapter
18 | app.kubernetes.io/name: prometheus-adapter
19 | policyTypes:
20 | - Egress
21 | - Ingress
22 |
--------------------------------------------------------------------------------
/deploy/manifests/pod-disruption-budget.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: policy/v1
2 | kind: PodDisruptionBudget
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: prometheus-adapter
9 | namespace: monitoring
10 | spec:
11 | minAvailable: 1
12 | selector:
13 | matchLabels:
14 | app.kubernetes.io/component: metrics-adapter
15 | app.kubernetes.io/name: prometheus-adapter
16 |
--------------------------------------------------------------------------------
/deploy/manifests/role-binding-auth-reader.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: RoleBinding
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: resource-metrics-auth-reader
9 | namespace: kube-system
10 | roleRef:
11 | apiGroup: rbac.authorization.k8s.io
12 | kind: Role
13 | name: extension-apiserver-authentication-reader
14 | subjects:
15 | - kind: ServiceAccount
16 | name: prometheus-adapter
17 | namespace: monitoring
18 |
--------------------------------------------------------------------------------
/deploy/manifests/service-account.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | automountServiceAccountToken: false
3 | kind: ServiceAccount
4 | metadata:
5 | labels:
6 | app.kubernetes.io/component: metrics-adapter
7 | app.kubernetes.io/name: prometheus-adapter
8 | app.kubernetes.io/version: 0.12.0
9 | name: prometheus-adapter
10 | namespace: monitoring
11 |
--------------------------------------------------------------------------------
/deploy/manifests/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | labels:
5 | app.kubernetes.io/component: metrics-adapter
6 | app.kubernetes.io/name: prometheus-adapter
7 | app.kubernetes.io/version: 0.12.0
8 | name: prometheus-adapter
9 | namespace: monitoring
10 | spec:
11 | ports:
12 | - name: https
13 | port: 443
14 | targetPort: 6443
15 | selector:
16 | app.kubernetes.io/component: metrics-adapter
17 | app.kubernetes.io/name: prometheus-adapter
18 |
--------------------------------------------------------------------------------
/docs/config-walkthrough.md:
--------------------------------------------------------------------------------
1 | Configuration Walkthroughs
2 | ==========================
3 |
4 | *If you're looking for reference documentation on configuration, please
5 | read the [configuration reference](/docs/config.md)*
6 |
7 | Per-pod HTTP Requests
8 | ---------------------
9 |
10 | ### Background
11 |
12 | *The [full walkthrough](/docs/walkthrough.md) sets up the background for
13 | something like this*
14 |
15 | Suppose we have some frontend webserver, and we're trying to write a
16 | configuration for the Prometheus adapter so that we can autoscale it based
17 | on the HTTP requests per second that it receives.
18 |
19 | Before starting, we've gone and instrumented our frontend server with
20 | a metric, `http_requests_total`. It is exposed with a single label,
21 | `method`, breaking down the requests by HTTP verb.
22 |
23 | We've configured our Prometheus to collect the metric, and it
24 | adds the `kubernetes_namespace` and `kubernetes_pod_name` labels,
25 | representing namespace and pod, respectively.
26 |
27 | If we query Prometheus, we see series that look like
28 |
29 | ```
30 | http_requests_total{method="GET",kubernetes_namespace="production",kubernetes_pod_name="frontend-server-abcd-0123"}
31 | ```
32 |
33 | ### Configuring the adapter
34 |
35 | The adapter considers metrics in the following ways:
36 |
37 | 1. First, it discovers the metrics available (*Discovery*)
38 |
39 | 2. Then, it figures out which Kubernetes resources each metric is
40 | associated with (*Association*)
41 |
42 | 3. Then, it figures out how it should expose them to the custom metrics
43 | API (*Naming*)
44 |
45 | 4. Finally, it figures out how it should query Prometheus to get the
46 | actual numbers (*Querying*)
47 |
48 | We need to inform the adapter how it should perform each of these steps
49 | for our metric, `http_requests_total`, so we'll need to add a new
50 | ***rule***. Each rule in the adapter encodes these steps. Let's add a new
51 | one to our configuration:
52 |
53 | ```yaml
54 | rules:
55 | - {}
56 | ```
57 |
58 | If we want to find all `http_requests_total` series ourselves in the
59 | Prometheus dashboard, we'd write
60 | `http_requests_total{kubernetes_namespace!="",kubernetes_pod_name!=""}` to
61 | find all `http_requests_total` series that were associated with
62 | a namespace and pod.
63 |
64 | We can add this to our rule in the `seriesQuery` field, to tell the
65 | adapter how *discover* the right series itself:
66 |
67 | ```yaml
68 | rules:
69 | - seriesQuery: 'http_requests_total{kubernetes_namespace!="",kubernetes_pod_name!=""}'
70 | ```
71 |
72 | Next, we'll need to tell the adapter how to figure out which Kubernetes
73 | resources are associated with the metric. We've already said that
74 | `kubernetes_namespace` represents the namespace name, and
75 | `kubernetes_pod_name` represents the pod name. Since these names don't
76 | quite follow a consistent pattern, we use the `overrides` section of the
77 | `resources` field in our rule:
78 |
79 | ```yaml
80 | rules:
81 | - seriesQuery: 'http_requests_total{kubernetes_namespace!="",kubernetes_pod_name!=""}'
82 | resources:
83 | overrides:
84 | kubernetes_namespace: {resource: "namespace"}
85 | kubernetes_pod_name: {resource: "pod"}
86 | ```
87 |
88 | This says that each label represents its corresponding resource. Since the
89 | resources are in the "core" kubernetes API, we don't need to specify
90 | a group. The adapter will automatically take care of pluralization, so we
91 | can specify either `pod` or `pods`, just the same way as in `kubectl get`.
92 | The resources can be any resource available in your kubernetes cluster, as
93 | long as you've got a corresponding label.
94 |
95 | If our labels followed a consistent pattern, like `kubernetes_`,
96 | we could specify `resources: {template: "kubernetes_<<.Resource>>"}`
97 | instead of specifying an override for each resource. If you want to see
98 | all resources currently available in your cluster, you can use the
99 | `kubectl api-resources` command (but the list of available resources can
100 | change as you add or remove CRDs or aggregated API servers). For more
101 | information on resources, see [Kinds, Resources, and
102 | Scopes](https://github.com/kubernetes-sigs/custom-metrics-apiserver/blob/master/docs/getting-started.md#kinds-resources-and-scopes)
103 | in the custom-metrics-apiserver boilerplate guide.
104 |
105 | Now, cumulative metrics (like those that end in `_total`) aren't
106 | particularly useful for autoscaling, so we want to convert them to rate
107 | metrics in the API. We'll call the rate version of our metric
108 | `http_requests_per_second`. We can use the `name` field to tell the
109 | adapter about that:
110 |
111 | ```yaml
112 | rules:
113 | - seriesQuery: 'http_requests_total{kubernetes_namespace!="",kubernetes_pod_name!=""}'
114 | resources:
115 | overrides:
116 | kubernetes_namespace: {resource: "namespace"}
117 | kubernetes_pod_name: {resource: "pod"}
118 | name:
119 | matches: "^(.*)_total"
120 | as: "${1}_per_second"
121 | ```
122 |
123 | Here, we've said that we should take the name matching
124 | `_total`, and turning it into `_per_second`.
125 |
126 | Finally, we need to tell the adapter how to actually query Prometheus to
127 | get some numbers. Since we want a rate, we might write:
128 | `sum(rate(http_requests_total{kubernetes_namespace="production",kubernetes_pod_name=~"frontend-server-abcd-0123|fronted-server-abcd-4567"}) by (kubernetes_pod_name)`,
129 | which would get us the total requests per second for each pod, summed across verbs.
130 |
131 | We can write something similar in the adapter, using the `metricsQuery`
132 | field:
133 |
134 | ```yaml
135 | rules:
136 | - seriesQuery: 'http_requests_total{kubernetes_namespace!="",kubernetes_pod_name!=""}'
137 | resources:
138 | overrides:
139 | kubernetes_namespace: {resource: "namespace"}
140 | kubernetes_pod_name: {resource: "pod"}
141 | name:
142 | matches: "^(.*)_total"
143 | as: "${1}_per_second"
144 | metricsQuery: 'sum(rate(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)'
145 | ```
146 |
147 | The adapter will automatically fill in the right series name, label
148 | matchers, and group-by clause, depending on what we put into the API.
149 | Since we're only working with a single metric anyway, we could replace
150 | `<<.Series>>` with `http_requests_total`.
151 |
152 | Now, if we run an instance of the Prometheus adapter with this
153 | configuration, we should see discovery information at
154 | `$KUBERNETES/apis/custom.metrics.k8s.io/v1beta1/` of
155 |
156 | ```json
157 | {
158 | "kind": "APIResourceList",
159 | "apiVersion": "v1",
160 | "groupVersion": "custom.metrics.k8s.io/v1beta1",
161 | "resources": [
162 | {
163 | "name": "pods/http_requests_total",
164 | "singularName": "",
165 | "namespaced": true,
166 | "kind": "MetricValueList",
167 | "verbs": ["get"]
168 | },
169 | {
170 | "name": "namespaces/http_requests_total",
171 | "singularName": "",
172 | "namespaced": false,
173 | "kind": "MetricValueList",
174 | "verbs": ["get"]
175 | }
176 | ]
177 | }
178 | ```
179 |
180 | Notice that we get an entry for both "pods" and "namespaces" -- the
181 | adapter exposes the metric on each resource that we've associated the
182 | metric with (and all namespaced resources must be associated with
183 | a namespace), and will fill in the `<<.GroupBy>>` section with the
184 | appropriate label depending on which we ask for.
185 |
186 | We can now connect to
187 | `$KUBERNETES/apis/custom.metrics.k8s.io/v1beta1/namespaces/production/pods/*/http_requests_per_second`,
188 | and we should see
189 |
190 | ```json
191 | {
192 | "kind": "MetricValueList",
193 | "apiVersion": "custom.metrics.k8s.io/v1beta1",
194 | "metadata": {
195 | "selfLink": "/apis/custom.metrics.k8s.io/v1beta1/namespaces/production/pods/*/http_requests_per_second",
196 | },
197 | "items": [
198 | {
199 | "describedObject": {
200 | "kind": "Pod",
201 | "name": "frontend-server-abcd-0123",
202 | "apiVersion": "/__internal",
203 | },
204 | "metricName": "http_requests_per_second",
205 | "timestamp": "2018-08-07T17:45:22Z",
206 | "value": "16m"
207 | },
208 | {
209 | "describedObject": {
210 | "kind": "Pod",
211 | "name": "frontend-server-abcd-4567",
212 | "apiVersion": "/__internal",
213 | },
214 | "metricName": "http_requests_per_second",
215 | "timestamp": "2018-08-07T17:45:22Z",
216 | "value": "22m"
217 | }
218 | ]
219 | }
220 | ```
221 |
222 | This says that our server pods are receiving 16 and 22 milli-requests per
223 | second (depending on the pod), which is 0.016 and 0.022 requests per
224 | second, written out as a decimal. That's about what we'd expect with
225 | little-to-no traffic except for the Prometheus scrape.
226 |
227 | If we added some traffic to our pods, we might see `1` or `20` instead of
228 | `16m`, which would be `1` or `20` requests per second. We might also see
229 | `20500m`, which would mean 20500 milli-requests per second, or 20.5
230 | requests per second in decimal form.
231 |
--------------------------------------------------------------------------------
/docs/config.md:
--------------------------------------------------------------------------------
1 | Metrics Discovery and Presentation Configuration
2 | ================================================
3 |
4 | *If you want a full walkthrough of configuring the adapter for a sample
5 | metric, please read the [configuration
6 | walkthrough](/docs/config-walkthrough.md)*
7 |
8 | The adapter determines which metrics to expose, and how to expose them,
9 | through a set of "discovery" rules. Each rule is executed independently
10 | (so make sure that your rules are mutually exclusive), and specifies each
11 | of the steps the adapter needs to take to expose a metric in the API.
12 |
13 | Each rule can be broken down into roughly four parts:
14 |
15 | - *Discovery*, which specifies how the adapter should find all Prometheus
16 | metrics for this rule.
17 |
18 | - *Association*, which specifies how the adapter should determine which
19 | Kubernetes resources a particular metric is associated with.
20 |
21 | - *Naming*, which specifies how the adapter should expose the metric in
22 | the custom metrics API.
23 |
24 | - *Querying*, which specifies how a request for a particular metric on one
25 | or more Kubernetes objects should be turned into a query to Prometheus.
26 |
27 | A more comprehensive configuration file can be found in
28 | [sample-config.yaml](sample-config.yaml), but a basic config with one rule
29 | might look like:
30 |
31 | ```yaml
32 | rules:
33 | # this rule matches cumulative cAdvisor metrics measured in seconds
34 | - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
35 | resources:
36 | # skip specifying generic resource<->label mappings, and just
37 | # attach only pod and namespace resources by mapping label names to group-resources
38 | overrides:
39 | namespace: {resource: "namespace"}
40 | pod: {resource: "pod"}
41 | # specify that the `container_` and `_seconds_total` suffixes should be removed.
42 | # this also introduces an implicit filter on metric family names
43 | name:
44 | # we use the value of the capture group implicitly as the API name
45 | # we could also explicitly write `as: "$1"`
46 | matches: "^container_(.*)_seconds_total$"
47 | # specify how to construct a query to fetch samples for a given series
48 | # This is a Go template where the `.Series` and `.LabelMatchers` string values
49 | # are available, and the delimiters are `<<` and `>>` to avoid conflicts with
50 | # the prometheus query language
51 | metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[2m])) by (<<.GroupBy>>)"
52 | ```
53 |
54 | Discovery
55 | ---------
56 |
57 | Discovery governs the process of finding the metrics that you want to
58 | expose in the custom metrics API. There are two fields that factor into
59 | discovery: `seriesQuery` and `seriesFilters`.
60 |
61 | `seriesQuery` specifies Prometheus series query (as passed to the
62 | `/api/v1/series` endpoint in Prometheus) to use to find some set of
63 | Prometheus series. The adapter will strip the label values from this
64 | series, and then use the resulting metric-name-label-names combinations
65 | later on.
66 |
67 | In many cases, `seriesQuery` will be sufficient to narrow down the list of
68 | Prometheus series. However, sometimes (especially if two rules might
69 | otherwise overlap), it's useful to do additional filtering on metric
70 | names. In this case, `seriesFilters` can be used. After the list of
71 | series is returned from `seriesQuery`, each series has its metric name
72 | filtered through any specified filters.
73 |
74 | Filters may be either:
75 |
76 | - `is: `, which matches any series whose name matches the specified
77 | regex.
78 |
79 | - `isNot: `, which matches any series whose name does not match the
80 | specified regex.
81 |
82 | For example:
83 |
84 | ```yaml
85 | # match all cAdvisor metrics that aren't measured in seconds
86 | seriesQuery: '{__name__=~"^container_.*_total",container!="POD",namespace!="",pod!=""}'
87 | seriesFilters:
88 | - isNot: "^container_.*_seconds_total"
89 | ```
90 |
91 | Association
92 | -----------
93 |
94 | Association governs the process of figuring out which Kubernetes resources
95 | a particular metric could be attached to. The `resources` field controls
96 | this process.
97 |
98 | There are two ways to associate resources with a particular metric. In
99 | both cases, the value of the label becomes the name of the particular
100 | object.
101 |
102 | One way is to specify that any label name that matches some particular
103 | pattern refers to some group-resource based on the label name. This can
104 | be done using the `template` field. The pattern is specified as a Go
105 | template, with the `Group` and `Resource` fields representing group and
106 | resource. You don't necessarily have to use the `Group` field (in which
107 | case the group is guessed by the system). For instance:
108 |
109 | ```yaml
110 | # any label `kube__` becomes . in Kubernetes
111 | resources:
112 | template: "kube_<<.Group>>_<<.Resource>>"
113 | ```
114 |
115 | The other way is to specify that some particular label represents some
116 | particular Kubernetes resource. This can be done using the `overrides`
117 | field. Each override maps a Prometheus label to a Kubernetes
118 | group-resource. For instance:
119 |
120 | ```yaml
121 | # the microservice label corresponds to the apps.deployment resource
122 | resources:
123 | overrides:
124 | microservice: {group: "apps", resource: "deployment"}
125 | ```
126 |
127 | These two can be combined, so you can specify both a template and some
128 | individual overrides.
129 |
130 | The resources mentioned can be any resource available in your kubernetes
131 | cluster, as long as you've got a corresponding label.
132 |
133 | Naming
134 | ------
135 |
136 | Naming governs the process of converting a Prometheus metric name into
137 | a metric in the custom metrics API, and vice versa. It's controlled by
138 | the `name` field.
139 |
140 | Naming is controlled by specifying a pattern to extract an API name from
141 | a Prometheus name, and potentially a transformation on that extracted
142 | value.
143 |
144 | The pattern is specified in the `matches` field, and is just a regular
145 | expression. If not specified, it defaults to `.*`.
146 |
147 | The transformation is specified by the `as` field. You can use any
148 | capture groups defined in the `matches` field. If the `matches` field
149 | doesn't contain capture groups, the `as` field defaults to `$0`. If it
150 | contains a single capture group, the `as` field defautls to `$1`.
151 | Otherwise, it's an error not to specify the as field.
152 |
153 | For example:
154 |
155 | ```yaml
156 | # match turn any name _total to _per_second
157 | # e.g. http_requests_total becomes http_requests_per_second
158 | name:
159 | matches: "^(.*)_total$"
160 | as: "${1}_per_second"
161 | ```
162 |
163 | Querying
164 | --------
165 |
166 | Querying governs the process of actually fetching values for a particular
167 | metric. It's controlled by the `metricsQuery` field.
168 |
169 | The `metricsQuery` field is a Go template that gets turned into
170 | a Prometheus query, using input from a particular call to the custom
171 | metrics API. A given call to the custom metrics API is distilled down to
172 | a metric name, a group-resource, and one or more objects of that
173 | group-resource. These get turned into the following fields in the
174 | template:
175 |
176 | - `Series`: the metric name
177 | - `LabelMatchers`: a comma-separated list of label matchers matching the
178 | given objects. Currently, this is the label for the particular
179 | group-resource, plus the label for namespace, if the group-resource is
180 | namespaced.
181 | - `GroupBy`: a comma-separated list of labels to group by. Currently,
182 | this contains the group-resource label used in `LabelMatchers`.
183 |
184 | For instance, suppose we had a series `http_requests_total` (exposed as
185 | `http_requests_per_second` in the API) with labels `service`, `pod`,
186 | `ingress`, `namespace`, and `verb`. The first four correspond to
187 | Kubernetes resources. Then, if someone requested the metric
188 | `pods/http_request_per_second` for the pods `pod1` and `pod2` in the
189 | `somens` namespace, we'd have:
190 |
191 | - `Series: "http_requests_total"`
192 | - `LabelMatchers: "pod=~\"pod1|pod2",namespace="somens"`
193 | - `GroupBy`: `pod`
194 |
195 | Additionally, there are two advanced fields that are "raw" forms of other
196 | fields:
197 |
198 | - `LabelValuesByName`: a map mapping the labels and values from the
199 | `LabelMatchers` field. The values are pre-joined by `|`
200 | (for used with the `=~` matcher in Prometheus).
201 | - `GroupBySlice`: the slice form of `GroupBy`.
202 |
203 | In general, you'll probably want to use the `Series`, `LabelMatchers`, and
204 | `GroupBy` fields. The other two are for advanced usage.
205 |
206 | The query is expected to return one value for each object requested. The
207 | adapter will use the labels on the returned series to associate a given
208 | series back to its corresponding object.
209 |
210 | For example:
211 |
212 | ```yaml
213 | # convert cumulative cAdvisor metrics into rates calculated over 2 minutes
214 | metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[2m])) by (<<.GroupBy>>)"
215 | ```
216 |
--------------------------------------------------------------------------------
/docs/externalmetrics.md:
--------------------------------------------------------------------------------
1 | External Metrics
2 | ===========
3 |
4 | It's possible to configure [Autoscaling on metrics not related to Kubernetes objects](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects) in Kubernetes. This is done with a special `External Metrics` system. Using external metrics in Kubernetes with the adapter requires you to configure special `external` rules in the configuration.
5 |
6 | The configuration for `external` metrics rules is almost identical to the normal `rules`:
7 |
8 | ```yaml
9 | externalRules:
10 | - seriesQuery: '{__name__="queue_consumer_lag",name!=""}'
11 | metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name)
12 | resources:
13 | overrides: { namespace: {resource: "namespace"} }
14 | ```
15 |
16 | Namespacing
17 | -----------
18 |
19 | All Kubernetes Horizontal Pod Autoscaler (HPA) resources are namespaced. And when you create an HPA that
20 | references an external metric the adapter will automatically add a `namespace` label to the `seriesQuery` you have configured.
21 |
22 | This is done because the External Merics API Specification *requires* a namespace component in the URL:
23 |
24 | ```shell
25 | kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/queue_consumer_lag"
26 | ```
27 |
28 | Cross-Namespace or No Namespace Queries
29 | ---------------------------------------
30 |
31 | A semi-common scenario is to have a `workload` in one namespace that needs to scale based on a metric from a different namespace. This is normally not
32 | possible with `external` rules because the `namespace` label is set to match that of the source `workload`.
33 |
34 | However, you can explicitly disable the automatic add of the HPA namepace to the query, and instead opt to not set a namespace at all, or to target a different namespace.
35 |
36 | This is done by setting `namespaced: false` in the `resources` section of the `external` rule:
37 |
38 | ```yaml
39 | # rules: ...
40 |
41 | externalRules:
42 | - seriesQuery: '{__name__="queue_depth",name!=""}'
43 | metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name)
44 | resources:
45 | namespaced: false
46 | ```
47 |
48 | Given the `external` rules defined above any `External` metric query for `queue_depth` will simply ignore the source `namespace` of the HPA. This allows you to explicilty not put a namespace into an external query, or to set the namespace to one that might be different from that of the HPA.
49 |
50 | ```yaml
51 | apiVersion: autoscaling/v1
52 | kind: HorizontalPodAutoscaler
53 | metadata:
54 | name: external-queue-scaler
55 | # the HPA and scaleTargetRef must exist in a namespace
56 | namespace: default
57 | annotations:
58 | # The "External" metric below targets a metricName that has namespaced=false
59 | # and this allows the metric to explicitly query a different
60 | # namespace than that of the HPA and scaleTargetRef
61 | autoscaling.alpha.kubernetes.io/metrics: |
62 | [
63 | {
64 | "type": "External",
65 | "external": {
66 | "metricName": "queue_depth",
67 | "metricSelector": {
68 | "matchLabels": {
69 | "namespace": "queue",
70 | "name": "my-sample-queue"
71 | }
72 | },
73 | "targetAverageValue": "50"
74 | }
75 | }
76 | ]
77 | spec:
78 | maxReplicas: 5
79 | minReplicas: 1
80 | scaleTargetRef:
81 | apiVersion: apps/v1
82 | kind: Deployment
83 | name: my-app
84 | ```
85 |
--------------------------------------------------------------------------------
/docs/sample-config.yaml:
--------------------------------------------------------------------------------
1 | rules:
2 | # Each rule represents a some naming and discovery logic.
3 | # Each rule is executed independently of the others, so
4 | # take care to avoid overlap. As an optimization, rules
5 | # with the same `seriesQuery` but different
6 | # `name` or `seriesFilters` will use only one query to
7 | # Prometheus for discovery.
8 |
9 | # some of these rules are taken from the "default" configuration, which
10 | # can be found in pkg/config/default.go
11 |
12 | # this rule matches cumulative cAdvisor metrics measured in seconds
13 | - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
14 | resources:
15 | # skip specifying generic resource<->label mappings, and just
16 | # attach only pod and namespace resources by mapping label names to group-resources
17 | overrides:
18 | namespace: {resource: "namespace"}
19 | pod: {resource: "pod"}
20 | # specify that the `container_` and `_seconds_total` suffixes should be removed.
21 | # this also introduces an implicit filter on metric family names
22 | name:
23 | # we use the value of the capture group implicitly as the API name
24 | # we could also explicitly write `as: "$1"`
25 | matches: "^container_(.*)_seconds_total$"
26 | # specify how to construct a query to fetch samples for a given series
27 | # This is a Go template where the `.Series` and `.LabelMatchers` string values
28 | # are available, and the delimiters are `<<` and `>>` to avoid conflicts with
29 | # the prometheus query language
30 | metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[2m])) by (<<.GroupBy>>)"
31 |
32 | # this rule matches cumulative cAdvisor metrics not measured in seconds
33 | - seriesQuery: '{__name__=~"^container_.*_total",container!="POD",namespace!="",pod!=""}'
34 | resources:
35 | overrides:
36 | namespace: {resource: "namespace"}
37 | pod: {resource: "pod"}
38 | seriesFilters:
39 | # since this is a superset of the query above, we introduce an additional filter here
40 | - isNot: "^container_.*_seconds_total$"
41 | name: {matches: "^container_(.*)_total$"}
42 | metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[2m])) by (<<.GroupBy>>)"
43 |
44 | # this rule matches cumulative non-cAdvisor metrics
45 | - seriesQuery: '{namespace!="",__name__!="^container_.*"}'
46 | name: {matches: "^(.*)_total$"}
47 | resources:
48 | # specify an a generic mapping between resources and labels. This
49 | # is a template, like the `metricsQuery` template, except with the `.Group`
50 | # and `.Resource` strings available. It will also be used to match labels,
51 | # so avoid using template functions which truncate the group or resource.
52 | # Group will be converted to a form acceptible for use as a label automatically.
53 | template: "<<.Resource>>"
54 | # if we wanted to, we could also specify overrides here
55 | metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[2m])) by (<<.GroupBy>>)"
56 |
57 | # this rule matches only a single metric, explicitly naming it something else
58 | # It's series query *must* return only a single metric family
59 | - seriesQuery: 'cheddar{sharp="true"}'
60 | # this metric will appear as "cheesy_goodness" in the custom metrics API
61 | name: {as: "cheesy_goodness"}
62 | resources:
63 | overrides:
64 | # this should still resolve in our cluster
65 | brand: {group: "cheese.io", resource: "brand"}
66 | metricsQuery: 'count(cheddar{sharp="true"})'
67 |
68 | # external rules are not tied to a Kubernetes resource and can reference any metric
69 | # https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects
70 | externalRules:
71 | - seriesQuery: '{__name__="queue_consumer_lag",name!=""}'
72 | metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name)
73 | - seriesQuery: '{__name__="queue_depth",topic!=""}'
74 | metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name)
75 | # Kubernetes metric queries include a namespace in the query by default
76 | # but you can explicitly disable namespaces if needed with "namespaced: false"
77 | # this is useful if you have an HPA with an external metric in namespace A
78 | # but want to query for metrics from namespace B
79 | resources:
80 | namespaced: false
81 |
82 | # TODO: should we be able to map to a constant instance of a resource
83 | # (e.g. `resources: {constant: [{resource: "namespace", name: "kube-system"}}]`)?
84 |
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module sigs.k8s.io/prometheus-adapter
2 |
3 | go 1.22.1
4 |
5 | toolchain go1.22.2
6 |
7 | require (
8 | github.com/onsi/ginkgo v1.16.5
9 | github.com/onsi/gomega v1.33.1
10 | github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.73.2
11 | github.com/prometheus-operator/prometheus-operator/pkg/client v0.73.2
12 | github.com/prometheus/client_golang v1.18.0
13 | github.com/prometheus/common v0.46.0
14 | github.com/spf13/cobra v1.8.0
15 | github.com/stretchr/testify v1.9.0
16 | gopkg.in/yaml.v2 v2.4.0
17 | k8s.io/api v0.30.0
18 | k8s.io/apimachinery v0.30.0
19 | k8s.io/apiserver v0.30.0
20 | k8s.io/client-go v0.30.0
21 | k8s.io/component-base v0.30.0
22 | k8s.io/klog/v2 v2.120.1
23 | k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f
24 | k8s.io/metrics v0.30.0
25 | sigs.k8s.io/custom-metrics-apiserver v1.30.0
26 | sigs.k8s.io/metrics-server v0.7.1
27 | )
28 |
29 | require (
30 | github.com/NYTimes/gziphandler v1.1.1 // indirect
31 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
32 | github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
33 | github.com/beorn7/perks v1.0.1 // indirect
34 | github.com/blang/semver/v4 v4.0.0 // indirect
35 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect
36 | github.com/cespare/xxhash/v2 v2.2.0 // indirect
37 | github.com/coreos/go-semver v0.3.1 // indirect
38 | github.com/coreos/go-systemd/v22 v22.5.0 // indirect
39 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
40 | github.com/emicklei/go-restful/v3 v3.12.0 // indirect
41 | github.com/evanphx/json-patch v5.9.0+incompatible // indirect
42 | github.com/felixge/httpsnoop v1.0.4 // indirect
43 | github.com/fsnotify/fsnotify v1.7.0 // indirect
44 | github.com/go-logr/logr v1.4.1 // indirect
45 | github.com/go-logr/stdr v1.2.2 // indirect
46 | github.com/go-openapi/jsonpointer v0.21.0 // indirect
47 | github.com/go-openapi/jsonreference v0.21.0 // indirect
48 | github.com/go-openapi/swag v0.23.0 // indirect
49 | github.com/gogo/protobuf v1.3.2 // indirect
50 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
51 | github.com/golang/protobuf v1.5.4 // indirect
52 | github.com/google/cel-go v0.17.8 // indirect
53 | github.com/google/gnostic-models v0.6.8 // indirect
54 | github.com/google/go-cmp v0.6.0 // indirect
55 | github.com/google/gofuzz v1.2.0 // indirect
56 | github.com/google/uuid v1.6.0 // indirect
57 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
58 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 // indirect
59 | github.com/imdario/mergo v0.3.16 // indirect
60 | github.com/inconshreveable/mousetrap v1.1.0 // indirect
61 | github.com/josharian/intern v1.0.0 // indirect
62 | github.com/json-iterator/go v1.1.12 // indirect
63 | github.com/mailru/easyjson v0.7.7 // indirect
64 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
65 | github.com/modern-go/reflect2 v1.0.2 // indirect
66 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
67 | github.com/nxadm/tail v1.4.8 // indirect
68 | github.com/pkg/errors v0.9.1 // indirect
69 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
70 | github.com/prometheus/client_model v0.5.0 // indirect
71 | github.com/prometheus/procfs v0.12.0 // indirect
72 | github.com/spf13/pflag v1.0.5 // indirect
73 | github.com/stoewer/go-strcase v1.3.0 // indirect
74 | go.etcd.io/etcd/api/v3 v3.5.11 // indirect
75 | go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect
76 | go.etcd.io/etcd/client/v3 v3.5.11 // indirect
77 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
78 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
79 | go.opentelemetry.io/otel v1.21.0 // indirect
80 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
81 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
82 | go.opentelemetry.io/otel/metric v1.21.0 // indirect
83 | go.opentelemetry.io/otel/sdk v1.21.0 // indirect
84 | go.opentelemetry.io/otel/trace v1.21.0 // indirect
85 | go.opentelemetry.io/proto/otlp v1.0.0 // indirect
86 | go.uber.org/multierr v1.11.0 // indirect
87 | go.uber.org/zap v1.26.0 // indirect
88 | golang.org/x/crypto v0.31.0 // indirect
89 | golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect
90 | golang.org/x/mod v0.17.0 // indirect
91 | golang.org/x/net v0.25.0 // indirect
92 | golang.org/x/oauth2 v0.18.0 // indirect
93 | golang.org/x/sync v0.10.0 // indirect
94 | golang.org/x/sys v0.28.0 // indirect
95 | golang.org/x/term v0.27.0 // indirect
96 | golang.org/x/text v0.21.0 // indirect
97 | golang.org/x/time v0.5.0 // indirect
98 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
99 | google.golang.org/appengine v1.6.8 // indirect
100 | google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 // indirect
101 | google.golang.org/genproto/googleapis/api v0.0.0-20231212172506-995d672761c0 // indirect
102 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect
103 | google.golang.org/grpc v1.60.1 // indirect
104 | google.golang.org/protobuf v1.33.0 // indirect
105 | gopkg.in/inf.v0 v0.9.1 // indirect
106 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
107 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
108 | gopkg.in/yaml.v3 v3.0.1 // indirect
109 | k8s.io/apiextensions-apiserver v0.29.3 // indirect
110 | k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect
111 | k8s.io/kms v0.30.0 // indirect
112 | k8s.io/utils v0.0.0-20240423183400-0849a56e8f22 // indirect
113 | sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect
114 | sigs.k8s.io/controller-runtime v0.17.2 // indirect
115 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
116 | sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
117 | sigs.k8s.io/yaml v1.4.0 // indirect
118 | )
119 |
--------------------------------------------------------------------------------
/hack/boilerplate.go.txt:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
--------------------------------------------------------------------------------
/hack/tools.go:
--------------------------------------------------------------------------------
1 | // Copyright 2021 The Kubernetes Authors.
2 | //
3 | // Licensed under the Apache License, Version 2.0 (the "License");
4 | // you may not use this file except in compliance with the License.
5 | // You may obtain a copy of the License at
6 | //
7 | // http://www.apache.org/licenses/LICENSE-2.0
8 | //
9 | // Unless required by applicable law or agreed to in writing, software
10 | // distributed under the License is distributed on an "AS IS" BASIS,
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | // See the License for the specific language governing permissions and
13 | // limitations under the License.
14 |
15 | //go:build tools
16 | // +build tools
17 |
18 | // Package tools tracks dependencies for tools that used in the build process.
19 | // See https://github.com/golang/go/wiki/Modules
20 | package tools
21 |
22 | import (
23 | _ "k8s.io/kube-openapi/cmd/openapi-gen"
24 | )
25 |
--------------------------------------------------------------------------------
/pkg/api/openapi-gen.go:
--------------------------------------------------------------------------------
1 | //go:build codegen
2 | // +build codegen
3 |
4 | /*
5 | Copyright 2018 The Kubernetes Authors.
6 |
7 | Licensed under the Apache License, Version 2.0 (the "License");
8 | you may not use this file except in compliance with the License.
9 | You may obtain a copy of the License at
10 |
11 | http://www.apache.org/licenses/LICENSE-2.0
12 |
13 | Unless required by applicable law or agreed to in writing, software
14 | distributed under the License is distributed on an "AS IS" BASIS,
15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 | See the License for the specific language governing permissions and
17 | limitations under the License.
18 | */
19 |
20 | // Package is only a stub to ensure k8s.io/kube-openapi/cmd/openapi-gen is vendored
21 | // so the same version of kube-openapi is used to generate and render the openapi spec
22 | package main
23 |
24 | import (
25 | _ "k8s.io/kube-openapi/cmd/openapi-gen"
26 | )
27 |
--------------------------------------------------------------------------------
/pkg/client/api.go:
--------------------------------------------------------------------------------
1 | // Copyright 2017 The Prometheus Authors
2 | // Licensed under the Apache License, Version 2.0 (the "License");
3 | // you may not use this file except in compliance with the License.
4 | // You may obtain a copy of the License at
5 | //
6 | // http://www.apache.org/licenses/LICENSE-2.0
7 | //
8 | // Unless required by applicable law or agreed to in writing, software
9 | // distributed under the License is distributed on an "AS IS" BASIS,
10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | // See the License for the specific language governing permissions and
12 | // limitations under the License.
13 |
14 | // Package prometheus provides bindings to the Prometheus HTTP API:
15 | // http://prometheus.io/docs/querying/api/
16 | package client
17 |
18 | import (
19 | "bytes"
20 | "context"
21 | "encoding/json"
22 | "fmt"
23 | "io"
24 | "net/http"
25 | "net/url"
26 | "path"
27 | "strings"
28 | "time"
29 |
30 | "github.com/prometheus/common/model"
31 | "k8s.io/klog/v2"
32 | )
33 |
34 | // APIClient is a raw client to the Prometheus Query API.
35 | // It knows how to appropriately deal with generic Prometheus API
36 | // responses, but does not know the specifics of different endpoints.
37 | // You can use this to call query endpoints not represented in Client.
38 | type GenericAPIClient interface {
39 | // Do makes a request to the Prometheus HTTP API against a particular endpoint. Query
40 | // parameters should be in `query`, not `endpoint`. An error will be returned on HTTP
41 | // status errors or errors making or unmarshalling the request, as well as when the
42 | // response has a Status of ResponseError.
43 | Do(ctx context.Context, verb, endpoint string, query url.Values) (APIResponse, error)
44 | }
45 |
46 | // httpAPIClient is a GenericAPIClient implemented in terms of an underlying http.Client.
47 | type httpAPIClient struct {
48 | client *http.Client
49 | baseURL *url.URL
50 | headers http.Header
51 | }
52 |
53 | func (c *httpAPIClient) Do(ctx context.Context, verb, endpoint string, query url.Values) (APIResponse, error) {
54 | u := *c.baseURL
55 | u.Path = path.Join(c.baseURL.Path, endpoint)
56 | var reqBody io.Reader
57 | if verb == http.MethodGet {
58 | u.RawQuery = query.Encode()
59 | } else if verb == http.MethodPost {
60 | reqBody = strings.NewReader(query.Encode())
61 | }
62 |
63 | req, err := http.NewRequestWithContext(ctx, verb, u.String(), reqBody)
64 | if err != nil {
65 | return APIResponse{}, fmt.Errorf("error constructing HTTP request to Prometheus: %v", err)
66 | }
67 | for key, values := range c.headers {
68 | for _, value := range values {
69 | req.Header.Add(key, value)
70 | }
71 | }
72 | if verb == http.MethodPost {
73 | req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
74 | }
75 |
76 | resp, err := c.client.Do(req)
77 | defer func() {
78 | if resp != nil {
79 | resp.Body.Close()
80 | }
81 | }()
82 |
83 | if err != nil {
84 | return APIResponse{}, err
85 | }
86 |
87 | if klog.V(6).Enabled() {
88 | klog.Infof("%s %s %s", verb, u.String(), resp.Status)
89 | }
90 |
91 | code := resp.StatusCode
92 |
93 | // codes that aren't 2xx, 400, 422, or 503 won't return JSON objects
94 | if code/100 != 2 && code != 400 && code != 422 && code != 503 {
95 | return APIResponse{}, &Error{
96 | Type: ErrBadResponse,
97 | Msg: fmt.Sprintf("unknown response code %d", code),
98 | }
99 | }
100 |
101 | var body io.Reader = resp.Body
102 | if klog.V(8).Enabled() {
103 | data, err := io.ReadAll(body)
104 | if err != nil {
105 | return APIResponse{}, fmt.Errorf("unable to log response body: %v", err)
106 | }
107 | klog.Infof("Response Body: %s", string(data))
108 | body = bytes.NewReader(data)
109 | }
110 |
111 | var res APIResponse
112 | if err = json.NewDecoder(body).Decode(&res); err != nil {
113 | return APIResponse{}, &Error{
114 | Type: ErrBadResponse,
115 | Msg: err.Error(),
116 | }
117 | }
118 |
119 | if res.Status == ResponseError {
120 | return res, &Error{
121 | Type: res.ErrorType,
122 | Msg: res.Error,
123 | }
124 | }
125 |
126 | return res, nil
127 | }
128 |
129 | // NewGenericAPIClient builds a new generic Prometheus API client for the given base URL and HTTP Client.
130 | func NewGenericAPIClient(client *http.Client, baseURL *url.URL, headers http.Header) GenericAPIClient {
131 | return &httpAPIClient{
132 | client: client,
133 | baseURL: baseURL,
134 | headers: headers,
135 | }
136 | }
137 |
138 | const (
139 | queryURL = "/api/v1/query"
140 | queryRangeURL = "/api/v1/query_range"
141 | seriesURL = "/api/v1/series"
142 | )
143 |
144 | // queryClient is a Client that connects to the Prometheus HTTP API.
145 | type queryClient struct {
146 | api GenericAPIClient
147 | verb string
148 | }
149 |
150 | // NewClientForAPI creates a Client for the given generic Prometheus API client.
151 | func NewClientForAPI(client GenericAPIClient, verb string) Client {
152 | return &queryClient{
153 | api: client,
154 | verb: verb,
155 | }
156 | }
157 |
158 | // NewClient creates a Client for the given HTTP client and base URL (the location of the Prometheus server).
159 | func NewClient(client *http.Client, baseURL *url.URL, headers http.Header, verb string) Client {
160 | genericClient := NewGenericAPIClient(client, baseURL, headers)
161 | return NewClientForAPI(genericClient, verb)
162 | }
163 |
164 | func (h *queryClient) Series(ctx context.Context, interval model.Interval, selectors ...Selector) ([]Series, error) {
165 | vals := url.Values{}
166 | if interval.Start != 0 {
167 | vals.Set("start", interval.Start.String())
168 | }
169 | if interval.End != 0 {
170 | vals.Set("end", interval.End.String())
171 | }
172 |
173 | for _, selector := range selectors {
174 | vals.Add("match[]", string(selector))
175 | }
176 |
177 | res, err := h.api.Do(ctx, h.verb, seriesURL, vals)
178 | if err != nil {
179 | return nil, err
180 | }
181 |
182 | var seriesRes []Series
183 | err = json.Unmarshal(res.Data, &seriesRes)
184 | return seriesRes, err
185 | }
186 |
187 | func (h *queryClient) Query(ctx context.Context, t model.Time, query Selector) (QueryResult, error) {
188 | vals := url.Values{}
189 | vals.Set("query", string(query))
190 | if t != 0 {
191 | vals.Set("time", t.String())
192 | }
193 | if timeout, hasTimeout := timeoutFromContext(ctx); hasTimeout {
194 | vals.Set("timeout", model.Duration(timeout).String())
195 | }
196 |
197 | res, err := h.api.Do(ctx, h.verb, queryURL, vals)
198 | if err != nil {
199 | return QueryResult{}, err
200 | }
201 |
202 | var queryRes QueryResult
203 | err = json.Unmarshal(res.Data, &queryRes)
204 | return queryRes, err
205 | }
206 |
207 | func (h *queryClient) QueryRange(ctx context.Context, r Range, query Selector) (QueryResult, error) {
208 | vals := url.Values{}
209 | vals.Set("query", string(query))
210 |
211 | if r.Start != 0 {
212 | vals.Set("start", r.Start.String())
213 | }
214 | if r.End != 0 {
215 | vals.Set("end", r.End.String())
216 | }
217 | if r.Step != 0 {
218 | vals.Set("step", model.Duration(r.Step).String())
219 | }
220 | if timeout, hasTimeout := timeoutFromContext(ctx); hasTimeout {
221 | vals.Set("timeout", model.Duration(timeout).String())
222 | }
223 |
224 | res, err := h.api.Do(ctx, h.verb, queryRangeURL, vals)
225 | if err != nil {
226 | return QueryResult{}, err
227 | }
228 |
229 | var queryRes QueryResult
230 | err = json.Unmarshal(res.Data, &queryRes)
231 | return queryRes, err
232 | }
233 |
234 | // timeoutFromContext checks the context for a deadline and calculates a "timeout" duration from it,
235 | // when present
236 | func timeoutFromContext(ctx context.Context) (time.Duration, bool) {
237 | if deadline, hasDeadline := ctx.Deadline(); hasDeadline {
238 | return time.Since(deadline), true
239 | }
240 |
241 | return time.Duration(0), false
242 | }
243 |
--------------------------------------------------------------------------------
/pkg/client/fake/client.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2018 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package fake
18 |
19 | import (
20 | "context"
21 | "fmt"
22 |
23 | pmodel "github.com/prometheus/common/model"
24 |
25 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
26 | )
27 |
28 | // FakePrometheusClient is a fake instance of prom.Client
29 | type FakePrometheusClient struct {
30 | // AcceptableInterval is the interval in which to return queries
31 | AcceptableInterval pmodel.Interval
32 | // ErrQueries are queries that result in an error (whether from Query or Series)
33 | ErrQueries map[prom.Selector]error
34 | // Series are non-error responses to partial Series calls
35 | SeriesResults map[prom.Selector][]prom.Series
36 | // QueryResults are non-error responses to Query
37 | QueryResults map[prom.Selector]prom.QueryResult
38 | }
39 |
40 | func (c *FakePrometheusClient) Series(_ context.Context, interval pmodel.Interval, selectors ...prom.Selector) ([]prom.Series, error) {
41 | if (interval.Start != 0 && interval.Start < c.AcceptableInterval.Start) || (interval.End != 0 && interval.End > c.AcceptableInterval.End) {
42 | return nil, fmt.Errorf("interval [%v, %v] for query is outside range [%v, %v]", interval.Start, interval.End, c.AcceptableInterval.Start, c.AcceptableInterval.End)
43 | }
44 | res := []prom.Series{}
45 | for _, sel := range selectors {
46 | if err, found := c.ErrQueries[sel]; found {
47 | return nil, err
48 | }
49 | if series, found := c.SeriesResults[sel]; found {
50 | res = append(res, series...)
51 | }
52 | }
53 |
54 | return res, nil
55 | }
56 |
57 | func (c *FakePrometheusClient) Query(_ context.Context, t pmodel.Time, query prom.Selector) (prom.QueryResult, error) {
58 | if t < c.AcceptableInterval.Start || t > c.AcceptableInterval.End {
59 | return prom.QueryResult{}, fmt.Errorf("time %v for query is outside range [%v, %v]", t, c.AcceptableInterval.Start, c.AcceptableInterval.End)
60 | }
61 |
62 | if err, found := c.ErrQueries[query]; found {
63 | return prom.QueryResult{}, err
64 | }
65 |
66 | if res, found := c.QueryResults[query]; found {
67 | return res, nil
68 | }
69 |
70 | return prom.QueryResult{
71 | Type: pmodel.ValVector,
72 | Vector: &pmodel.Vector{},
73 | }, nil
74 | }
75 |
76 | func (c *FakePrometheusClient) QueryRange(_ context.Context, r prom.Range, query prom.Selector) (prom.QueryResult, error) {
77 | return prom.QueryResult{}, nil
78 | }
79 |
--------------------------------------------------------------------------------
/pkg/client/helpers.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package client
18 |
19 | import (
20 | "fmt"
21 | "strings"
22 | )
23 |
24 | // LabelNeq produces a not-equal label selector expression.
25 | // Label is passed verbatim, and value is double-quote escaped
26 | // using Go's escaping (as per the PromQL rules).
27 | func LabelNeq(label string, value string) string {
28 | return fmt.Sprintf("%s!=%q", label, value)
29 | }
30 |
31 | // LabelEq produces a equal label selector expression.
32 | // Label is passed verbatim, and value is double-quote escaped
33 | // using Go's escaping (as per the PromQL rules).
34 | func LabelEq(label string, value string) string {
35 | return fmt.Sprintf("%s=%q", label, value)
36 | }
37 |
38 | // LabelMatches produces a regexp-matching label selector expression.
39 | // It has similar constraints to LabelNeq.
40 | func LabelMatches(label string, expr string) string {
41 | return fmt.Sprintf("%s=~%q", label, expr)
42 | }
43 |
44 | // LabelNotMatches produces a inverse regexp-matching label selector expression (the opposite of LabelMatches).
45 | func LabelNotMatches(label string, expr string) string {
46 | return fmt.Sprintf("%s!~%q", label, expr)
47 | }
48 |
49 | // NameMatches produces a label selector expression that checks that the series name matches the given expression.
50 | // It's a convinience wrapper around LabelMatches.
51 | func NameMatches(expr string) string {
52 | return LabelMatches("__name__", expr)
53 | }
54 |
55 | // NameNotMatches produces a label selector expression that checks that the series name doesn't matches the given expression.
56 | // It's a convenience wrapper around LabelNotMatches.
57 | func NameNotMatches(expr string) string {
58 | return LabelNotMatches("__name__", expr)
59 | }
60 |
61 | // MatchSeries takes a series name, and optionally some label expressions, and returns a series selector.
62 | // TODO: validate series name and expressions?
63 | func MatchSeries(name string, labelExpressions ...string) Selector {
64 | if len(labelExpressions) == 0 {
65 | return Selector(name)
66 | }
67 |
68 | return Selector(fmt.Sprintf("%s{%s}", name, strings.Join(labelExpressions, ",")))
69 | }
70 |
--------------------------------------------------------------------------------
/pkg/client/interfaces.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package client
18 |
19 | import (
20 | "context"
21 | "encoding/json"
22 | "fmt"
23 | "strings"
24 | "time"
25 |
26 | "github.com/prometheus/common/model"
27 | )
28 |
29 | // NB: the official prometheus API client at https://github.com/prometheus/client_golang
30 | // is rather lackluster -- as of the time of writing of this file, it lacked support
31 | // for querying the series metadata, which we need for the adapter. Instead, we use
32 | // this client.
33 |
34 | // Selector represents a series selector
35 | type Selector string
36 |
37 | // Range represents a sliced time range with increments.
38 | type Range struct {
39 | // Start and End are the boundaries of the time range.
40 | Start, End model.Time
41 | // Step is the maximum time between two slices within the boundaries.
42 | Step time.Duration
43 | }
44 |
45 | // Client is a Prometheus client for the Prometheus HTTP API.
46 | // The "timeout" parameter for the HTTP API is set based on the context's deadline,
47 | // when present and applicable.
48 | type Client interface {
49 | // Series lists the time series matching the given series selectors
50 | Series(ctx context.Context, interval model.Interval, selectors ...Selector) ([]Series, error)
51 | // Query runs a non-range query at the given time.
52 | Query(ctx context.Context, t model.Time, query Selector) (QueryResult, error)
53 | // QueryRange runs a range query at the given time.
54 | QueryRange(ctx context.Context, r Range, query Selector) (QueryResult, error)
55 | }
56 |
57 | // QueryResult is the result of a query.
58 | // Type will always be set, as well as one of the other fields, matching the type.
59 | type QueryResult struct {
60 | Type model.ValueType
61 |
62 | Vector *model.Vector
63 | Scalar *model.Scalar
64 | Matrix *model.Matrix
65 | }
66 |
67 | func (qr *QueryResult) UnmarshalJSON(b []byte) error {
68 | v := struct {
69 | Type model.ValueType `json:"resultType"`
70 | Result json.RawMessage `json:"result"`
71 | }{}
72 |
73 | err := json.Unmarshal(b, &v)
74 | if err != nil {
75 | return err
76 | }
77 |
78 | qr.Type = v.Type
79 |
80 | switch v.Type {
81 | case model.ValScalar:
82 | var sv model.Scalar
83 | err = json.Unmarshal(v.Result, &sv)
84 | qr.Scalar = &sv
85 |
86 | case model.ValVector:
87 | var vv model.Vector
88 | err = json.Unmarshal(v.Result, &vv)
89 | qr.Vector = &vv
90 |
91 | case model.ValMatrix:
92 | var mv model.Matrix
93 | err = json.Unmarshal(v.Result, &mv)
94 | qr.Matrix = &mv
95 |
96 | default:
97 | err = fmt.Errorf("unexpected value type %q", v.Type)
98 | }
99 | return err
100 | }
101 |
102 | // Series represents a description of a series: a name and a set of labels.
103 | // Series is roughly equivalent to model.Metrics, but has easy access to name
104 | // and the set of non-name labels.
105 | type Series struct {
106 | Name string
107 | Labels model.LabelSet
108 | }
109 |
110 | func (s *Series) UnmarshalJSON(data []byte) error {
111 | var rawMetric model.Metric
112 | err := json.Unmarshal(data, &rawMetric)
113 | if err != nil {
114 | return err
115 | }
116 |
117 | if name, ok := rawMetric[model.MetricNameLabel]; ok {
118 | s.Name = string(name)
119 | delete(rawMetric, model.MetricNameLabel)
120 | }
121 |
122 | s.Labels = model.LabelSet(rawMetric)
123 |
124 | return nil
125 | }
126 |
127 | func (s *Series) String() string {
128 | lblStrings := make([]string, 0, len(s.Labels))
129 | for k, v := range s.Labels {
130 | lblStrings = append(lblStrings, fmt.Sprintf("%s=%q", k, v))
131 | }
132 | return fmt.Sprintf("%s{%s}", s.Name, strings.Join(lblStrings, ","))
133 | }
134 |
--------------------------------------------------------------------------------
/pkg/client/metrics/metrics.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package metrics
18 |
19 | import (
20 | "context"
21 | "net/http"
22 | "net/url"
23 | "time"
24 |
25 | "github.com/prometheus/client_golang/prometheus"
26 |
27 | apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
28 | "k8s.io/component-base/metrics"
29 | "k8s.io/component-base/metrics/legacyregistry"
30 |
31 | "sigs.k8s.io/prometheus-adapter/pkg/client"
32 | )
33 |
34 | var (
35 | // queryLatency is the total latency of any query going through the
36 | // various endpoints (query, range-query, series). It includes some deserialization
37 | // overhead and HTTP overhead.
38 | queryLatency = metrics.NewHistogramVec(
39 | &metrics.HistogramOpts{
40 | Namespace: "prometheus_adapter",
41 | Subsystem: "prometheus_client",
42 | Name: "request_duration_seconds",
43 | Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server",
44 | Buckets: prometheus.DefBuckets,
45 | },
46 | []string{"path", "server"},
47 | )
48 | )
49 |
50 | func MetricsHandler() (http.HandlerFunc, error) {
51 | registry := metrics.NewKubeRegistry()
52 | err := registry.Register(queryLatency)
53 | if err != nil {
54 | return nil, err
55 | }
56 | apimetrics.Register()
57 | return func(w http.ResponseWriter, req *http.Request) {
58 | legacyregistry.Handler().ServeHTTP(w, req)
59 | metrics.HandlerFor(registry, metrics.HandlerOpts{}).ServeHTTP(w, req)
60 | }, nil
61 | }
62 |
63 | // instrumentedClient is a client.GenericAPIClient which instruments calls to Do,
64 | // capturing request latency.
65 | type instrumentedGenericClient struct {
66 | serverName string
67 | client client.GenericAPIClient
68 | }
69 |
70 | func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint string, query url.Values) (client.APIResponse, error) {
71 | startTime := time.Now()
72 | var err error
73 | defer func() {
74 | endTime := time.Now()
75 | // skip calls where we don't make the actual request
76 | if err != nil {
77 | if _, wasAPIErr := err.(*client.Error); !wasAPIErr {
78 | // TODO: measure API errors by code?
79 | return
80 | }
81 | }
82 | queryLatency.With(prometheus.Labels{"path": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds())
83 | }()
84 |
85 | var resp client.APIResponse
86 | resp, err = c.client.Do(ctx, verb, endpoint, query)
87 | return resp, err
88 | }
89 |
90 | func InstrumentGenericAPIClient(client client.GenericAPIClient, serverName string) client.GenericAPIClient {
91 | return &instrumentedGenericClient{
92 | serverName: serverName,
93 | client: client,
94 | }
95 | }
96 |
--------------------------------------------------------------------------------
/pkg/client/types.go:
--------------------------------------------------------------------------------
1 | // Copyright 2017 The Prometheus Authors
2 | // Licensed under the Apache License, Version 2.0 (the "License");
3 | // you may not use this file except in compliance with the License.
4 | // You may obtain a copy of the License at
5 | //
6 | // http://www.apache.org/licenses/LICENSE-2.0
7 | //
8 | // Unless required by applicable law or agreed to in writing, software
9 | // distributed under the License is distributed on an "AS IS" BASIS,
10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | // See the License for the specific language governing permissions and
12 | // limitations under the License.
13 |
14 | // Package prometheus provides bindings to the Prometheus HTTP API:
15 | // http://prometheus.io/docs/querying/api/
16 | package client
17 |
18 | import (
19 | "encoding/json"
20 | "fmt"
21 | )
22 |
23 | // ErrorType is the type of the API error.
24 | type ErrorType string
25 |
26 | const (
27 | ErrBadData ErrorType = "bad_data"
28 | ErrTimeout ErrorType = "timeout"
29 | ErrCanceled ErrorType = "canceled"
30 | ErrExec ErrorType = "execution"
31 | ErrBadResponse ErrorType = "bad_response"
32 | )
33 |
34 | // Error is an error returned by the API.
35 | type Error struct {
36 | Type ErrorType
37 | Msg string
38 | }
39 |
40 | func (e *Error) Error() string {
41 | return fmt.Sprintf("%s: %s", e.Type, e.Msg)
42 | }
43 |
44 | // ResponseStatus is the type of response from the API: succeeded or error.
45 | type ResponseStatus string
46 |
47 | const (
48 | ResponseSucceeded ResponseStatus = "succeeded"
49 | ResponseError ResponseStatus = "error"
50 | )
51 |
52 | // APIResponse represents the raw response returned by the API.
53 | type APIResponse struct {
54 | // Status indicates whether this request was successful or whether it errored out.
55 | Status ResponseStatus `json:"status"`
56 | // Data contains the raw data response for this request.
57 | Data json.RawMessage `json:"data"`
58 |
59 | // ErrorType is the type of error, if this is an error response.
60 | ErrorType ErrorType `json:"errorType"`
61 | // Error is the error message, if this is an error response.
62 | Error string `json:"error"`
63 | }
64 |
--------------------------------------------------------------------------------
/pkg/config/config.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | pmodel "github.com/prometheus/common/model"
5 | )
6 |
7 | type MetricsDiscoveryConfig struct {
8 | // Rules specifies how to discover and map Prometheus metrics to
9 | // custom metrics API resources. The rules are applied independently,
10 | // and thus must be mutually exclusive. Rules with the same SeriesQuery
11 | // will make only a single API call.
12 | Rules []DiscoveryRule `json:"rules" yaml:"rules"`
13 | ResourceRules *ResourceRules `json:"resourceRules,omitempty" yaml:"resourceRules,omitempty"`
14 | ExternalRules []DiscoveryRule `json:"externalRules,omitempty" yaml:"externalRules,omitempty"`
15 | }
16 |
17 | // DiscoveryRule describes a set of rules for transforming Prometheus metrics to/from
18 | // custom metrics API resources.
19 | type DiscoveryRule struct {
20 | // SeriesQuery specifies which metrics this rule should consider via a Prometheus query
21 | // series selector query.
22 | SeriesQuery string `json:"seriesQuery" yaml:"seriesQuery"`
23 | // SeriesFilters specifies additional regular expressions to be applied on
24 | // the series names returned from the query. This is useful for constraints
25 | // that can't be represented in the SeriesQuery (e.g. series matching `container_.+`
26 | // not matching `container_.+_total`. A filter will be automatically appended to
27 | // match the form specified in Name.
28 | SeriesFilters []RegexFilter `json:"seriesFilters" yaml:"seriesFilters"`
29 | // Resources specifies how associated Kubernetes resources should be discovered for
30 | // the given metrics.
31 | Resources ResourceMapping `json:"resources" yaml:"resources"`
32 | // Name specifies how the metric name should be transformed between custom metric
33 | // API resources, and Prometheus metric names.
34 | Name NameMapping `json:"name" yaml:"name"`
35 | // MetricsQuery specifies modifications to the metrics query, such as converting
36 | // cumulative metrics to rate metrics. It is a template where `.LabelMatchers` is
37 | // a the comma-separated base label matchers and `.Series` is the series name, and
38 | // `.GroupBy` is the comma-separated expected group-by label names. The delimeters
39 | // are `<<` and `>>`.
40 | MetricsQuery string `json:"metricsQuery,omitempty" yaml:"metricsQuery,omitempty"`
41 | }
42 |
43 | // RegexFilter is a filter that matches positively or negatively against a regex.
44 | // Only one field may be set at a time.
45 | type RegexFilter struct {
46 | Is string `json:"is,omitempty" yaml:"is,omitempty"`
47 | IsNot string `json:"isNot,omitempty" yaml:"isNot,omitempty"`
48 | }
49 |
50 | // ResourceMapping specifies how to map Kubernetes resources to Prometheus labels
51 | type ResourceMapping struct {
52 | // Template specifies a golang string template for converting a Kubernetes
53 | // group-resource to a Prometheus label. The template object contains
54 | // the `.Group` and `.Resource` fields. The `.Group` field will have
55 | // dots replaced with underscores, and the `.Resource` field will be
56 | // singularized. The delimiters are `<<` and `>>`.
57 | Template string `json:"template,omitempty" yaml:"template,omitempty"`
58 | // Overrides specifies exceptions to the above template, mapping label names
59 | // to group-resources
60 | Overrides map[string]GroupResource `json:"overrides,omitempty" yaml:"overrides,omitempty"`
61 | // Namespaced ignores the source namespace of the requester and requires one in the query
62 | Namespaced *bool `json:"namespaced,omitempty" yaml:"namespaced,omitempty"`
63 | }
64 |
65 | // GroupResource represents a Kubernetes group-resource.
66 | type GroupResource struct {
67 | Group string `json:"group,omitempty" yaml:"group,omitempty"`
68 | Resource string `json:"resource" yaml:"resource"`
69 | }
70 |
71 | // NameMapping specifies how to convert Prometheus metrics
72 | // to/from custom metrics API resources.
73 | type NameMapping struct {
74 | // Matches is a regular expression that is used to match
75 | // Prometheus series names. It may be left blank, in which
76 | // case it is equivalent to `.*`.
77 | Matches string `json:"matches" yaml:"matches"`
78 | // As is the name used in the API. Captures from Matches
79 | // are available for use here. If not specified, it defaults
80 | // to $0 if no capture groups are present in Matches, or $1
81 | // if only one is present, and will error if multiple are.
82 | As string `json:"as" yaml:"as"`
83 | }
84 |
85 | // ResourceRules describe the rules for querying resource metrics
86 | // API results. It's assumed that the same metrics can be used
87 | // to aggregate across different resources.
88 | type ResourceRules struct {
89 | CPU ResourceRule `json:"cpu" yaml:"cpu"`
90 | Memory ResourceRule `json:"memory" yaml:"memory"`
91 | // Window is the window size reported by the resource metrics API. It should match the value used
92 | // in your containerQuery and nodeQuery if you use a `rate` function.
93 | Window pmodel.Duration `json:"window" yaml:"window"`
94 | }
95 |
96 | // ResourceRule describes how to query metrics for some particular
97 | // system resource metric.
98 | type ResourceRule struct {
99 | // Container is the query used to fetch the metrics for containers.
100 | ContainerQuery string `json:"containerQuery" yaml:"containerQuery"`
101 | // NodeQuery is the query used to fetch the metrics for nodes
102 | // (for instance, simply aggregating by node label is insufficient for
103 | // cadvisor metrics -- you need to select the `/` container).
104 | NodeQuery string `json:"nodeQuery" yaml:"nodeQuery"`
105 | // Resources specifies how associated Kubernetes resources should be discovered for
106 | // the given metrics.
107 | Resources ResourceMapping `json:"resources" yaml:"resources"`
108 | // ContainerLabel indicates the name of the Prometheus label containing the container name
109 | // (since "container" is not a resource, this can't go in the `resources` block, but is similar).
110 | ContainerLabel string `json:"containerLabel" yaml:"containerLabel"`
111 | }
112 |
--------------------------------------------------------------------------------
/pkg/config/loader.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "os"
7 |
8 | yaml "gopkg.in/yaml.v2"
9 | )
10 |
11 | // FromFile loads the configuration from a particular file.
12 | func FromFile(filename string) (*MetricsDiscoveryConfig, error) {
13 | file, err := os.Open(filename)
14 | if err != nil {
15 | return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err)
16 | }
17 | defer file.Close()
18 | contents, err := io.ReadAll(file)
19 | if err != nil {
20 | return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err)
21 | }
22 | return FromYAML(contents)
23 | }
24 |
25 | // FromYAML loads the configuration from a blob of YAML.
26 | func FromYAML(contents []byte) (*MetricsDiscoveryConfig, error) {
27 | var cfg MetricsDiscoveryConfig
28 | if err := yaml.UnmarshalStrict(contents, &cfg); err != nil {
29 | return nil, fmt.Errorf("unable to parse metrics discovery config: %v", err)
30 | }
31 | return &cfg, nil
32 | }
33 |
--------------------------------------------------------------------------------
/pkg/custom-provider/provider.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package provider
18 |
19 | import (
20 | "context"
21 | "fmt"
22 | "math"
23 | "time"
24 |
25 | pmodel "github.com/prometheus/common/model"
26 |
27 | apierr "k8s.io/apimachinery/pkg/api/errors"
28 | apimeta "k8s.io/apimachinery/pkg/api/meta"
29 | "k8s.io/apimachinery/pkg/api/resource"
30 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31 | "k8s.io/apimachinery/pkg/labels"
32 | "k8s.io/apimachinery/pkg/types"
33 | utilruntime "k8s.io/apimachinery/pkg/util/runtime"
34 | "k8s.io/apimachinery/pkg/util/wait"
35 | "k8s.io/client-go/dynamic"
36 | "k8s.io/klog/v2"
37 | "k8s.io/metrics/pkg/apis/custom_metrics"
38 |
39 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
40 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider/helpers"
41 |
42 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
43 | "sigs.k8s.io/prometheus-adapter/pkg/naming"
44 | )
45 |
46 | // Runnable represents something that can be run until told to stop.
47 | type Runnable interface {
48 | // Run runs the runnable forever.
49 | Run()
50 | // RunUntil runs the runnable until the given channel is closed.
51 | RunUntil(stopChan <-chan struct{})
52 | }
53 |
54 | type prometheusProvider struct {
55 | mapper apimeta.RESTMapper
56 | kubeClient dynamic.Interface
57 | promClient prom.Client
58 |
59 | SeriesRegistry
60 | }
61 |
62 | func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.Interface, promClient prom.Client, namers []naming.MetricNamer, updateInterval time.Duration, maxAge time.Duration) (provider.CustomMetricsProvider, Runnable) {
63 | lister := &cachingMetricsLister{
64 | updateInterval: updateInterval,
65 | maxAge: maxAge,
66 | promClient: promClient,
67 | namers: namers,
68 |
69 | SeriesRegistry: &basicSeriesRegistry{
70 | mapper: mapper,
71 | },
72 | }
73 |
74 | return &prometheusProvider{
75 | mapper: mapper,
76 | kubeClient: kubeClient,
77 | promClient: promClient,
78 |
79 | SeriesRegistry: lister,
80 | }, lister
81 | }
82 |
83 | func (p *prometheusProvider) metricFor(value pmodel.SampleValue, name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
84 | ref, err := helpers.ReferenceFor(p.mapper, name, info)
85 | if err != nil {
86 | return nil, err
87 | }
88 |
89 | var q *resource.Quantity
90 | if math.IsNaN(float64(value)) {
91 | q = resource.NewQuantity(0, resource.DecimalSI)
92 | } else {
93 | q = resource.NewMilliQuantity(int64(value*1000.0), resource.DecimalSI)
94 | }
95 |
96 | metric := &custom_metrics.MetricValue{
97 | DescribedObject: ref,
98 | Metric: custom_metrics.MetricIdentifier{
99 | Name: info.Metric,
100 | },
101 | // TODO(directxman12): use the right timestamp
102 | Timestamp: metav1.Time{Time: time.Now()},
103 | Value: *q,
104 | }
105 |
106 | if !metricSelector.Empty() {
107 | sel, err := metav1.ParseToLabelSelector(metricSelector.String())
108 | if err != nil {
109 | return nil, err
110 | }
111 | metric.Metric.Selector = sel
112 | }
113 |
114 | return metric, nil
115 | }
116 |
117 | func (p *prometheusProvider) metricsFor(valueSet pmodel.Vector, namespace string, names []string, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
118 | values, found := p.MatchValuesToNames(info, valueSet)
119 | if !found {
120 | return nil, provider.NewMetricNotFoundError(info.GroupResource, info.Metric)
121 | }
122 | res := []custom_metrics.MetricValue{}
123 |
124 | for _, name := range names {
125 | if _, found := values[name]; !found {
126 | continue
127 | }
128 |
129 | value, err := p.metricFor(values[name], types.NamespacedName{Namespace: namespace, Name: name}, info, metricSelector)
130 | if err != nil {
131 | return nil, err
132 | }
133 | res = append(res, *value)
134 | }
135 |
136 | return &custom_metrics.MetricValueList{
137 | Items: res,
138 | }, nil
139 | }
140 |
141 | func (p *prometheusProvider) buildQuery(ctx context.Context, info provider.CustomMetricInfo, namespace string, metricSelector labels.Selector, names ...string) (pmodel.Vector, error) {
142 | query, found := p.QueryForMetric(info, namespace, metricSelector, names...)
143 | if !found {
144 | return nil, provider.NewMetricNotFoundError(info.GroupResource, info.Metric)
145 | }
146 |
147 | // TODO: use an actual context
148 | queryResults, err := p.promClient.Query(ctx, pmodel.Now(), query)
149 | if err != nil {
150 | klog.Errorf("unable to fetch metrics from prometheus: %v", err)
151 | // don't leak implementation details to the user
152 | return nil, apierr.NewInternalError(fmt.Errorf("unable to fetch metrics"))
153 | }
154 |
155 | if queryResults.Type != pmodel.ValVector {
156 | klog.Errorf("unexpected results from prometheus: expected %s, got %s on results %v", pmodel.ValVector, queryResults.Type, queryResults)
157 | return nil, apierr.NewInternalError(fmt.Errorf("unable to fetch metrics"))
158 | }
159 |
160 | return *queryResults.Vector, nil
161 | }
162 |
163 | func (p *prometheusProvider) GetMetricByName(ctx context.Context, name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
164 | // construct a query
165 | queryResults, err := p.buildQuery(ctx, info, name.Namespace, metricSelector, name.Name)
166 | if err != nil {
167 | return nil, err
168 | }
169 |
170 | // associate the metrics
171 | if len(queryResults) < 1 {
172 | return nil, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name)
173 | }
174 |
175 | namedValues, found := p.MatchValuesToNames(info, queryResults)
176 | if !found {
177 | return nil, provider.NewMetricNotFoundError(info.GroupResource, info.Metric)
178 | }
179 |
180 | if len(namedValues) > 1 {
181 | klog.V(2).Infof("Got more than one result (%v results) when fetching metric %s for %q, using the first one with a matching name...", len(queryResults), info.String(), name)
182 | }
183 |
184 | resultValue, nameFound := namedValues[name.Name]
185 | if !nameFound {
186 | klog.Errorf("None of the results returned by when fetching metric %s for %q matched the resource name", info.String(), name)
187 | return nil, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name)
188 | }
189 |
190 | // return the resulting metric
191 | return p.metricFor(resultValue, name, info, metricSelector)
192 | }
193 |
194 | func (p *prometheusProvider) GetMetricBySelector(ctx context.Context, namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
195 | // fetch a list of relevant resource names
196 | resourceNames, err := helpers.ListObjectNames(p.mapper, p.kubeClient, namespace, selector, info)
197 | if err != nil {
198 | klog.Errorf("unable to list matching resource names: %v", err)
199 | // don't leak implementation details to the user
200 | return nil, apierr.NewInternalError(fmt.Errorf("unable to list matching resources"))
201 | }
202 |
203 | // construct the actual query
204 | queryResults, err := p.buildQuery(ctx, info, namespace, metricSelector, resourceNames...)
205 | if err != nil {
206 | return nil, err
207 | }
208 |
209 | // return the resulting metrics
210 | return p.metricsFor(queryResults, namespace, resourceNames, info, metricSelector)
211 | }
212 |
213 | type cachingMetricsLister struct {
214 | SeriesRegistry
215 |
216 | promClient prom.Client
217 | updateInterval time.Duration
218 | maxAge time.Duration
219 | namers []naming.MetricNamer
220 | }
221 |
222 | func (l *cachingMetricsLister) Run() {
223 | l.RunUntil(wait.NeverStop)
224 | }
225 |
226 | func (l *cachingMetricsLister) RunUntil(stopChan <-chan struct{}) {
227 | go wait.Until(func() {
228 | if err := l.updateMetrics(); err != nil {
229 | utilruntime.HandleError(err)
230 | }
231 | }, l.updateInterval, stopChan)
232 | }
233 |
234 | type selectorSeries struct {
235 | selector prom.Selector
236 | series []prom.Series
237 | }
238 |
239 | func (l *cachingMetricsLister) updateMetrics() error {
240 | startTime := pmodel.Now().Add(-1 * l.maxAge)
241 |
242 | // don't do duplicate queries when it's just the matchers that change
243 | seriesCacheByQuery := make(map[prom.Selector][]prom.Series)
244 |
245 | // these can take a while on large clusters, so launch in parallel
246 | // and don't duplicate
247 | selectors := make(map[prom.Selector]struct{})
248 | selectorSeriesChan := make(chan selectorSeries, len(l.namers))
249 | errs := make(chan error, len(l.namers))
250 | for _, namer := range l.namers {
251 | sel := namer.Selector()
252 | if _, ok := selectors[sel]; ok {
253 | errs <- nil
254 | selectorSeriesChan <- selectorSeries{}
255 | continue
256 | }
257 | selectors[sel] = struct{}{}
258 | go func() {
259 | series, err := l.promClient.Series(context.TODO(), pmodel.Interval{Start: startTime, End: 0}, sel)
260 | if err != nil {
261 | errs <- fmt.Errorf("unable to fetch metrics for query %q: %v", sel, err)
262 | return
263 | }
264 | errs <- nil
265 | selectorSeriesChan <- selectorSeries{
266 | selector: sel,
267 | series: series,
268 | }
269 | }()
270 | }
271 |
272 | // iterate through, blocking until we've got all results
273 | for range l.namers {
274 | if err := <-errs; err != nil {
275 | return fmt.Errorf("unable to update list of all metrics: %v", err)
276 | }
277 | if ss := <-selectorSeriesChan; ss.series != nil {
278 | seriesCacheByQuery[ss.selector] = ss.series
279 | }
280 | }
281 | close(errs)
282 |
283 | newSeries := make([][]prom.Series, len(l.namers))
284 | for i, namer := range l.namers {
285 | series, cached := seriesCacheByQuery[namer.Selector()]
286 | if !cached {
287 | return fmt.Errorf("unable to update list of all metrics: no metrics retrieved for query %q", namer.Selector())
288 | }
289 | newSeries[i] = namer.FilterSeries(series)
290 | }
291 |
292 | klog.V(10).Infof("Set available metric list from Prometheus to: %v", newSeries)
293 |
294 | return l.SetSeries(newSeries, l.namers)
295 | }
296 |
--------------------------------------------------------------------------------
/pkg/custom-provider/provider_suite_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2018 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package provider_test
18 |
19 | import (
20 | "testing"
21 |
22 | . "github.com/onsi/ginkgo"
23 | . "github.com/onsi/gomega"
24 | )
25 |
26 | func TestProvider(t *testing.T) {
27 | RegisterFailHandler(Fail)
28 | RunSpecs(t, "Custom Metrics Provider Suite")
29 | }
30 |
--------------------------------------------------------------------------------
/pkg/custom-provider/provider_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package provider
18 |
19 | import (
20 | "time"
21 |
22 | . "github.com/onsi/ginkgo"
23 | . "github.com/onsi/gomega"
24 | pmodel "github.com/prometheus/common/model"
25 |
26 | "k8s.io/apimachinery/pkg/runtime/schema"
27 | fakedyn "k8s.io/client-go/dynamic/fake"
28 |
29 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
30 |
31 | config "sigs.k8s.io/prometheus-adapter/cmd/config-gen/utils"
32 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
33 | fakeprom "sigs.k8s.io/prometheus-adapter/pkg/client/fake"
34 | "sigs.k8s.io/prometheus-adapter/pkg/naming"
35 | )
36 |
37 | const fakeProviderUpdateInterval = 2 * time.Second
38 | const fakeProviderStartDuration = 2 * time.Second
39 |
40 | func setupPrometheusProvider() (provider.CustomMetricsProvider, *fakeprom.FakePrometheusClient) {
41 | fakeProm := &fakeprom.FakePrometheusClient{}
42 | fakeKubeClient := &fakedyn.FakeDynamicClient{}
43 |
44 | cfg := config.DefaultConfig(1*time.Minute, "")
45 | namers, err := naming.NamersFromConfig(cfg.Rules, restMapper())
46 | Expect(err).NotTo(HaveOccurred())
47 |
48 | prov, _ := NewPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, namers, fakeProviderUpdateInterval, fakeProviderStartDuration)
49 |
50 | containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod", ""))
51 | namespacedSel := prom.MatchSeries("", prom.LabelNeq("namespace", ""), prom.NameNotMatches("^container_.*"))
52 | fakeProm.SeriesResults = map[prom.Selector][]prom.Series{
53 | containerSel: {
54 | {
55 | Name: "container_some_usage",
56 | Labels: pmodel.LabelSet{"pod": "somepod", "namespace": "somens", "container": "somecont"},
57 | },
58 | },
59 | namespacedSel: {
60 | {
61 | Name: "ingress_hits_total",
62 | Labels: pmodel.LabelSet{"ingress": "someingress", "service": "somesvc", "pod": "backend1", "namespace": "somens"},
63 | },
64 | {
65 | Name: "ingress_hits_total",
66 | Labels: pmodel.LabelSet{"ingress": "someingress", "service": "somesvc", "pod": "backend2", "namespace": "somens"},
67 | },
68 | {
69 | Name: "service_proxy_packets",
70 | Labels: pmodel.LabelSet{"service": "somesvc", "namespace": "somens"},
71 | },
72 | {
73 | Name: "work_queue_wait_seconds_total",
74 | Labels: pmodel.LabelSet{"deployment": "somedep", "namespace": "somens"},
75 | },
76 | },
77 | }
78 |
79 | return prov, fakeProm
80 | }
81 |
82 | var _ = Describe("Custom Metrics Provider", func() {
83 | It("should be able to list all metrics", func() {
84 | By("setting up the provider")
85 | prov, fakeProm := setupPrometheusProvider()
86 |
87 | By("ensuring that no metrics are present before we start listing")
88 | Expect(prov.ListAllMetrics()).To(BeEmpty())
89 |
90 | By("setting the acceptable interval to now until the next update, with a bit of wiggle room")
91 | startTime := pmodel.Now().Add(-1*fakeProviderUpdateInterval - fakeProviderUpdateInterval/10)
92 | fakeProm.AcceptableInterval = pmodel.Interval{Start: startTime, End: 0}
93 |
94 | By("updating the list of available metrics")
95 | // don't call RunUntil to avoid timing issue
96 | lister := prov.(*prometheusProvider).SeriesRegistry.(*cachingMetricsLister)
97 | Expect(lister.updateMetrics()).To(Succeed())
98 |
99 | By("listing all metrics, and checking that they contain the expected results")
100 | Expect(prov.ListAllMetrics()).To(ConsistOf(
101 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "services"}, Namespaced: true, Metric: "ingress_hits"},
102 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Group: "extensions", Resource: "ingresses"}, Namespaced: true, Metric: "ingress_hits"},
103 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "pods"}, Namespaced: true, Metric: "ingress_hits"},
104 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "namespaces"}, Namespaced: false, Metric: "ingress_hits"},
105 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "services"}, Namespaced: true, Metric: "service_proxy_packets"},
106 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "namespaces"}, Namespaced: false, Metric: "service_proxy_packets"},
107 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Group: "extensions", Resource: "deployments"}, Namespaced: true, Metric: "work_queue_wait"},
108 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "namespaces"}, Namespaced: false, Metric: "work_queue_wait"},
109 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "namespaces"}, Namespaced: false, Metric: "some_usage"},
110 | provider.CustomMetricInfo{GroupResource: schema.GroupResource{Resource: "pods"}, Namespaced: true, Metric: "some_usage"},
111 | ))
112 | })
113 | })
114 |
--------------------------------------------------------------------------------
/pkg/custom-provider/series_registry.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package provider
18 |
19 | import (
20 | "fmt"
21 | "sync"
22 |
23 | pmodel "github.com/prometheus/common/model"
24 |
25 | apimeta "k8s.io/apimachinery/pkg/api/meta"
26 | "k8s.io/apimachinery/pkg/labels"
27 | "k8s.io/klog/v2"
28 |
29 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
30 |
31 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
32 | "sigs.k8s.io/prometheus-adapter/pkg/naming"
33 | )
34 |
35 | // NB: container metrics sourced from cAdvisor don't consistently follow naming conventions,
36 | // so we need to whitelist them and handle them on a case-by-case basis. Metrics ending in `_total`
37 | // *should* be counters, but may actually be guages in this case.
38 |
39 | // SeriesType represents the kind of series backing a metric.
40 | type SeriesType int
41 |
42 | const (
43 | CounterSeries SeriesType = iota
44 | SecondsCounterSeries
45 | GaugeSeries
46 | )
47 |
48 | // SeriesRegistry provides conversions between Prometheus series and MetricInfo
49 | type SeriesRegistry interface {
50 | // SetSeries replaces the known series in this registry.
51 | // Each slice in series should correspond to a MetricNamer in namers.
52 | SetSeries(series [][]prom.Series, namers []naming.MetricNamer) error
53 | // ListAllMetrics lists all metrics known to this registry
54 | ListAllMetrics() []provider.CustomMetricInfo
55 | // SeriesForMetric looks up the minimum required series information to make a query for the given metric
56 | // against the given resource (namespace may be empty for non-namespaced resources)
57 | QueryForMetric(info provider.CustomMetricInfo, namespace string, metricSelector labels.Selector, resourceNames ...string) (query prom.Selector, found bool)
58 | // MatchValuesToNames matches result values to resource names for the given metric and value set
59 | MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool)
60 | }
61 |
62 | type seriesInfo struct {
63 | // seriesName is the name of the corresponding Prometheus series
64 | seriesName string
65 |
66 | // namer is the MetricNamer used to name this series
67 | namer naming.MetricNamer
68 | }
69 |
70 | // overridableSeriesRegistry is a basic SeriesRegistry
71 | type basicSeriesRegistry struct {
72 | mu sync.RWMutex
73 |
74 | // info maps metric info to information about the corresponding series
75 | info map[provider.CustomMetricInfo]seriesInfo
76 | // metrics is the list of all known metrics
77 | metrics []provider.CustomMetricInfo
78 |
79 | mapper apimeta.RESTMapper
80 | }
81 |
82 | func (r *basicSeriesRegistry) SetSeries(newSeriesSlices [][]prom.Series, namers []naming.MetricNamer) error {
83 | if len(newSeriesSlices) != len(namers) {
84 | return fmt.Errorf("need one set of series per namer")
85 | }
86 |
87 | newInfo := make(map[provider.CustomMetricInfo]seriesInfo)
88 | for i, newSeries := range newSeriesSlices {
89 | namer := namers[i]
90 | for _, series := range newSeries {
91 | // TODO: warn if it doesn't match any resources
92 | resources, namespaced := namer.ResourcesForSeries(series)
93 | name, err := namer.MetricNameForSeries(series)
94 | if err != nil {
95 | klog.Errorf("unable to name series %q, skipping: %v", series.String(), err)
96 | continue
97 | }
98 | for _, resource := range resources {
99 | info := provider.CustomMetricInfo{
100 | GroupResource: resource,
101 | Namespaced: namespaced,
102 | Metric: name,
103 | }
104 |
105 | // some metrics aren't counted as namespaced
106 | if resource == naming.NsGroupResource || resource == naming.NodeGroupResource || resource == naming.PVGroupResource {
107 | info.Namespaced = false
108 | }
109 |
110 | // we don't need to re-normalize, because the metric namer should have already normalized for us
111 | newInfo[info] = seriesInfo{
112 | seriesName: series.Name,
113 | namer: namer,
114 | }
115 | }
116 | }
117 | }
118 |
119 | // regenerate metrics
120 | newMetrics := make([]provider.CustomMetricInfo, 0, len(newInfo))
121 | for info := range newInfo {
122 | newMetrics = append(newMetrics, info)
123 | }
124 |
125 | r.mu.Lock()
126 | defer r.mu.Unlock()
127 |
128 | r.info = newInfo
129 | r.metrics = newMetrics
130 |
131 | return nil
132 | }
133 |
134 | func (r *basicSeriesRegistry) ListAllMetrics() []provider.CustomMetricInfo {
135 | r.mu.RLock()
136 | defer r.mu.RUnlock()
137 |
138 | return r.metrics
139 | }
140 |
141 | func (r *basicSeriesRegistry) QueryForMetric(metricInfo provider.CustomMetricInfo, namespace string, metricSelector labels.Selector, resourceNames ...string) (prom.Selector, bool) {
142 | r.mu.RLock()
143 | defer r.mu.RUnlock()
144 |
145 | if len(resourceNames) == 0 {
146 | klog.Errorf("no resource names requested while producing a query for metric %s", metricInfo.String())
147 | return "", false
148 | }
149 |
150 | metricInfo, _, err := metricInfo.Normalized(r.mapper)
151 | if err != nil {
152 | klog.Errorf("unable to normalize group resource while producing a query: %v", err)
153 | return "", false
154 | }
155 |
156 | info, infoFound := r.info[metricInfo]
157 | if !infoFound {
158 | klog.V(10).Infof("metric %v not registered", metricInfo)
159 | return "", false
160 | }
161 |
162 | query, err := info.namer.QueryForSeries(info.seriesName, metricInfo.GroupResource, namespace, metricSelector, resourceNames...)
163 | if err != nil {
164 | klog.Errorf("unable to construct query for metric %s: %v", metricInfo.String(), err)
165 | return "", false
166 | }
167 |
168 | return query, true
169 | }
170 |
171 | func (r *basicSeriesRegistry) MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) {
172 | r.mu.RLock()
173 | defer r.mu.RUnlock()
174 |
175 | metricInfo, _, err := metricInfo.Normalized(r.mapper)
176 | if err != nil {
177 | klog.Errorf("unable to normalize group resource while matching values to names: %v", err)
178 | return nil, false
179 | }
180 |
181 | info, infoFound := r.info[metricInfo]
182 | if !infoFound {
183 | return nil, false
184 | }
185 |
186 | resourceLbl, err := info.namer.LabelForResource(metricInfo.GroupResource)
187 | if err != nil {
188 | klog.Errorf("unable to construct resource label for metric %s: %v", metricInfo.String(), err)
189 | return nil, false
190 | }
191 |
192 | res := make(map[string]pmodel.SampleValue, len(values))
193 | for _, val := range values {
194 | if val == nil {
195 | // skip empty values
196 | continue
197 | }
198 | res[string(val.Metric[resourceLbl])] = val.Value
199 | }
200 |
201 | return res, true
202 | }
203 |
--------------------------------------------------------------------------------
/pkg/external-provider/basic_metric_lister.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package provider
18 |
19 | import (
20 | "context"
21 | "fmt"
22 | "time"
23 |
24 | "k8s.io/klog/v2"
25 |
26 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
27 | "sigs.k8s.io/prometheus-adapter/pkg/naming"
28 |
29 | pmodel "github.com/prometheus/common/model"
30 | )
31 |
32 | // Runnable represents something that can be run until told to stop.
33 | type Runnable interface {
34 | // Run runs the runnable forever.
35 | Run()
36 | // RunUntil runs the runnable until the given channel is closed.
37 | RunUntil(stopChan <-chan struct{})
38 | }
39 |
40 | // A MetricLister provides a window into all of the metrics that are available within a given
41 | // Prometheus instance, classified as either Custom or External metrics, but presented generically
42 | // so that it can manage both types simultaneously.
43 | type MetricLister interface {
44 | ListAllMetrics() (MetricUpdateResult, error)
45 | }
46 |
47 | // A MetricListerWithNotification is a MetricLister that has the ability to notify listeners
48 | // when new metric data is available.
49 | type MetricListerWithNotification interface {
50 | MetricLister
51 | Runnable
52 |
53 | // AddNotificationReceiver registers a callback to be invoked when new metric data is available.
54 | AddNotificationReceiver(MetricUpdateCallback)
55 | // UpdateNow forces an immediate refresh from the source data. Primarily for test purposes.
56 | UpdateNow()
57 | }
58 |
59 | type basicMetricLister struct {
60 | promClient prom.Client
61 | namers []naming.MetricNamer
62 | lookback time.Duration
63 | }
64 |
65 | // NewBasicMetricLister creates a MetricLister that is capable of interactly directly with Prometheus to list metrics.
66 | func NewBasicMetricLister(promClient prom.Client, namers []naming.MetricNamer, lookback time.Duration) MetricLister {
67 | lister := basicMetricLister{
68 | promClient: promClient,
69 | namers: namers,
70 | lookback: lookback,
71 | }
72 |
73 | return &lister
74 | }
75 |
76 | type selectorSeries struct {
77 | selector prom.Selector
78 | series []prom.Series
79 | }
80 |
81 | func (l *basicMetricLister) ListAllMetrics() (MetricUpdateResult, error) {
82 | result := MetricUpdateResult{
83 | series: make([][]prom.Series, 0),
84 | namers: make([]naming.MetricNamer, 0),
85 | }
86 |
87 | startTime := pmodel.Now().Add(-1 * l.lookback)
88 |
89 | // these can take a while on large clusters, so launch in parallel
90 | // and don't duplicate
91 | selectors := make(map[prom.Selector]struct{})
92 | selectorSeriesChan := make(chan selectorSeries, len(l.namers))
93 | errs := make(chan error, len(l.namers))
94 | for _, converter := range l.namers {
95 | sel := converter.Selector()
96 | if _, ok := selectors[sel]; ok {
97 | errs <- nil
98 | selectorSeriesChan <- selectorSeries{}
99 | continue
100 | }
101 | selectors[sel] = struct{}{}
102 | go func() {
103 | series, err := l.promClient.Series(context.TODO(), pmodel.Interval{Start: startTime, End: 0}, sel)
104 | if err != nil {
105 | errs <- fmt.Errorf("unable to fetch metrics for query %q: %v", sel, err)
106 | return
107 | }
108 | errs <- nil
109 | // Push into the channel: "this selector produced these series"
110 | selectorSeriesChan <- selectorSeries{
111 | selector: sel,
112 | series: series,
113 | }
114 | }()
115 | }
116 |
117 | // don't do duplicate queries when it's just the matchers that change
118 | seriesCacheByQuery := make(map[prom.Selector][]prom.Series)
119 |
120 | // iterate through, blocking until we've got all results
121 | // We know that, from above, we should have pushed one item into the channel
122 | // for each converter. So here, we'll assume that we should receive one item per converter.
123 | for range l.namers {
124 | if err := <-errs; err != nil {
125 | return result, fmt.Errorf("unable to update list of all metrics: %v", err)
126 | }
127 | // Receive from the channel: "this selector produced these series"
128 | // We stuff that into this map so that we can collect the data as it arrives
129 | // and then, once we've received it all, we can process it below.
130 | if ss := <-selectorSeriesChan; ss.series != nil {
131 | seriesCacheByQuery[ss.selector] = ss.series
132 | }
133 | }
134 | close(errs)
135 |
136 | // Now that we've collected all of the results into `seriesCacheByQuery`
137 | // we can start processing them.
138 | newSeries := make([][]prom.Series, len(l.namers))
139 | for i, namer := range l.namers {
140 | series, cached := seriesCacheByQuery[namer.Selector()]
141 | if !cached {
142 | return result, fmt.Errorf("unable to update list of all metrics: no metrics retrieved for query %q", namer.Selector())
143 | }
144 | // Because converters provide a "post-filtering" option, it's not enough to
145 | // simply take all the series that were produced. We need to further filter them.
146 | newSeries[i] = namer.FilterSeries(series)
147 | }
148 |
149 | klog.V(10).Infof("Set available metric list from Prometheus to: %v", newSeries)
150 |
151 | result.series = newSeries
152 | result.namers = l.namers
153 | return result, nil
154 | }
155 |
156 | // MetricUpdateResult represents the output of a periodic inspection of metrics found to be
157 | // available in Prometheus.
158 | // It includes both the series data the Prometheus exposed, as well as the configurational
159 | // object that led to their discovery.
160 | type MetricUpdateResult struct {
161 | series [][]prom.Series
162 | namers []naming.MetricNamer
163 | }
164 |
165 | // MetricUpdateCallback is a function signature for receiving periodic updates about
166 | // available metrics.
167 | type MetricUpdateCallback func(MetricUpdateResult)
168 |
--------------------------------------------------------------------------------
/pkg/external-provider/external_series_registry.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 | http://www.apache.org/licenses/LICENSE-2.0
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 | */
13 |
14 | package provider
15 |
16 | import (
17 | "sync"
18 |
19 | "k8s.io/apimachinery/pkg/labels"
20 | "k8s.io/klog/v2"
21 |
22 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
23 |
24 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
25 | "sigs.k8s.io/prometheus-adapter/pkg/naming"
26 | )
27 |
28 | // ExternalSeriesRegistry acts as the top-level converter for transforming Kubernetes requests
29 | // for external metrics into Prometheus queries.
30 | type ExternalSeriesRegistry interface {
31 | // ListAllMetrics lists all metrics known to this registry
32 | ListAllMetrics() []provider.ExternalMetricInfo
33 | QueryForMetric(namespace string, metricName string, metricSelector labels.Selector) (prom.Selector, bool, error)
34 | }
35 |
36 | // overridableSeriesRegistry is a basic SeriesRegistry
37 | type externalSeriesRegistry struct {
38 | // We lock when reading/writing metrics, and metricsInfo to prevent inconsistencies.
39 | mu sync.RWMutex
40 |
41 | // metrics is the list of all known metrics, ready to return from the API
42 | metrics []provider.ExternalMetricInfo
43 | // metricsInfo is a lookup from a metric to SeriesConverter for the sake of generating queries
44 | metricsInfo map[string]seriesInfo
45 | }
46 |
47 | type seriesInfo struct {
48 | // seriesName is the name of the corresponding Prometheus series
49 | seriesName string
50 |
51 | // namer is the MetricNamer used to name this series
52 | namer naming.MetricNamer
53 | }
54 |
55 | // NewExternalSeriesRegistry creates an ExternalSeriesRegistry driven by the data from the provided MetricLister.
56 | func NewExternalSeriesRegistry(lister MetricListerWithNotification) ExternalSeriesRegistry {
57 | var registry = externalSeriesRegistry{
58 | metrics: make([]provider.ExternalMetricInfo, 0),
59 | metricsInfo: map[string]seriesInfo{},
60 | }
61 |
62 | lister.AddNotificationReceiver(registry.filterAndStoreMetrics)
63 |
64 | return ®istry
65 | }
66 |
67 | func (r *externalSeriesRegistry) filterAndStoreMetrics(result MetricUpdateResult) {
68 | newSeriesSlices := result.series
69 | namers := result.namers
70 |
71 | if len(newSeriesSlices) != len(namers) {
72 | klog.Fatal("need one set of series per converter")
73 | }
74 | apiMetricsCache := make([]provider.ExternalMetricInfo, 0)
75 | rawMetricsCache := make(map[string]seriesInfo)
76 |
77 | for i, newSeries := range newSeriesSlices {
78 | namer := namers[i]
79 | for _, series := range newSeries {
80 | identity, err := namer.MetricNameForSeries(series)
81 |
82 | if err != nil {
83 | klog.Errorf("unable to name series %q, skipping: %v", series.String(), err)
84 | continue
85 | }
86 |
87 | name := identity
88 | rawMetricsCache[name] = seriesInfo{
89 | seriesName: series.Name,
90 | namer: namer,
91 | }
92 | }
93 | }
94 |
95 | for metricName := range rawMetricsCache {
96 | apiMetricsCache = append(apiMetricsCache, provider.ExternalMetricInfo{
97 | Metric: metricName,
98 | })
99 | }
100 |
101 | r.mu.Lock()
102 | defer r.mu.Unlock()
103 |
104 | r.metrics = apiMetricsCache
105 | r.metricsInfo = rawMetricsCache
106 | }
107 |
108 | func (r *externalSeriesRegistry) ListAllMetrics() []provider.ExternalMetricInfo {
109 | r.mu.RLock()
110 | defer r.mu.RUnlock()
111 |
112 | return r.metrics
113 | }
114 |
115 | func (r *externalSeriesRegistry) QueryForMetric(namespace string, metricName string, metricSelector labels.Selector) (prom.Selector, bool, error) {
116 | r.mu.RLock()
117 | defer r.mu.RUnlock()
118 |
119 | info, found := r.metricsInfo[metricName]
120 |
121 | if !found {
122 | klog.V(10).Infof("external metric %q not found", metricName)
123 | return "", false, nil
124 | }
125 | query, err := info.namer.QueryForExternalSeries(info.seriesName, namespace, metricSelector)
126 |
127 | return query, found, err
128 | }
129 |
--------------------------------------------------------------------------------
/pkg/external-provider/metric_converter.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 | http://www.apache.org/licenses/LICENSE-2.0
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 | */
13 |
14 | package provider
15 |
16 | import (
17 | "errors"
18 | "fmt"
19 |
20 | "github.com/prometheus/common/model"
21 |
22 | "k8s.io/apimachinery/pkg/api/resource"
23 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24 | "k8s.io/metrics/pkg/apis/external_metrics"
25 |
26 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
27 |
28 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
29 | )
30 |
31 | // MetricConverter provides a unified interface for converting the results of
32 | // Prometheus queries into external metric types.
33 | type MetricConverter interface {
34 | Convert(info provider.ExternalMetricInfo, queryResult prom.QueryResult) (*external_metrics.ExternalMetricValueList, error)
35 | }
36 |
37 | type metricConverter struct {
38 | }
39 |
40 | // NewMetricConverter creates a MetricCoverter, capable of converting any of the three metric types
41 | // returned by the Prometheus client into external metrics types.
42 | func NewMetricConverter() MetricConverter {
43 | return &metricConverter{}
44 | }
45 |
46 | func (c *metricConverter) Convert(info provider.ExternalMetricInfo, queryResult prom.QueryResult) (*external_metrics.ExternalMetricValueList, error) {
47 | if queryResult.Type == model.ValScalar {
48 | return c.convertScalar(info, queryResult)
49 | }
50 |
51 | if queryResult.Type == model.ValVector {
52 | return c.convertVector(info, queryResult)
53 | }
54 |
55 | return nil, errors.New("encountered an unexpected query result type")
56 | }
57 |
58 | func (c *metricConverter) convertSample(info provider.ExternalMetricInfo, sample *model.Sample) (*external_metrics.ExternalMetricValue, error) {
59 | labels := c.convertLabels(sample.Metric)
60 |
61 | singleMetric := external_metrics.ExternalMetricValue{
62 | MetricName: info.Metric,
63 | Timestamp: metav1.Time{
64 | Time: sample.Timestamp.Time(),
65 | },
66 | Value: *resource.NewMilliQuantity(int64(sample.Value*1000.0), resource.DecimalSI),
67 | MetricLabels: labels,
68 | }
69 |
70 | return &singleMetric, nil
71 | }
72 |
73 | func (c *metricConverter) convertLabels(inLabels model.Metric) map[string]string {
74 | numLabels := len(inLabels)
75 | outLabels := make(map[string]string, numLabels)
76 | for labelName, labelVal := range inLabels {
77 | outLabels[string(labelName)] = string(labelVal)
78 | }
79 |
80 | return outLabels
81 | }
82 |
83 | func (c *metricConverter) convertVector(info provider.ExternalMetricInfo, queryResult prom.QueryResult) (*external_metrics.ExternalMetricValueList, error) {
84 | if queryResult.Type != model.ValVector {
85 | return nil, errors.New("incorrect query result type")
86 | }
87 |
88 | toConvert := *queryResult.Vector
89 |
90 | if toConvert == nil {
91 | return nil, errors.New("the provided input did not contain vector query results")
92 | }
93 |
94 | items := []external_metrics.ExternalMetricValue{}
95 | metricValueList := external_metrics.ExternalMetricValueList{
96 | Items: items,
97 | }
98 |
99 | numSamples := toConvert.Len()
100 | if numSamples == 0 {
101 | return &metricValueList, nil
102 | }
103 |
104 | for _, val := range toConvert {
105 | singleMetric, err := c.convertSample(info, val)
106 |
107 | if err != nil {
108 | return nil, fmt.Errorf("unable to convert vector: %v", err)
109 | }
110 |
111 | items = append(items, *singleMetric)
112 | }
113 |
114 | metricValueList = external_metrics.ExternalMetricValueList{
115 | Items: items,
116 | }
117 | return &metricValueList, nil
118 | }
119 |
120 | func (c *metricConverter) convertScalar(info provider.ExternalMetricInfo, queryResult prom.QueryResult) (*external_metrics.ExternalMetricValueList, error) {
121 | if queryResult.Type != model.ValScalar {
122 | return nil, errors.New("scalarConverter can only convert scalar query results")
123 | }
124 |
125 | toConvert := queryResult.Scalar
126 |
127 | if toConvert == nil {
128 | return nil, errors.New("the provided input did not contain scalar query results")
129 | }
130 |
131 | result := external_metrics.ExternalMetricValueList{
132 | Items: []external_metrics.ExternalMetricValue{
133 | {
134 | MetricName: info.Metric,
135 | Timestamp: metav1.Time{
136 | Time: toConvert.Timestamp.Time(),
137 | },
138 | Value: *resource.NewMilliQuantity(int64(toConvert.Value*1000.0), resource.DecimalSI),
139 | },
140 | },
141 | }
142 | return &result, nil
143 | }
144 |
--------------------------------------------------------------------------------
/pkg/external-provider/periodic_metric_lister.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package provider
18 |
19 | import (
20 | "time"
21 |
22 | utilruntime "k8s.io/apimachinery/pkg/util/runtime"
23 | "k8s.io/apimachinery/pkg/util/wait"
24 | )
25 |
26 | type periodicMetricLister struct {
27 | realLister MetricLister
28 | updateInterval time.Duration
29 | mostRecentResult MetricUpdateResult
30 | callbacks []MetricUpdateCallback
31 | }
32 |
33 | // NewPeriodicMetricLister creates a MetricLister that periodically pulls the list of available metrics
34 | // at the provided interval, but defers the actual act of retrieving the metrics to the supplied MetricLister.
35 | func NewPeriodicMetricLister(realLister MetricLister, updateInterval time.Duration) (MetricListerWithNotification, Runnable) {
36 | lister := periodicMetricLister{
37 | updateInterval: updateInterval,
38 | realLister: realLister,
39 | callbacks: make([]MetricUpdateCallback, 0),
40 | }
41 |
42 | return &lister, &lister
43 | }
44 |
45 | func (l *periodicMetricLister) AddNotificationReceiver(callback MetricUpdateCallback) {
46 | l.callbacks = append(l.callbacks, callback)
47 | }
48 |
49 | func (l *periodicMetricLister) ListAllMetrics() (MetricUpdateResult, error) {
50 | return l.mostRecentResult, nil
51 | }
52 |
53 | func (l *periodicMetricLister) Run() {
54 | l.RunUntil(wait.NeverStop)
55 | }
56 |
57 | func (l *periodicMetricLister) RunUntil(stopChan <-chan struct{}) {
58 | go wait.Until(func() {
59 | if err := l.updateMetrics(); err != nil {
60 | utilruntime.HandleError(err)
61 | }
62 | }, l.updateInterval, stopChan)
63 | }
64 |
65 | func (l *periodicMetricLister) updateMetrics() error {
66 | result, err := l.realLister.ListAllMetrics()
67 |
68 | if err != nil {
69 | return err
70 | }
71 |
72 | // Cache the result.
73 | l.mostRecentResult = result
74 | // Let our listeners know we've got new data ready for them.
75 | l.notifyListeners()
76 | return nil
77 | }
78 |
79 | func (l *periodicMetricLister) notifyListeners() {
80 | for _, listener := range l.callbacks {
81 | if listener != nil {
82 | listener(l.mostRecentResult)
83 | }
84 | }
85 | }
86 |
87 | func (l *periodicMetricLister) UpdateNow() {
88 | if err := l.updateMetrics(); err != nil {
89 | utilruntime.HandleError(err)
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/pkg/external-provider/periodic_metric_lister_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 | http://www.apache.org/licenses/LICENSE-2.0
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 | */
13 |
14 | package provider
15 |
16 | import (
17 | "testing"
18 | "time"
19 |
20 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
21 |
22 | "github.com/stretchr/testify/require"
23 | )
24 |
25 | type fakeLister struct {
26 | callCount int
27 | }
28 |
29 | func (f *fakeLister) ListAllMetrics() (MetricUpdateResult, error) {
30 | f.callCount++
31 |
32 | return MetricUpdateResult{
33 | series: [][]prom.Series{
34 | {
35 | {
36 | Name: "a_series",
37 | },
38 | },
39 | },
40 | }, nil
41 | }
42 |
43 | func TestWhenNewMetricsAvailableCallbackIsInvoked(t *testing.T) {
44 | fakeLister := &fakeLister{}
45 | targetLister, _ := NewPeriodicMetricLister(fakeLister, time.Duration(1000))
46 | periodicLister := targetLister.(*periodicMetricLister)
47 |
48 | callbackInvoked := false
49 | callback := func(r MetricUpdateResult) {
50 | callbackInvoked = true
51 | }
52 |
53 | periodicLister.AddNotificationReceiver(callback)
54 | err := periodicLister.updateMetrics()
55 | require.NoError(t, err)
56 | require.True(t, callbackInvoked)
57 | }
58 |
59 | func TestWhenListingMetricsReturnsCachedValues(t *testing.T) {
60 | fakeLister := &fakeLister{}
61 | targetLister, _ := NewPeriodicMetricLister(fakeLister, time.Duration(1000))
62 | periodicLister := targetLister.(*periodicMetricLister)
63 |
64 | // We haven't invoked the inner lister yet, so we should have no results.
65 | resultBeforeUpdate, err := periodicLister.ListAllMetrics()
66 | require.NoError(t, err)
67 | require.Equal(t, 0, len(resultBeforeUpdate.series))
68 | require.Equal(t, 0, fakeLister.callCount)
69 |
70 | // We can simulate waiting for the udpate interval to pass...
71 | // which should result in calling the inner lister to get the metrics.
72 | err = periodicLister.updateMetrics()
73 | require.NoError(t, err)
74 | require.Equal(t, 1, fakeLister.callCount)
75 |
76 | // If we list now, we should return the cached values.
77 | // Make sure we got some results this time
78 | // as well as that we didn't unnecessarily invoke the inner lister.
79 | resultAfterUpdate, err := periodicLister.ListAllMetrics()
80 | require.NoError(t, err)
81 | require.NotEqual(t, 0, len(resultAfterUpdate.series))
82 | require.Equal(t, 1, fakeLister.callCount)
83 | }
84 |
--------------------------------------------------------------------------------
/pkg/external-provider/provider.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 | http://www.apache.org/licenses/LICENSE-2.0
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 | */
13 |
14 | package provider
15 |
16 | import (
17 | "context"
18 | "fmt"
19 | "time"
20 |
21 | pmodel "github.com/prometheus/common/model"
22 |
23 | apierr "k8s.io/apimachinery/pkg/api/errors"
24 | "k8s.io/apimachinery/pkg/labels"
25 | "k8s.io/apimachinery/pkg/runtime/schema"
26 | "k8s.io/klog/v2"
27 | "k8s.io/metrics/pkg/apis/external_metrics"
28 |
29 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
30 |
31 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
32 | "sigs.k8s.io/prometheus-adapter/pkg/naming"
33 | )
34 |
35 | type externalPrometheusProvider struct {
36 | promClient prom.Client
37 | metricConverter MetricConverter
38 |
39 | seriesRegistry ExternalSeriesRegistry
40 | }
41 |
42 | func (p *externalPrometheusProvider) GetExternalMetric(ctx context.Context, namespace string, metricSelector labels.Selector, info provider.ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) {
43 | selector, found, err := p.seriesRegistry.QueryForMetric(namespace, info.Metric, metricSelector)
44 |
45 | if err != nil {
46 | klog.Errorf("unable to generate a query for the metric: %v", err)
47 | return nil, apierr.NewInternalError(fmt.Errorf("unable to fetch metrics"))
48 | }
49 |
50 | if !found {
51 | return nil, provider.NewMetricNotFoundError(p.selectGroupResource(namespace), info.Metric)
52 | }
53 | // Here is where we're making the query, need to be before here xD
54 | queryResults, err := p.promClient.Query(ctx, pmodel.Now(), selector)
55 |
56 | if err != nil {
57 | klog.Errorf("unable to fetch metrics from prometheus: %v", err)
58 | // don't leak implementation details to the user
59 | return nil, apierr.NewInternalError(fmt.Errorf("unable to fetch metrics"))
60 | }
61 | return p.metricConverter.Convert(info, queryResults)
62 | }
63 |
64 | func (p *externalPrometheusProvider) ListAllExternalMetrics() []provider.ExternalMetricInfo {
65 | return p.seriesRegistry.ListAllMetrics()
66 | }
67 |
68 | func (p *externalPrometheusProvider) selectGroupResource(namespace string) schema.GroupResource {
69 | if namespace == "default" {
70 | return naming.NsGroupResource
71 | }
72 |
73 | return schema.GroupResource{
74 | Group: "",
75 | Resource: "",
76 | }
77 | }
78 |
79 | // NewExternalPrometheusProvider creates an ExternalMetricsProvider capable of responding to Kubernetes requests for external metric data
80 | func NewExternalPrometheusProvider(promClient prom.Client, namers []naming.MetricNamer, updateInterval time.Duration, maxAge time.Duration) (provider.ExternalMetricsProvider, Runnable) {
81 | metricConverter := NewMetricConverter()
82 | basicLister := NewBasicMetricLister(promClient, namers, maxAge)
83 | periodicLister, _ := NewPeriodicMetricLister(basicLister, updateInterval)
84 | seriesRegistry := NewExternalSeriesRegistry(periodicLister)
85 | return &externalPrometheusProvider{
86 | promClient: promClient,
87 | seriesRegistry: seriesRegistry,
88 | metricConverter: metricConverter,
89 | }, periodicLister
90 | }
91 |
--------------------------------------------------------------------------------
/pkg/naming/errors.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2017 The Kubernetes Authors.
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 | http://www.apache.org/licenses/LICENSE-2.0
7 | Unless required by applicable law or agreed to in writing, software
8 | distributed under the License is distributed on an "AS IS" BASIS,
9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | See the License for the specific language governing permissions and
11 | limitations under the License.
12 | */
13 |
14 | package naming
15 |
16 | import "errors"
17 |
18 | var (
19 | // ErrUnsupportedOperator creates an error that represents the fact that we were requested to service a query that
20 | // Prometheus would be unable to support.
21 | ErrUnsupportedOperator = errors.New("operator not supported by prometheus")
22 |
23 | // ErrMalformedQuery creates an error that represents the fact that we were requested to service a query
24 | // that was malformed in its operator/value combination.
25 | ErrMalformedQuery = errors.New("operator requires values")
26 |
27 | // ErrQueryUnsupportedValues creates an error that represents an unsupported return value from the
28 | // specified query.
29 | ErrQueryUnsupportedValues = errors.New("operator does not support values")
30 |
31 | // ErrLabelNotSpecified creates an error that represents the fact that we were requested to service a query
32 | // that was malformed in its label specification.
33 | ErrLabelNotSpecified = errors.New("label not specified")
34 | )
35 |
--------------------------------------------------------------------------------
/pkg/naming/lbl_res.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2019 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package naming
18 |
19 | import (
20 | "bytes"
21 | "fmt"
22 | "regexp"
23 | "text/template"
24 |
25 | "k8s.io/apimachinery/pkg/runtime/schema"
26 |
27 | pmodel "github.com/prometheus/common/model"
28 | )
29 |
30 | // labelGroupResExtractor extracts schema.GroupResources from series labels.
31 | type labelGroupResExtractor struct {
32 | regex *regexp.Regexp
33 |
34 | resourceInd int
35 | groupInd *int
36 | }
37 |
38 | // newLabelGroupResExtractor creates a new labelGroupResExtractor for labels whose form
39 | // matches the given template. It does so by creating a regular expression from the template,
40 | // so anything in the template which limits resource or group name length will cause issues.
41 | func newLabelGroupResExtractor(labelTemplate *template.Template) (*labelGroupResExtractor, error) {
42 | labelRegexBuff := new(bytes.Buffer)
43 | if err := labelTemplate.Execute(labelRegexBuff, schema.GroupResource{
44 | Group: "(?P.+?)",
45 | Resource: "(?P.+?)"},
46 | ); err != nil {
47 | return nil, fmt.Errorf("unable to convert label template to matcher: %v", err)
48 | }
49 | if labelRegexBuff.Len() == 0 {
50 | return nil, fmt.Errorf("unable to convert label template to matcher: empty template")
51 | }
52 | labelRegexRaw := "^" + labelRegexBuff.String() + "$"
53 | labelRegex, err := regexp.Compile(labelRegexRaw)
54 | if err != nil {
55 | return nil, fmt.Errorf("unable to convert label template to matcher: %v", err)
56 | }
57 |
58 | var groupInd *int
59 | var resInd *int
60 |
61 | for i, name := range labelRegex.SubexpNames() {
62 | switch name {
63 | case "group":
64 | ind := i // copy to avoid iteration variable reference
65 | groupInd = &ind
66 | case "resource":
67 | ind := i // copy to avoid iteration variable reference
68 | resInd = &ind
69 | }
70 | }
71 |
72 | if resInd == nil {
73 | return nil, fmt.Errorf("must include at least `{{.Resource}}` in the label template")
74 | }
75 |
76 | return &labelGroupResExtractor{
77 | regex: labelRegex,
78 | resourceInd: *resInd,
79 | groupInd: groupInd,
80 | }, nil
81 | }
82 |
83 | // GroupResourceForLabel extracts a schema.GroupResource from the given label, if possible.
84 | // The second return value indicates whether or not a potential group-resource was found in this label.
85 | func (e *labelGroupResExtractor) GroupResourceForLabel(lbl pmodel.LabelName) (schema.GroupResource, bool) {
86 | matchGroups := e.regex.FindStringSubmatch(string(lbl))
87 | if matchGroups != nil {
88 | group := ""
89 | if e.groupInd != nil {
90 | group = matchGroups[*e.groupInd]
91 | }
92 |
93 | return schema.GroupResource{
94 | Group: group,
95 | Resource: matchGroups[e.resourceInd],
96 | }, true
97 | }
98 |
99 | return schema.GroupResource{}, false
100 | }
101 |
--------------------------------------------------------------------------------
/pkg/naming/metric_namer.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2019 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package naming
18 |
19 | import (
20 | "fmt"
21 | "regexp"
22 |
23 | apimeta "k8s.io/apimachinery/pkg/api/meta"
24 | "k8s.io/apimachinery/pkg/labels"
25 | "k8s.io/apimachinery/pkg/runtime/schema"
26 |
27 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
28 | "sigs.k8s.io/prometheus-adapter/pkg/config"
29 | )
30 |
31 | // MetricNamer knows how to convert Prometheus series names and label names to
32 | // metrics API resources, and vice-versa. MetricNamers should be safe to access
33 | // concurrently. Returned group-resources are "normalized" as per the
34 | // MetricInfo#Normalized method. Group-resources passed as arguments must
35 | // themselves be normalized.
36 | type MetricNamer interface {
37 | // Selector produces the appropriate Prometheus series selector to match all
38 | // series handable by this namer.
39 | Selector() prom.Selector
40 | // FilterSeries checks to see which of the given series match any additional
41 | // constraints beyond the series query. It's assumed that the series given
42 | // already match the series query.
43 | FilterSeries(series []prom.Series) []prom.Series
44 | // MetricNameForSeries returns the name (as presented in the API) for a given series.
45 | MetricNameForSeries(series prom.Series) (string, error)
46 | // QueryForSeries returns the query for a given series (not API metric name), with
47 | // the given namespace name (if relevant), resource, and resource names.
48 | QueryForSeries(series string, resource schema.GroupResource, namespace string, metricSelector labels.Selector, names ...string) (prom.Selector, error)
49 | // QueryForExternalSeries returns the query for a given series (not API metric name), with
50 | // the given namespace name (if relevant), resource, and resource names.
51 | QueryForExternalSeries(series string, namespace string, targetLabels labels.Selector) (prom.Selector, error)
52 |
53 | ResourceConverter
54 | }
55 |
56 | func (n *metricNamer) Selector() prom.Selector {
57 | return n.seriesQuery
58 | }
59 |
60 | // ReMatcher either positively or negatively matches a regex
61 | type ReMatcher struct {
62 | regex *regexp.Regexp
63 | positive bool
64 | }
65 |
66 | func NewReMatcher(cfg config.RegexFilter) (*ReMatcher, error) {
67 | if cfg.Is != "" && cfg.IsNot != "" {
68 | return nil, fmt.Errorf("cannot have both an `is` (%q) and `isNot` (%q) expression in a single filter", cfg.Is, cfg.IsNot)
69 | }
70 | if cfg.Is == "" && cfg.IsNot == "" {
71 | return nil, fmt.Errorf("must have either an `is` or `isNot` expression in a filter")
72 | }
73 |
74 | var positive bool
75 | var regexRaw string
76 | if cfg.Is != "" {
77 | positive = true
78 | regexRaw = cfg.Is
79 | } else {
80 | positive = false
81 | regexRaw = cfg.IsNot
82 | }
83 |
84 | regex, err := regexp.Compile(regexRaw)
85 | if err != nil {
86 | return nil, fmt.Errorf("unable to compile series filter %q: %v", regexRaw, err)
87 | }
88 |
89 | return &ReMatcher{
90 | regex: regex,
91 | positive: positive,
92 | }, nil
93 | }
94 |
95 | func (m *ReMatcher) Matches(val string) bool {
96 | return m.regex.MatchString(val) == m.positive
97 | }
98 |
99 | type metricNamer struct {
100 | seriesQuery prom.Selector
101 | metricsQuery MetricsQuery
102 | nameMatches *regexp.Regexp
103 | nameAs string
104 | seriesMatchers []*ReMatcher
105 |
106 | ResourceConverter
107 | }
108 |
109 | // queryTemplateArgs are the arguments for the metrics query template.
110 | func (n *metricNamer) FilterSeries(initialSeries []prom.Series) []prom.Series {
111 | if len(n.seriesMatchers) == 0 {
112 | return initialSeries
113 | }
114 |
115 | finalSeries := make([]prom.Series, 0, len(initialSeries))
116 | SeriesLoop:
117 | for _, series := range initialSeries {
118 | for _, matcher := range n.seriesMatchers {
119 | if !matcher.Matches(series.Name) {
120 | continue SeriesLoop
121 | }
122 | }
123 | finalSeries = append(finalSeries, series)
124 | }
125 |
126 | return finalSeries
127 | }
128 |
129 | func (n *metricNamer) QueryForSeries(series string, resource schema.GroupResource, namespace string, metricSelector labels.Selector, names ...string) (prom.Selector, error) {
130 | return n.metricsQuery.Build(series, resource, namespace, nil, metricSelector, names...)
131 | }
132 |
133 | func (n *metricNamer) QueryForExternalSeries(series string, namespace string, metricSelector labels.Selector) (prom.Selector, error) {
134 | return n.metricsQuery.BuildExternal(series, namespace, "", []string{}, metricSelector)
135 | }
136 |
137 | func (n *metricNamer) MetricNameForSeries(series prom.Series) (string, error) {
138 | matches := n.nameMatches.FindStringSubmatchIndex(series.Name)
139 | if matches == nil {
140 | return "", fmt.Errorf("series name %q did not match expected pattern %q", series.Name, n.nameMatches.String())
141 | }
142 | outNameBytes := n.nameMatches.ExpandString(nil, n.nameAs, series.Name, matches)
143 | return string(outNameBytes), nil
144 | }
145 |
146 | // NamersFromConfig produces a MetricNamer for each rule in the given config.
147 | func NamersFromConfig(cfg []config.DiscoveryRule, mapper apimeta.RESTMapper) ([]MetricNamer, error) {
148 | namers := make([]MetricNamer, len(cfg))
149 |
150 | for i, rule := range cfg {
151 | resConv, err := NewResourceConverter(rule.Resources.Template, rule.Resources.Overrides, mapper)
152 | if err != nil {
153 | return nil, err
154 | }
155 |
156 | // queries are namespaced by default unless the rule specifically disables it
157 | namespaced := true
158 | if rule.Resources.Namespaced != nil {
159 | namespaced = *rule.Resources.Namespaced
160 | }
161 |
162 | metricsQuery, err := NewExternalMetricsQuery(rule.MetricsQuery, resConv, namespaced)
163 | if err != nil {
164 | return nil, fmt.Errorf("unable to construct metrics query associated with series query %q: %v", rule.SeriesQuery, err)
165 | }
166 |
167 | seriesMatchers := make([]*ReMatcher, len(rule.SeriesFilters))
168 | for i, filterRaw := range rule.SeriesFilters {
169 | matcher, err := NewReMatcher(filterRaw)
170 | if err != nil {
171 | return nil, fmt.Errorf("unable to generate series name filter associated with series query %q: %v", rule.SeriesQuery, err)
172 | }
173 | seriesMatchers[i] = matcher
174 | }
175 | if rule.Name.Matches != "" {
176 | matcher, err := NewReMatcher(config.RegexFilter{Is: rule.Name.Matches})
177 | if err != nil {
178 | return nil, fmt.Errorf("unable to generate series name filter from name rules associated with series query %q: %v", rule.SeriesQuery, err)
179 | }
180 | seriesMatchers = append(seriesMatchers, matcher)
181 | }
182 |
183 | var nameMatches *regexp.Regexp
184 | if rule.Name.Matches != "" {
185 | nameMatches, err = regexp.Compile(rule.Name.Matches)
186 | if err != nil {
187 | return nil, fmt.Errorf("unable to compile series name match expression %q associated with series query %q: %v", rule.Name.Matches, rule.SeriesQuery, err)
188 | }
189 | } else {
190 | // this will always succeed
191 | nameMatches = regexp.MustCompile(".*")
192 | }
193 | nameAs := rule.Name.As
194 | if nameAs == "" {
195 | // check if we have an obvious default
196 | subexpNames := nameMatches.SubexpNames()
197 | switch len(subexpNames) {
198 | case 1:
199 | // no capture groups, use the whole thing
200 | nameAs = "$0"
201 | case 2:
202 | // one capture group, use that
203 | nameAs = "$1"
204 | default:
205 | return nil, fmt.Errorf("must specify an 'as' value for name matcher %q associated with series query %q", rule.Name.Matches, rule.SeriesQuery)
206 | }
207 | }
208 |
209 | namer := &metricNamer{
210 | seriesQuery: prom.Selector(rule.SeriesQuery),
211 | metricsQuery: metricsQuery,
212 | nameMatches: nameMatches,
213 | nameAs: nameAs,
214 | seriesMatchers: seriesMatchers,
215 | ResourceConverter: resConv,
216 | }
217 |
218 | namers[i] = namer
219 | }
220 |
221 | return namers, nil
222 | }
223 |
--------------------------------------------------------------------------------
/pkg/naming/regex_matcher_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2019 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package naming
18 |
19 | import (
20 | "testing"
21 |
22 | "github.com/stretchr/testify/require"
23 |
24 | "sigs.k8s.io/prometheus-adapter/pkg/config"
25 | )
26 |
27 | func TestReMatcherIs(t *testing.T) {
28 | filter := config.RegexFilter{
29 | Is: "my_.*",
30 | }
31 |
32 | matcher, err := NewReMatcher(filter)
33 | require.NoError(t, err)
34 |
35 | result := matcher.Matches("my_label")
36 | require.True(t, result)
37 |
38 | result = matcher.Matches("your_label")
39 | require.False(t, result)
40 | }
41 |
42 | func TestReMatcherIsNot(t *testing.T) {
43 | filter := config.RegexFilter{
44 | IsNot: "my_.*",
45 | }
46 |
47 | matcher, err := NewReMatcher(filter)
48 | require.NoError(t, err)
49 |
50 | result := matcher.Matches("my_label")
51 | require.False(t, result)
52 |
53 | result = matcher.Matches("your_label")
54 | require.True(t, result)
55 | }
56 |
57 | func TestEnforcesIsOrIsNotButNotBoth(t *testing.T) {
58 | filter := config.RegexFilter{
59 | Is: "my_.*",
60 | IsNot: "your_.*",
61 | }
62 |
63 | _, err := NewReMatcher(filter)
64 | require.Error(t, err)
65 | }
66 |
--------------------------------------------------------------------------------
/pkg/naming/resource_converter.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2019 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package naming
18 |
19 | import (
20 | "bytes"
21 | "fmt"
22 | "strings"
23 | "sync"
24 | "text/template"
25 |
26 | pmodel "github.com/prometheus/common/model"
27 |
28 | apimeta "k8s.io/apimachinery/pkg/api/meta"
29 | "k8s.io/apimachinery/pkg/runtime/schema"
30 | "k8s.io/klog/v2"
31 |
32 | "sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
33 |
34 | prom "sigs.k8s.io/prometheus-adapter/pkg/client"
35 | "sigs.k8s.io/prometheus-adapter/pkg/config"
36 | )
37 |
38 | var (
39 | GroupNameSanitizer = strings.NewReplacer(".", "_", "-", "_")
40 | NsGroupResource = schema.GroupResource{Resource: "namespaces"}
41 | NodeGroupResource = schema.GroupResource{Resource: "nodes"}
42 | PVGroupResource = schema.GroupResource{Resource: "persistentvolumes"}
43 | )
44 |
45 | // ResourceConverter knows the relationship between Kubernetes group-resources and Prometheus labels,
46 | // and can convert between the two for any given label or series.
47 | type ResourceConverter interface {
48 | // ResourcesForSeries returns the group-resources associated with the given series,
49 | // as well as whether or not the given series has the "namespace" resource).
50 | ResourcesForSeries(series prom.Series) (res []schema.GroupResource, namespaced bool)
51 | // LabelForResource returns the appropriate label for the given resource.
52 | LabelForResource(resource schema.GroupResource) (pmodel.LabelName, error)
53 | }
54 |
55 | type resourceConverter struct {
56 | labelResourceMu sync.RWMutex
57 | labelToResource map[pmodel.LabelName]schema.GroupResource
58 | resourceToLabel map[schema.GroupResource]pmodel.LabelName
59 | labelResExtractor *labelGroupResExtractor
60 | mapper apimeta.RESTMapper
61 | labelTemplate *template.Template
62 | }
63 |
64 | // NewResourceConverter creates a ResourceConverter based on a generic template plus any overrides.
65 | // Either overrides or the template may be empty, but not both.
66 | func NewResourceConverter(resourceTemplate string, overrides map[string]config.GroupResource, mapper apimeta.RESTMapper) (ResourceConverter, error) {
67 | converter := &resourceConverter{
68 | labelToResource: make(map[pmodel.LabelName]schema.GroupResource),
69 | resourceToLabel: make(map[schema.GroupResource]pmodel.LabelName),
70 | mapper: mapper,
71 | }
72 |
73 | if resourceTemplate != "" {
74 | labelTemplate, err := template.New("resource-label").Delims("<<", ">>").Parse(resourceTemplate)
75 | if err != nil {
76 | return converter, fmt.Errorf("unable to parse label template %q: %v", resourceTemplate, err)
77 | }
78 | converter.labelTemplate = labelTemplate
79 |
80 | labelResExtractor, err := newLabelGroupResExtractor(labelTemplate)
81 | if err != nil {
82 | return converter, fmt.Errorf("unable to generate label format from template %q: %v", resourceTemplate, err)
83 | }
84 | converter.labelResExtractor = labelResExtractor
85 | }
86 |
87 | // invert the structure for consistency with the template
88 | for lbl, groupRes := range overrides {
89 | infoRaw := provider.CustomMetricInfo{
90 | GroupResource: schema.GroupResource{
91 | Group: groupRes.Group,
92 | Resource: groupRes.Resource,
93 | },
94 | }
95 | info, _, err := infoRaw.Normalized(converter.mapper)
96 | if err != nil {
97 | return nil, fmt.Errorf("unable to normalize group-resource %v: %v", groupRes, err)
98 | }
99 |
100 | converter.labelToResource[pmodel.LabelName(lbl)] = info.GroupResource
101 | converter.resourceToLabel[info.GroupResource] = pmodel.LabelName(lbl)
102 | }
103 |
104 | return converter, nil
105 | }
106 |
107 | func (r *resourceConverter) LabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) {
108 | r.labelResourceMu.RLock()
109 | // check if we have a cached copy or override
110 | lbl, ok := r.resourceToLabel[resource]
111 | r.labelResourceMu.RUnlock() // release before we call makeLabelForResource
112 | if ok {
113 | return lbl, nil
114 | }
115 |
116 | // NB: we don't actually care about the gap between releasing read lock
117 | // and acquiring the write lock -- if we do duplicate work sometimes, so be
118 | // it, as long as we're correct.
119 |
120 | // otherwise, use the template and save the result
121 | lbl, err := r.makeLabelForResource(resource)
122 | if err != nil {
123 | return "", fmt.Errorf("unable to convert resource %s into label: %v", resource.String(), err)
124 | }
125 | return lbl, nil
126 | }
127 |
128 | // makeLabelForResource constructs a label name for the given resource, and saves the result.
129 | // It must *not* be called under an existing lock.
130 | func (r *resourceConverter) makeLabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) {
131 | if r.labelTemplate == nil {
132 | return "", fmt.Errorf("no generic resource label form specified for this metric")
133 | }
134 | buff := new(bytes.Buffer)
135 |
136 | singularRes, err := r.mapper.ResourceSingularizer(resource.Resource)
137 | if err != nil {
138 | return "", fmt.Errorf("unable to singularize resource %s: %v", resource.String(), err)
139 | }
140 | convResource := schema.GroupResource{
141 | Group: GroupNameSanitizer.Replace(resource.Group),
142 | Resource: singularRes,
143 | }
144 |
145 | if err := r.labelTemplate.Execute(buff, convResource); err != nil {
146 | return "", err
147 | }
148 | if buff.Len() == 0 {
149 | return "", fmt.Errorf("empty label produced by label template")
150 | }
151 | lbl := pmodel.LabelName(buff.String())
152 |
153 | r.labelResourceMu.Lock()
154 | defer r.labelResourceMu.Unlock()
155 |
156 | r.resourceToLabel[resource] = lbl
157 | r.labelToResource[lbl] = resource
158 | return lbl, nil
159 | }
160 |
161 | func (r *resourceConverter) ResourcesForSeries(series prom.Series) ([]schema.GroupResource, bool) {
162 | // use an updates map to avoid having to drop the read lock to update the cache
163 | // until the end. Since we'll probably have few updates after the first run,
164 | // this should mean that we rarely have to hold the write lock.
165 | var resources []schema.GroupResource
166 | updates := make(map[pmodel.LabelName]schema.GroupResource)
167 | namespaced := false
168 |
169 | // use an anon func to get the right defer behavior
170 | func() {
171 | r.labelResourceMu.RLock()
172 | defer r.labelResourceMu.RUnlock()
173 |
174 | for lbl := range series.Labels {
175 | var groupRes schema.GroupResource
176 | var ok bool
177 |
178 | // check if we have an override
179 | if groupRes, ok = r.labelToResource[lbl]; ok {
180 | resources = append(resources, groupRes)
181 | } else if groupRes, ok = updates[lbl]; ok {
182 | resources = append(resources, groupRes)
183 | } else if r.labelResExtractor != nil {
184 | // if not, check if it matches the form we expect, and if so,
185 | // convert to a group-resource.
186 | if groupRes, ok = r.labelResExtractor.GroupResourceForLabel(lbl); ok {
187 | info, _, err := provider.CustomMetricInfo{GroupResource: groupRes}.Normalized(r.mapper)
188 | if err != nil {
189 | // this is likely to show up for a lot of labels, so make it a verbose info log
190 | klog.V(9).Infof("unable to normalize group-resource %s from label %q, skipping: %v", groupRes.String(), lbl, err)
191 | continue
192 | }
193 |
194 | groupRes = info.GroupResource
195 | resources = append(resources, groupRes)
196 | updates[lbl] = groupRes
197 | }
198 | }
199 |
200 | if groupRes != NsGroupResource && groupRes != NodeGroupResource && groupRes != PVGroupResource {
201 | namespaced = true
202 | }
203 | }
204 | }()
205 |
206 | // update the cache for next time. This should only be called by discovery,
207 | // so we don't really have to worry about the gap between read and write locks
208 | // (plus, we don't care if someone else updates the cache first, since the results
209 | // are necessarily the same, so at most we've done extra work).
210 | if len(updates) > 0 {
211 | r.labelResourceMu.Lock()
212 | defer r.labelResourceMu.Unlock()
213 |
214 | for lbl, groupRes := range updates {
215 | r.labelToResource[lbl] = groupRes
216 | }
217 | }
218 |
219 | return resources, namespaced
220 | }
221 |
--------------------------------------------------------------------------------
/pkg/resourceprovider/provider_suite_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2018 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package resourceprovider
18 |
19 | import (
20 | "testing"
21 |
22 | . "github.com/onsi/ginkgo"
23 | . "github.com/onsi/gomega"
24 | )
25 |
26 | func TestProvider(t *testing.T) {
27 | RegisterFailHandler(Fail)
28 | RunSpecs(t, "Resource Metrics Provider Suite")
29 | }
30 |
--------------------------------------------------------------------------------
/test/README.md:
--------------------------------------------------------------------------------
1 | # End-to-end tests
2 |
3 | ## With [kind](https://kind.sigs.k8s.io/)
4 |
5 | [`kind`](https://kind.sigs.k8s.io/) and `kubectl` are automatically downloaded
6 | except if `SKIP_INSTALL=true` is set.
7 | A `kind` cluster is automatically created before the tests, and deleted after
8 | the tests.
9 | The `prometheus-adapter` container image is build locally and imported
10 | into the cluster.
11 |
12 | ```bash
13 | KIND_E2E=true make test-e2e
14 | ```
15 |
16 | ## With an existing Kubernetes cluster
17 |
18 | If you already have a Kubernetes cluster, you can use:
19 |
20 | ```bash
21 | KUBECONFIG="/path/to/kube/config" REGISTRY="my.registry/prefix" make test-e2e
22 | ```
23 |
24 | - The cluster should not have a namespace `prometheus-adapter-e2e`.
25 | The namespace will be created and deleted as part of the E2E tests.
26 | - `KUBECONFIG` is the path of the [`kubeconfig` file].
27 | **Optional**, defaults to `${HOME}/.kube/config`
28 | - `REGISTRY` is the image registry where the container image should be pushed.
29 | **Required**.
30 |
31 | [`kubeconfig` file]: https://kubernetes.io/docs/concepts/configuration/organize-cluster-access-kubeconfig/
32 |
33 | ## Additional environment variables
34 |
35 | These environment variables may also be used (with any non-empty value):
36 |
37 | - `SKIP_INSTALL`: skip the installation of `kind` and `kubectl` binaries;
38 | - `SKIP_CLEAN_AFTER`: skip the deletion of resources (`Kind` cluster or
39 | Kubernetes namespace) and of the temporary directory `.e2e`;
40 | - `CLEAN_BEFORE`: clean before running the tests, e.g. if `SKIP_CLEAN_AFTER`
41 | was used on the previous run.
42 |
--------------------------------------------------------------------------------
/test/e2e/e2e_test.go:
--------------------------------------------------------------------------------
1 | /*
2 | Copyright 2022 The Kubernetes Authors.
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 | http://www.apache.org/licenses/LICENSE-2.0
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 | */
16 |
17 | package e2e
18 |
19 | import (
20 | "context"
21 | "fmt"
22 | "log"
23 | "os"
24 | "testing"
25 | "time"
26 |
27 | monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
28 | monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
29 | "github.com/stretchr/testify/assert"
30 | "github.com/stretchr/testify/require"
31 |
32 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33 | "k8s.io/apimachinery/pkg/util/wait"
34 | clientset "k8s.io/client-go/kubernetes"
35 | "k8s.io/client-go/tools/clientcmd"
36 | metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1"
37 | metrics "k8s.io/metrics/pkg/client/clientset/versioned"
38 | )
39 |
40 | const (
41 | ns = "prometheus-adapter-e2e"
42 | prometheusInstance = "prometheus"
43 | deployment = "prometheus-adapter"
44 | )
45 |
46 | var (
47 | client clientset.Interface
48 | promOpClient monitoring.Interface
49 | metricsClient metrics.Interface
50 | )
51 |
52 | func TestMain(m *testing.M) {
53 | kubeconfig := os.Getenv("KUBECONFIG")
54 | if len(kubeconfig) == 0 {
55 | log.Fatal("KUBECONFIG not provided")
56 | }
57 |
58 | var err error
59 | client, promOpClient, metricsClient, err = initializeClients(kubeconfig)
60 | if err != nil {
61 | log.Fatalf("Cannot create clients: %v", err)
62 | }
63 |
64 | ctx := context.Background()
65 | err = waitForPrometheusReady(ctx, ns, prometheusInstance)
66 | if err != nil {
67 | log.Fatalf("Prometheus instance 'prometheus' not ready: %v", err)
68 | }
69 | err = waitForDeploymentReady(ctx, ns, deployment)
70 | if err != nil {
71 | log.Fatalf("Deployment prometheus-adapter not ready: %v", err)
72 | }
73 |
74 | exitVal := m.Run()
75 | os.Exit(exitVal)
76 | }
77 |
78 | func initializeClients(kubeconfig string) (clientset.Interface, monitoring.Interface, metrics.Interface, error) {
79 | cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
80 | if err != nil {
81 | return nil, nil, nil, fmt.Errorf("Error during client configuration with %v", err)
82 | }
83 |
84 | clientSet, err := clientset.NewForConfig(cfg)
85 | if err != nil {
86 | return nil, nil, nil, fmt.Errorf("Error during client creation with %v", err)
87 | }
88 |
89 | promOpClient, err := monitoring.NewForConfig(cfg)
90 | if err != nil {
91 | return nil, nil, nil, fmt.Errorf("Error during dynamic client creation with %v", err)
92 | }
93 |
94 | metricsClientSet, err := metrics.NewForConfig(cfg)
95 | if err != nil {
96 | return nil, nil, nil, fmt.Errorf("Error during metrics client creation with %v", err)
97 | }
98 |
99 | return clientSet, promOpClient, metricsClientSet, nil
100 | }
101 |
102 | func waitForPrometheusReady(ctx context.Context, namespace string, name string) error {
103 | return wait.PollUntilContextTimeout(ctx, 5*time.Second, 120*time.Second, true, func(ctx context.Context) (bool, error) {
104 | prom, err := promOpClient.MonitoringV1().Prometheuses(ns).Get(ctx, name, metav1.GetOptions{})
105 | if err != nil {
106 | return false, err
107 | }
108 |
109 | var reconciled, available *monitoringv1.Condition
110 | for _, condition := range prom.Status.Conditions {
111 | cond := condition
112 | if cond.Type == monitoringv1.Reconciled {
113 | reconciled = &cond
114 | } else if cond.Type == monitoringv1.Available {
115 | available = &cond
116 | }
117 | }
118 |
119 | if reconciled == nil {
120 | log.Printf("Prometheus instance '%s': Waiting for reconciliation status...", name)
121 | return false, nil
122 | }
123 | if reconciled.Status != monitoringv1.ConditionTrue {
124 | log.Printf("Prometheus instance '%s': Reconciiled = %v. Waiting for reconciliation (reason %s, %q)...", name, reconciled.Status, reconciled.Reason, reconciled.Message)
125 | return false, nil
126 | }
127 |
128 | specReplicas := *prom.Spec.Replicas
129 | availableReplicas := prom.Status.AvailableReplicas
130 | if specReplicas != availableReplicas {
131 | log.Printf("Prometheus instance '%s': %v/%v pods are ready. Waiting for all pods to be ready...", name, availableReplicas, specReplicas)
132 | return false, err
133 | }
134 |
135 | if available == nil {
136 | log.Printf("Prometheus instance '%s': Waiting for Available status...", name)
137 | return false, nil
138 | }
139 | if available.Status != monitoringv1.ConditionTrue {
140 | log.Printf("Prometheus instance '%s': Available = %v. Waiting for Available status... (reason %s, %q)", name, available.Status, available.Reason, available.Message)
141 | return false, nil
142 | }
143 |
144 | log.Printf("Prometheus instance '%s': Ready.", name)
145 | return true, nil
146 | })
147 | }
148 |
149 | func waitForDeploymentReady(ctx context.Context, namespace string, name string) error {
150 | return wait.PollUntilContextTimeout(ctx, 5*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
151 | sts, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
152 | if err != nil {
153 | return false, err
154 | }
155 | if sts.Status.ReadyReplicas == *sts.Spec.Replicas {
156 | log.Printf("Deployment %s: %v/%v pods are ready.", name, sts.Status.ReadyReplicas, *sts.Spec.Replicas)
157 | return true, nil
158 | }
159 | log.Printf("Deployment %s: %v/%v pods are ready. Waiting for all pods to be ready...", name, sts.Status.ReadyReplicas, *sts.Spec.Replicas)
160 | return false, nil
161 | })
162 | }
163 |
164 | func TestNodeMetrics(t *testing.T) {
165 | ctx := context.Background()
166 | var nodeMetrics *metricsv1beta1.NodeMetricsList
167 | err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
168 | var err error
169 | nodeMetrics, err = metricsClient.MetricsV1beta1().NodeMetricses().List(ctx, metav1.ListOptions{})
170 | if err != nil {
171 | return false, err
172 | }
173 | nonEmptyNodeMetrics := len(nodeMetrics.Items) > 0
174 | if !nonEmptyNodeMetrics {
175 | t.Logf("Node metrics empty... Retrying.")
176 | }
177 | return nonEmptyNodeMetrics, nil
178 | })
179 | require.NoErrorf(t, err, "Node metrics should not be empty")
180 |
181 | for _, nodeMetric := range nodeMetrics.Items {
182 | positiveMemory := nodeMetric.Usage.Memory().CmpInt64(0)
183 | assert.Positivef(t, positiveMemory, "Memory usage for node %s is %v, should be > 0", nodeMetric.Name, nodeMetric.Usage.Memory())
184 |
185 | positiveCPU := nodeMetric.Usage.Cpu().CmpInt64(0)
186 | assert.Positivef(t, positiveCPU, "CPU usage for node %s is %v, should be > 0", nodeMetric.Name, nodeMetric.Usage.Cpu())
187 | }
188 | }
189 |
190 | func TestPodMetrics(t *testing.T) {
191 | ctx := context.Background()
192 | var podMetrics *metricsv1beta1.PodMetricsList
193 | err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 30*time.Second, true, func(ctx context.Context) (bool, error) {
194 | var err error
195 | podMetrics, err = metricsClient.MetricsV1beta1().PodMetricses(ns).List(ctx, metav1.ListOptions{})
196 | if err != nil {
197 | return false, err
198 | }
199 | nonEmptyNodeMetrics := len(podMetrics.Items) > 0
200 | if !nonEmptyNodeMetrics {
201 | t.Logf("Pod metrics empty... Retrying.")
202 | }
203 | return nonEmptyNodeMetrics, nil
204 | })
205 | require.NoErrorf(t, err, "Pod metrics should not be empty")
206 |
207 | for _, pod := range podMetrics.Items {
208 | for _, containerMetric := range pod.Containers {
209 | positiveMemory := containerMetric.Usage.Memory().CmpInt64(0)
210 | assert.Positivef(t, positiveMemory, "Memory usage for pod %s/%s is %v, should be > 0", pod.Name, containerMetric.Name, containerMetric.Usage.Memory())
211 | }
212 | }
213 | }
214 |
--------------------------------------------------------------------------------
/test/prometheus-manifests/cluster-role-binding.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRoleBinding
3 | metadata:
4 | name: prometheus
5 | roleRef:
6 | apiGroup: rbac.authorization.k8s.io
7 | kind: ClusterRole
8 | name: prometheus
9 | subjects:
10 | - kind: ServiceAccount
11 | name: prometheus
12 | namespace: prometheus-adapter-e2e
13 |
--------------------------------------------------------------------------------
/test/prometheus-manifests/cluster-role.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: rbac.authorization.k8s.io/v1
2 | kind: ClusterRole
3 | metadata:
4 | name: prometheus
5 | rules:
6 | - apiGroups: [""]
7 | resources:
8 | - nodes
9 | - nodes/metrics
10 | - services
11 | - endpoints
12 | - pods
13 | verbs: ["get", "list", "watch"]
14 | - apiGroups: [""]
15 | resources:
16 | - configmaps
17 | verbs: ["get"]
18 | - apiGroups:
19 | - networking.k8s.io
20 | resources:
21 | - ingresses
22 | verbs: ["get", "list", "watch"]
23 | - nonResourceURLs: ["/metrics"]
24 | verbs: ["get"]
25 |
--------------------------------------------------------------------------------
/test/prometheus-manifests/prometheus.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: monitoring.coreos.com/v1
2 | kind: Prometheus
3 | metadata:
4 | name: prometheus
5 | namespace: prometheus-adapter-e2e
6 | spec:
7 | replicas: 2
8 | serviceAccountName: prometheus
9 | serviceMonitorSelector: {}
10 |
--------------------------------------------------------------------------------
/test/prometheus-manifests/service-account.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: ServiceAccount
3 | metadata:
4 | name: prometheus
5 | namespace: prometheus-adapter-e2e
6 |
--------------------------------------------------------------------------------
/test/prometheus-manifests/service-monitor-kubelet.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: monitoring.coreos.com/v1
2 | kind: ServiceMonitor
3 | metadata:
4 | labels:
5 | app.kubernetes.io/name: kubelet
6 | name: kubelet
7 | namespace: prometheus-adapter-e2e
8 | spec:
9 | endpoints:
10 | - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
11 | honorLabels: true
12 | honorTimestamps: false
13 | interval: 10s
14 | path: /metrics/resource
15 | port: https-metrics
16 | scheme: https
17 | tlsConfig:
18 | insecureSkipVerify: true
19 | jobLabel: app.kubernetes.io/name
20 | namespaceSelector:
21 | matchNames:
22 | - kube-system
23 | selector:
24 | matchLabels:
25 | app.kubernetes.io/name: kubelet
26 |
--------------------------------------------------------------------------------
/test/prometheus-manifests/service.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | kind: Service
3 | metadata:
4 | name: prometheus
5 | namespace: prometheus-adapter-e2e
6 | spec:
7 | ports:
8 | - name: web
9 | port: 9090
10 | targetPort: web
11 | selector:
12 | app.kubernetes.io/instance: prometheus
13 | app.kubernetes.io/name: prometheus
14 | sessionAffinity: ClientIP
15 |
--------------------------------------------------------------------------------
/test/run-e2e-tests.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | # Copyright 2022 The Kubernetes Authors.
4 | #
5 | # Licensed under the Apache License, Version 2.0 (the "License");
6 | # you may not use this file except in compliance with the License.
7 | # You may obtain a copy of the License at
8 | #
9 | # http://www.apache.org/licenses/LICENSE-2.0
10 | #
11 | # Unless required by applicable law or agreed to in writing, software
12 | # distributed under the License is distributed on an "AS IS" BASIS,
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | # See the License for the specific language governing permissions and
15 | # limitations under the License.
16 |
17 | set -x
18 | set -o errexit
19 | set -o nounset
20 |
21 | # Tool versions
22 | K8S_VERSION=${KUBERNETES_VERSION:-v1.30.0} # cf https://hub.docker.com/r/kindest/node/tags
23 | KIND_VERSION=${KIND_VERSION:-v0.23.0} # cf https://github.com/kubernetes-sigs/kind/releases
24 | PROM_OPERATOR_VERSION=${PROM_OPERATOR_VERSION:-v0.73.2} # cf https://github.com/prometheus-operator/prometheus-operator/releases
25 |
26 | # Variables; set to empty if unbound/empty
27 | REGISTRY=${REGISTRY:-}
28 | KIND_E2E=${KIND_E2E:-}
29 | SKIP_INSTALL=${SKIP_INSTALL:-}
30 | SKIP_CLEAN_AFTER=${SKIP_CLEAN_AFTER:-}
31 | CLEAN_BEFORE=${CLEAN_BEFORE:-}
32 |
33 | # KUBECONFIG - will be overriden if a cluster is deployed with Kind
34 | KUBECONFIG=${KUBECONFIG:-"${HOME}/.kube/config"}
35 |
36 | # A temporary directory used by the tests
37 | E2E_DIR="${PWD}/.e2e"
38 |
39 | # The namespace where prometheus-adapter is deployed
40 | NAMESPACE="prometheus-adapter-e2e"
41 |
42 | if [[ -z "${REGISTRY}" && -z "${KIND_E2E}" ]]; then
43 | echo -e "Either REGISTRY or KIND_E2E should be set."
44 | exit 1
45 | fi
46 |
47 | function clean {
48 | if [[ -n "${KIND_E2E}" ]]; then
49 | kind delete cluster || true
50 | else
51 | kubectl delete -f ./deploy/manifests || true
52 | kubectl delete -f ./test/prometheus-manifests || true
53 | kubectl delete namespace "${NAMESPACE}" || true
54 | fi
55 |
56 | rm -rf "${E2E_DIR}"
57 | }
58 |
59 | if [[ -n "${CLEAN_BEFORE}" ]]; then
60 | clean
61 | fi
62 |
63 | function on_exit {
64 | local error_code="$?"
65 |
66 | echo "Obtaining prometheus-adapter pod logs..."
67 | kubectl logs -l app.kubernetes.io/name=prometheus-adapter -n "${NAMESPACE}" || true
68 |
69 | if [[ -z "${SKIP_CLEAN_AFTER}" ]]; then
70 | clean
71 | fi
72 |
73 | test "${error_code}" == 0 && return;
74 | }
75 | trap on_exit EXIT
76 |
77 | if [[ -d "${E2E_DIR}" ]]; then
78 | echo -e "${E2E_DIR} already exists."
79 | exit 1
80 | fi
81 | mkdir -p "${E2E_DIR}"
82 |
83 | if [[ -n "${KIND_E2E}" ]]; then
84 | # Install kubectl and kind, if we did not set SKIP_INSTALL
85 | if [[ -z "${SKIP_INSTALL}" ]]; then
86 | BIN="${E2E_DIR}/bin"
87 | mkdir -p "${BIN}"
88 | curl -Lo "${BIN}/kubectl" "https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/amd64/kubectl" && chmod +x "${BIN}/kubectl"
89 | curl -Lo "${BIN}/kind" "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" && chmod +x "${BIN}/kind"
90 | export PATH="${BIN}:${PATH}"
91 | fi
92 |
93 | kind create cluster --image "kindest/node:${K8S_VERSION}"
94 |
95 | REGISTRY="localhost"
96 |
97 | KUBECONFIG="${E2E_DIR}/kubeconfig"
98 | kind get kubeconfig > "${KUBECONFIG}"
99 | fi
100 |
101 | # Create the test namespace
102 | kubectl create namespace "${NAMESPACE}"
103 |
104 | export REGISTRY
105 | IMAGE_NAME="${REGISTRY}/prometheus-adapter-$(go env GOARCH)"
106 | IMAGE_TAG="v$(cat VERSION)"
107 |
108 | if [[ -n "${KIND_E2E}" ]]; then
109 | make container
110 | kind load docker-image "${IMAGE_NAME}:${IMAGE_TAG}"
111 | else
112 | make push
113 | fi
114 |
115 | # Install prometheus-operator
116 | kubectl apply -f "https://github.com/prometheus-operator/prometheus-operator/releases/download/${PROM_OPERATOR_VERSION}/bundle.yaml" --server-side
117 |
118 | # Install and setup prometheus
119 | kubectl apply -f ./test/prometheus-manifests --server-side
120 |
121 | # Customize prometheus-adapter manifests
122 | # TODO: use Kustomize or generate manifests from Jsonnet
123 | cp -r ./deploy/manifests "${E2E_DIR}/manifests"
124 | prom_url="http://prometheus.${NAMESPACE}.svc:9090/"
125 | sed -i -e "s|--prometheus-url=.*$|--prometheus-url=${prom_url}|g" "${E2E_DIR}/manifests/deployment.yaml"
126 | sed -i -e "s|image: .*$|image: ${IMAGE_NAME}:${IMAGE_TAG}|g" "${E2E_DIR}/manifests/deployment.yaml"
127 | find "${E2E_DIR}/manifests" -type f -exec sed -i -e "s|namespace: monitoring|namespace: ${NAMESPACE}|g" {} \;
128 |
129 | # Deploy prometheus-adapter
130 | kubectl apply -f "${E2E_DIR}/manifests" --server-side
131 |
132 | PROJECT_PREFIX="sigs.k8s.io/prometheus-adapter"
133 | export KUBECONFIG
134 | go test "${PROJECT_PREFIX}/test/e2e/" -v -count=1
135 |
--------------------------------------------------------------------------------