├── .gitignore ├── kubernetes ├── base │ ├── serviceaccount.yaml │ ├── kustomization.yaml │ ├── clusterrole.yaml │ ├── clusterrolebinding.yaml │ └── deployment.yaml ├── Dockerfile ├── overlays │ └── examples │ │ └── kustomization.yaml └── README.md ├── .golangci.yml ├── go.mod ├── SECURITY.md ├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ └── tag.yml ├── hack ├── check_golangci-lint.sh └── check_spdx.sh ├── pkg └── metrics │ ├── metrics.go │ ├── kubernetes.go │ ├── openstack.go │ ├── neutron.go │ ├── fwaasv1.go │ ├── fwaasv2.go │ ├── nova.go │ ├── loadbalancer.go │ └── cinder.go ├── LICENSE ├── Makefile ├── CONTRIBUTING.md ├── MAINTAINERS.md ├── docs ├── alerts.md └── metrics.md ├── README.md └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | kosmoo 2 | tmp/ 3 | -------------------------------------------------------------------------------- /kubernetes/base/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: kosmoo 6 | namespace: kube-system -------------------------------------------------------------------------------- /kubernetes/Dockerfile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | FROM alpine:3.19.1 3 | 4 | LABEL description="Kosmoo" 5 | 6 | RUN apk add --no-cache ca-certificates 7 | 8 | ADD kosmoo /bin/ 9 | 10 | ENTRYPOINT ["/bin/kosmoo"] 11 | -------------------------------------------------------------------------------- /kubernetes/overlays/examples/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | apiVersion: kustomize.config.k8s.io/v1beta1 3 | kind: Kustomization 4 | bases: 5 | - ../../base 6 | images: 7 | - name: ghcr.io/mercedes-benz/kosmoo/kosmoo 8 | newName: other/kosmoo 9 | newTag: new -------------------------------------------------------------------------------- /kubernetes/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | apiVersion: kustomize.config.k8s.io/v1beta1 3 | kind: Kustomization 4 | commonLabels: 5 | app: kosmoo 6 | resources: 7 | - clusterrole.yaml 8 | - clusterrolebinding.yaml 9 | - deployment.yaml 10 | - serviceaccount.yaml -------------------------------------------------------------------------------- /kubernetes/base/clusterrole.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | apiVersion: rbac.authorization.k8s.io/v1 3 | kind: ClusterRole 4 | metadata: 5 | name: monitoring:kosmoo 6 | rules: 7 | - apiGroups: [""] 8 | resources: 9 | - persistentvolumes 10 | verbs: ["get", "list", "watch"] -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | run: 3 | # timeout for analysis, e.g. 30s, 5m, default is 1m 4 | timeout: 5m 5 | linters: 6 | enable: 7 | - gofmt 8 | - goheader 9 | - govet 10 | - dupword 11 | linters-settings: 12 | goheader: 13 | template: | 14 | SPDX-License-Identifier: MIT -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | module github.com/mercedes-benz/kosmoo 3 | 4 | go 1.12 5 | 6 | require ( 7 | github.com/gophercloud/gophercloud v1.9.0 8 | github.com/prometheus/client_golang v1.18.0 9 | gopkg.in/ini.v1 v1.67.0 10 | k8s.io/api v0.27.10 11 | k8s.io/apimachinery v0.27.10 12 | k8s.io/client-go v0.27.10 13 | k8s.io/klog/v2 v2.120.1 14 | ) 15 | -------------------------------------------------------------------------------- /kubernetes/base/clusterrolebinding.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | kind: ClusterRoleBinding 3 | apiVersion: rbac.authorization.k8s.io/v1 4 | metadata: 5 | name: monitoring:kosmoo 6 | subjects: 7 | - kind: ServiceAccount 8 | name: kosmoo 9 | namespace: kube-system 10 | roleRef: 11 | kind: ClusterRole 12 | name: monitoring:kosmoo 13 | apiGroup: rbac.authorization.k8s.io 14 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | It is Daimler’s goal to offer its customers the best and most secure products such as connected cars and other services. Daimler values the work of security researchers and whitehat hackers who spend time and effort helping us to achieve this goal. 3 | 4 | For further Information please visit our Vulnerability Reporting Policy: 5 | 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | # To get started with Dependabot version updates, you'll need to specify which 3 | # package ecosystems to update and where the package manifests are located. 4 | # Please see the documentation for all configuration options: 5 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 6 | 7 | version: 2 8 | updates: 9 | - package-ecosystem: github-actions 10 | directory: / 11 | schedule: 12 | interval: weekly 13 | - package-ecosystem: gomod 14 | directory: / 15 | schedule: 16 | interval: weekly 17 | - package-ecosystem: docker 18 | directory: /kubernetes 19 | schedule: 20 | interval: weekly -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | # Pipeline to run CI 3 | name: Build 4 | on: 5 | pull_request: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | golangci-lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: golangci-lint 15 | uses: golangci/golangci-lint-action@v3 16 | with: 17 | version: latest 18 | build: 19 | runs-on: ubuntu-latest 20 | steps: 21 | - uses: actions/checkout@v4 22 | - uses: actions/cache@v4 23 | with: 24 | path: ~/go/pkg/mod 25 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 26 | restore-keys: | 27 | ${{ runner.os }}-go- 28 | - uses: actions/setup-go@v5 29 | with: 30 | go-version: '1.20.13' 31 | - name: make all 32 | run: | 33 | make all 34 | -------------------------------------------------------------------------------- /kubernetes/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Kosmoo Deployment 3 | 4 | *Kosmoo* uses [Kustomize](https://github.com/kubernetes-sigs/kustomize) to manage kubernetes yaml files. 5 | *Kustomize* is included in `kubectl` since version v1.14. 6 | Because of that the following documentation requires `kubectl` in version v1.14 or newer. 7 | 8 | ## Usage 9 | 10 | Execute from the `kosmoo` directory, to generate the base deployment: 11 | ``` 12 | kubectl kustomize kubernetes/base 13 | ``` 14 | To apply the base deployment to your cluster: 15 | ``` 16 | kubectl apply -k kubernetes/base 17 | ``` 18 | To generate the deployment with patches for the container image: 19 | ``` 20 | kubectl kustomize kubernetes/overlays/examples 21 | ``` 22 | To apply the deployment with patches for the container image to your cluster: 23 | ``` 24 | kubectl apply -k kubernetes/overlays/examples 25 | ``` -------------------------------------------------------------------------------- /hack/check_golangci-lint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: MIT 3 | 4 | set -e 5 | 6 | GOLANGCILINT_VERSION="1.52.2" 7 | GOLANGCILINT_FILENAME="golangci-lint-${GOLANGCILINT_VERSION}-linux-amd64.tar.gz" 8 | GOLANGCILINT_URL="https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCILINT_VERSION}/${GOLANGCILINT_FILENAME}" 9 | 10 | TMP_BIN="$(pwd)/tmp/bin" 11 | export PATH="${PATH}:${TMP_BIN}" 12 | 13 | if ! [ -x "$(command -v golangci-lint-${GOLANGCILINT_VERSION})" ]; then 14 | # pushd $(mktemp -d) 15 | echo '[golangci-lint/prepare]: golangci-lint is not installed. Downloading to tmp/bin' >&2 16 | 17 | wget -q "${GOLANGCILINT_URL}" 18 | tar -xf "${GOLANGCILINT_FILENAME}" 19 | 20 | mkdir -p "${TMP_BIN}" 21 | mv "${GOLANGCILINT_FILENAME%.tar.gz}/golangci-lint" "${TMP_BIN}/golangci-lint-${GOLANGCILINT_VERSION}" 22 | 23 | rm -rf "${GOLANGCILINT_FILENAME%.tar.gz}" "${GOLANGCILINT_FILENAME}" 24 | # popd 25 | fi 26 | 27 | echo '[golangci-lint/prepare]: running golangci-lint' >&2 28 | golangci-lint-${GOLANGCILINT_VERSION} run --skip-dirs ./vendor 29 | -------------------------------------------------------------------------------- /pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | // DefaultMetricsPrefix is the default prefix used for the exposed metrics. 11 | const DefaultMetricsPrefix = "kos" 12 | 13 | var ( 14 | metricsPrefix string 15 | ) 16 | 17 | // RegisterMetrics creates and registers the metrics for the /metrics endpoint. 18 | // It needs to get called before any scraping activity. 19 | func RegisterMetrics(prefix string) { 20 | metricsPrefix = prefix 21 | registerCinderMetrics() 22 | registerNeutronMetrics() 23 | registerLoadBalancerMetrics() 24 | registerOpenStackMetrics() 25 | registerFWaaSV1Metrics() 26 | registerFWaaSV2Metrics() 27 | registerServerMetrics() 28 | } 29 | 30 | // AddPrefix adds the given prefix to the string, if set 31 | func AddPrefix(name, prefix string) string { 32 | if prefix == "" { 33 | return name 34 | } 35 | return strings.ToLower(fmt.Sprintf("%s_%s", prefix, name)) 36 | } 37 | 38 | func generateName(name string) string { 39 | return AddPrefix(name, metricsPrefix) 40 | } 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2019 Daimler TSS GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | SHELL := /bin/bash 4 | 5 | # The name of the executable (default is current directory name) 6 | TARGET := $(shell echo $${PWD\#\#*/}) 7 | 8 | GOOS ?= $(shell go env GOOS) 9 | VERSION ?= $(shell git describe --tags --exact-match || \ 10 | git describe --exact-match 2> /dev/null || \ 11 | git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8) 12 | 13 | SRCS = $(shell find . -type f -name '*.go' -not -path "./vendor/*") 14 | 15 | REGISTRY ?= ghcr.io/mercedes-benz/kosmoo 16 | 17 | all: test build docker 18 | 19 | build: 20 | CGO_ENABLED=0 GOOS=$(GOOS) go build \ 21 | -o kosmoo \ 22 | ./main.go 23 | 24 | docker: build 25 | cp kosmoo kubernetes/ 26 | docker build -t $(REGISTRY)/kosmoo:$(VERSION) kubernetes/ 27 | rm kubernetes/kosmoo 28 | 29 | push: 30 | docker push $(REGISTRY)/kosmoo:$(VERSION) 31 | 32 | fmt: 33 | @gofmt -l -w $(SRCS) 34 | 35 | test: vet fmtcheck spdxcheck lint 36 | 37 | vet: 38 | go vet ./... 39 | 40 | lint: 41 | @hack/check_golangci-lint.sh 42 | 43 | fmtcheck: 44 | @gofmt -l -s $(SRCS) | read; if [ $$? == 0 ]; then echo "gofmt check failed for:"; gofmt -l -s $(SRCS); exit 1; fi 45 | 46 | spdxcheck: 47 | @hack/check_spdx.sh 48 | 49 | version: 50 | @echo $(VERSION) 51 | -------------------------------------------------------------------------------- /.github/workflows/tag.yml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | 3 | name: release 4 | 5 | on: 6 | push: 7 | tags: 8 | - 'v*.*.*' 9 | 10 | jobs: 11 | publish: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: checkout 15 | uses: actions/checkout@v4 16 | - uses: actions/cache@v4 17 | with: 18 | path: ~/go/pkg/mod 19 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 20 | restore-keys: | 21 | ${{ runner.os }}-go- 22 | - uses: actions/setup-go@v5 23 | with: 24 | go-version: '1.20.4' 25 | - name: make all 26 | run: | 27 | make all 28 | - name: generate changelog from git 29 | run: | 30 | echo "Image is available at \`ghcr.io/mercedes-benz/kosmoo/kosmoo:$(git describe --tags --exact-match)\`." > ${{ github.workflow }}-CHANGELOG.txt 31 | git log --format=format:"* %h %s" $(git describe --tags --abbrev=0 @^)..@ >> ${{ github.workflow }}-CHANGELOG.txt 32 | - name: push to package registry 33 | run: | 34 | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin 35 | make push 36 | - name: Release 37 | uses: ncipollo/release-action@v1 38 | with: 39 | artifacts: "kosmoo,LICENSE" 40 | bodyFile: "${{ github.workflow }}-CHANGELOG.txt" 41 | token: ${{ secrets.GITHUB_TOKEN }} 42 | draft: true 43 | -------------------------------------------------------------------------------- /kubernetes/base/deployment.yaml: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: MIT 2 | apiVersion: apps/v1 3 | kind: Deployment 4 | metadata: 5 | name: kosmoo 6 | namespace: kube-system 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | name: kosmoo 12 | template: 13 | metadata: 14 | annotations: 15 | prometheus.io/path: /metrics 16 | prometheus.io/port: "9183" 17 | prometheus.io/scrape: "true" 18 | labels: 19 | name: kosmoo 20 | spec: 21 | containers: 22 | - args: 23 | - -refresh-interval=300 24 | - -cloud-conf=/etc/cloud.conf 25 | image: ghcr.io/mercedes-benz/kosmoo/kosmoo:latest 26 | imagePullPolicy: Always 27 | name: exporter 28 | ports: 29 | - containerPort: 9183 30 | name: kosmoo 31 | protocol: TCP 32 | securityContext: 33 | privileged: false 34 | # necessary because of /etc/cloud.conf readability 35 | runAsUser: 0 36 | volumeMounts: 37 | - mountPath: /etc/cloud.conf 38 | name: cloud-conf 39 | resources: 40 | limits: 41 | memory: 40Mi 42 | requests: 43 | cpu: 5m 44 | memory: 40Mi 45 | volumes: 46 | - hostPath: 47 | path: /etc/cloud.conf 48 | type: "" 49 | name: cloud-conf 50 | serviceAccountName: kosmoo -------------------------------------------------------------------------------- /hack/check_spdx.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # SPDX-License-Identifier: MIT 3 | 4 | set -e 5 | 6 | array_contains() { 7 | local search="$1" 8 | local element 9 | shift 10 | for element; do 11 | if [[ "${element}" == "${search}" ]]; then 12 | return 0 13 | fi 14 | done 15 | return 1 16 | } 17 | 18 | HEADER="SPDX-License-Identifier: MIT" 19 | HEADER_WHITELIST=("./go.sum" "./LICENSE" "./kosmoo" "./logo/logo.png") 20 | 21 | all_files=() 22 | export IFS=$'\n' 23 | while IFS='' read -r line; do all_error_files+=("$line"); done < <(find . -type f -not -path './.git*' -not -path './vendor/*' -not -path './tmp/*' -not -path './.idea/*') 24 | unset IFS 25 | 26 | errors=() 27 | for file in "${all_error_files[@]}"; do 28 | array_contains "$file" "${HEADER_WHITELIST[@]}" && in_whitelist=$? || in_whitelist=$? 29 | if [[ "${in_whitelist}" -eq "0" ]]; then 30 | continue 31 | fi 32 | set +e 33 | matches=$(head -n 2 $file | grep "$HEADER" | wc -l) 34 | set -e 35 | if [[ "$matches" -ne "1" ]]; then 36 | errors+=( "${file}" ) 37 | echo "error checking ${file} for the SPDX header" 38 | fi 39 | done 40 | 41 | if [ ${#errors[@]} -eq 0 ]; then 42 | echo 'Congratulations! All source files have been checked for SPDX header.' 43 | else 44 | echo 45 | echo 'Please review the above files. They seem to miss the following header as comment:' 46 | echo "$HEADER" 47 | echo 48 | exit 1 49 | fi 50 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | # Contributing 3 | 4 | This document explains how to contribute to this project. 5 | By contributing you will agree that your contribution will be put under the same license as this repository. 6 | 7 | ## Table of Contents 8 | - CLA 9 | - Communication 10 | - Contributions 11 | - Quality 12 | 13 | ## CLA 14 | 15 | Before you can contribute, you will need to sign the [Contributor License Agreement](https://github.com/mercedes-benz/foss/blob/master/CONTRIBUTORS_LICENSE_AGREEMENT.md). 16 | 17 | ## Communication 18 | 19 | For communication please respect our [FOSS Code of Conduct](https://github.com/mercedes-benz/foss/blob/master/CODE_OF_CONDUCT.md). 20 | 21 | The following communication channels exist for this project: 22 | - GitHub for reporting and claiming issues: https://github.com/mercedes-benz/kosmoo 23 | 24 | Transparent and open communication is important to us. 25 | Thus, all project-related communication should happen only through these channels and in English. 26 | Issue-related communication should happen within the concerned issue. 27 | 28 | ## Contributions 29 | If you would like to contribute code you can do so through GitHub by forking the repository and sending a pull request. 30 | 31 | When submitting code, please make every effort to follow existing conventions and style in order to keep the code as readable as possible. 32 | 33 | If you are new to contributing in GitHub, [First Contributions](https://github.com/firstcontributions/first-contributions) might be a good starting point. 34 | 35 | Before you can contribute, you will need to sign our CLA and send the signed CLA to CLA-mbti@mercedes-benz.com. 36 | 37 | ## Quality 38 | Please ensure that for all contributions, the corresponding documentation is in-sync and up-to-date. All documentation should be in English language. 39 | 40 | We assume that for every non-trivial contribution, the project has been built and tested prior to the contribution. 41 | -------------------------------------------------------------------------------- /pkg/metrics/kubernetes.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | 9 | corev1 "k8s.io/api/core/v1" 10 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | const ( 16 | cinderCSIDriver = "cinder.csi.openstack.org" 17 | ) 18 | 19 | func getPVsByCinderID(clientset *kubernetes.Clientset) (map[string]corev1.PersistentVolume, error) { 20 | pvsList, err := clientset.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{}) 21 | if err != nil { 22 | return nil, fmt.Errorf("unable to list pvs: %s", err) 23 | } 24 | 25 | pvs := map[string]corev1.PersistentVolume{} 26 | for _, pv := range pvsList.Items { 27 | if pv.Spec.Cinder != nil { 28 | pvs[pv.Spec.Cinder.VolumeID] = pv 29 | } else if pv.Spec.CSI != nil { 30 | if pv.Spec.CSI.Driver == cinderCSIDriver { 31 | pvs[pv.Spec.CSI.VolumeHandle] = pv 32 | } else { 33 | klog.V(8).Infof("ignoring pv %s: unimplemented csi-driver", pv.GetName()) 34 | } 35 | } else { 36 | klog.V(8).Infof("ignoring pv %s: unimplemented volume plugin", pv.GetName()) 37 | } 38 | } 39 | 40 | return pvs, nil 41 | } 42 | 43 | // extractK8sMetadata tries to extract the following data from a pv: 44 | // "pvc_name", "pvc_namespace", "pv_name", "storage_class", "reclaim_policy", "fs_type" 45 | func extractK8sMetadata(pv *corev1.PersistentVolume) []string { 46 | if pv == nil { 47 | return []string{"", "", "", "", "", ""} 48 | } 49 | 50 | var fsType string 51 | if pv.Spec.Cinder != nil { 52 | fsType = pv.Spec.Cinder.FSType 53 | } else if pv.Spec.CSI != nil { 54 | fsType = pv.Spec.CSI.FSType 55 | } 56 | 57 | var claimName, claimNamespace string 58 | if pv.Spec.ClaimRef != nil { 59 | claimName = pv.Spec.ClaimRef.Name 60 | claimNamespace = pv.Spec.ClaimRef.Namespace 61 | } 62 | 63 | return []string{ 64 | claimName, 65 | claimNamespace, 66 | pv.GetName(), 67 | pv.Spec.StorageClassName, 68 | string(pv.Spec.PersistentVolumeReclaimPolicy), 69 | fsType, 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/metrics/openstack.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "time" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | type openstackMetrics struct { 12 | duration *prometheus.HistogramVec 13 | total *prometheus.CounterVec 14 | errors *prometheus.CounterVec 15 | } 16 | 17 | var ( 18 | requestMetrics *openstackMetrics 19 | ) 20 | 21 | func registerOpenStackMetrics() { 22 | requestMetrics = &openstackMetrics{ 23 | duration: prometheus.NewHistogramVec( 24 | prometheus.HistogramOpts{ 25 | Name: generateName("openstack_api_request_duration_seconds"), 26 | Help: "Latency of an OpenStack API call", 27 | }, []string{"request"}), 28 | total: prometheus.NewCounterVec( 29 | prometheus.CounterOpts{ 30 | Name: generateName("openstack_api_requests_total"), 31 | Help: "Total number of OpenStack API calls", 32 | }, []string{"request"}), 33 | errors: prometheus.NewCounterVec( 34 | prometheus.CounterOpts{ 35 | Name: generateName("openstack_api_request_errors_total"), 36 | Help: "Total number of errors for an OpenStack API call", 37 | }, []string{"request"}), 38 | } 39 | 40 | prometheus.MustRegister(requestMetrics.duration) 41 | prometheus.MustRegister(requestMetrics.total) 42 | prometheus.MustRegister(requestMetrics.errors) 43 | } 44 | 45 | // openStackMetric indicates the context for OpenStack metrics. 46 | type openStackMetric struct { 47 | start time.Time 48 | attributes []string 49 | } 50 | 51 | // newOpenStackMetric creates a new MetricContext. 52 | func newOpenStackMetric(resource string, request string) *openStackMetric { 53 | return &openStackMetric{ 54 | start: time.Now(), 55 | attributes: []string{resource + "_" + request}, 56 | } 57 | } 58 | 59 | // Observe records the request latency and counts the errors. 60 | func (mc *openStackMetric) Observe(err error) error { 61 | requestMetrics.duration.WithLabelValues(mc.attributes...).Observe( 62 | time.Since(mc.start).Seconds()) 63 | requestMetrics.total.WithLabelValues(mc.attributes...).Inc() 64 | if err != nil { 65 | requestMetrics.errors.WithLabelValues(mc.attributes...).Inc() 66 | } 67 | return err 68 | } 69 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Kosmoo Maintainers 4 | 5 | ## Maintainers 6 | 7 | | Maintainer | Email | GitHub ID | Affiliation | Joined | 8 | |------------------|--------------------------------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|------------| 9 | | Sean Schneeweiss | | [seanschneeweiss](https://github.com/seanschneeweiss) | Mercedes-Benz Tech Innovation GmbH, [imprint](https://github.com/mercedes-benz/foss/blob/master/PROVIDER_INFORMATION.md) | 2022-03-18 | 10 | 11 | ## Emeritus Maintainers 12 | 13 | | Maintainer | Email | GitHub ID | Affiliation | Joined | Left | 14 | |---------------------|-----------------------------------|-------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------|------------|------------| 15 | | Stefan Büringer | | [sbueringer](https://github.com/sbueringer) | Daimler TSS GmbH, [imprint](https://github.com/mercedes-benz/foss/blob/master/PROVIDER_INFORMATION.md) | - | 2021-06-30 | 16 | | Christian Schlotter | | [chrischdi](https://github.com/chrischdi) | Daimler TSS GmbH, [imprint](https://github.com/mercedes-benz/foss/blob/master/PROVIDER_INFORMATION.md) | 2019-08-07 | 2022-03-31 | 17 | | Johannes Frey | | [johannesfrey](https://github.com/johannesfrey) | Mercedes-Benz Tech Innovation GmbH, [imprint](https://github.com/mercedes-benz/foss/blob/master/PROVIDER_INFORMATION.md) | 2022-04-20 | 2023-12-18 | 18 | | Tobias Giese | | [tobiasgiese](https://github.com/tobiasgiese) | Mercedes-Benz Tech Innovation GmbH, [imprint](https://github.com/mercedes-benz/foss/blob/master/PROVIDER_INFORMATION.md) | 2022-03-18 | 2024-06-30 | 19 | -------------------------------------------------------------------------------- /docs/alerts.md: -------------------------------------------------------------------------------- 1 | 2 | # Prometheus alerts 3 | 4 | The following snippet shows some alerts which may be helpful for monitoring a Kubernetes cluster which runs on OpenStack. 5 | The rules may also depend on metrics exposed by [kube-state-metrics](https://github.com/kubernetes/kube-state-metrics). 6 | This alerts assume that the parameter `metrics-prefix` was not changed and still set to the default `kos` 7 | 8 | ``` 9 | - alert: CinderDiskStuck 10 | expr: kos_cinder_volume_status{status!~"available|in-use"} == 1 11 | for: 30m 12 | labels: 13 | severity: critical 14 | team: iaas 15 | annotations: 16 | summary: Cinder disk {{ $labels.id }} is stuck in {{ $labels.status }} (node {{ $labels.node }}) 17 | impact: Cinder disk cannot be attached/detached/deleted. Pods may not come up again. 18 | action: Check PVC {{ $labels.pvc_name }} in {{ $labels.pvc_namespace}} namespace and OpenStack disk. 19 | - alert: CinderDiskWithoutPV 20 | expr: | 21 | kos_cinder_volume_status{name=~"(pvc-.+|kubernetes-dynamic-pvc.+)",pv_name=""} == 1 22 | for: 30m 23 | labels: 24 | severity: warning 25 | team: caas 26 | annotations: 27 | summary: Cinder disk {{ $labels.id }} has no corresponding Kubernetes PV 28 | impact: The Cinder disk exists but has no corresponding Kubernetes PV. It occupies disk quota and cannot be deleted by CaaS users any more. 29 | action: Check and delete Cinder volume. 30 | - alert: CinderDiskAvailableStateMismatch 31 | expr: kos_cinder_volume_attached_at{status!="in-use"} > 0 32 | for: 30m 33 | labels: 34 | severity: critical 35 | team: iaas 36 | annotations: 37 | summary: Cinder disk {{ $labels.id }} is available but attached to node {{ $labels.node }} 38 | impact: Cinder disk is in available state, but has attachment information. Reattaching the disk will likely fail. 39 | action: Check and repair availability/attachment information. Attaching to the node might repair the broken state. 40 | - alert: CinderStateUnknown 41 | expr: sum(kos_cinder_volume_status) without(status) == 0 42 | for: 10m 43 | labels: 44 | severity: warning 45 | team: caas 46 | annotations: 47 | summary: Cinder disk {{ $labels.id }} has unknown state 48 | impact: Aliens invaded. A bit flipped. OpenStack my be broken. At least, we don't know what's going on right now. 49 | action: Check volume state and extend OpenStack Exporter code if required. 50 | ``` -------------------------------------------------------------------------------- /pkg/metrics/neutron.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "github.com/gophercloud/gophercloud" 7 | "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | var ( 13 | neutronFloatingIPStatus *prometheus.GaugeVec 14 | neutronFloatingIPCreated *prometheus.GaugeVec 15 | neutronFloatingIPUpdatedAt *prometheus.GaugeVec 16 | 17 | // Status from https://docs.openstack.org/api-ref/network/v2/index.html?expanded=show-floating-ip-details-detail#show-floating-ip-details 18 | floatingIpStatus = []string{"ACTIVE", "DOWN", "ERROR"} 19 | 20 | floatingIPLabels = []string{"id", "floating_ip", "fixed_ip", "port_id"} 21 | ) 22 | 23 | func registerNeutronMetrics() { 24 | neutronFloatingIPStatus = prometheus.NewGaugeVec( 25 | prometheus.GaugeOpts{ 26 | Name: generateName("neutron_floating_ip_status"), 27 | Help: "Neutron floating ip status", 28 | }, 29 | append(floatingIPLabels, "status"), 30 | ) 31 | neutronFloatingIPCreated = prometheus.NewGaugeVec( 32 | prometheus.GaugeOpts{ 33 | Name: generateName("neutron_floatingip_created_at"), 34 | Help: "Neutron floating ip created at", 35 | }, 36 | floatingIPLabels, 37 | ) 38 | neutronFloatingIPUpdatedAt = prometheus.NewGaugeVec( 39 | prometheus.GaugeOpts{ 40 | Name: generateName("neutron_floatingip_updated_at"), 41 | Help: "Neutron floating ip updated at", 42 | }, 43 | floatingIPLabels, 44 | ) 45 | 46 | prometheus.MustRegister(neutronFloatingIPStatus) 47 | prometheus.MustRegister(neutronFloatingIPCreated) 48 | prometheus.MustRegister(neutronFloatingIPUpdatedAt) 49 | } 50 | 51 | // PublishNeutronMetrics makes the list request to the neutron api and passes 52 | // the result to a publish function. 53 | func PublishNeutronMetrics(neutronClient *gophercloud.ServiceClient, tenantID string) error { 54 | // first step: gather the data 55 | mc := newOpenStackMetric("floating_ip", "list") 56 | pages, err := floatingips.List(neutronClient, floatingips.ListOpts{}).AllPages() 57 | if mc.Observe(err) != nil { 58 | // only warn, maybe the next list will work. 59 | klog.Warningf("Unable to list floating ips: %v", err) 60 | return err 61 | } 62 | floatingIPList, err := floatingips.ExtractFloatingIPs(pages) 63 | if err != nil { 64 | // only warn, maybe the next extract will work. 65 | klog.Warningf("Unable to extract floating ips: %v", err) 66 | return err 67 | } 68 | 69 | // second step: reset the old metrics 70 | neutronFloatingIPStatus.Reset() 71 | neutronFloatingIPUpdatedAt.Reset() 72 | 73 | // third step: publish the metrics 74 | for _, fip := range floatingIPList { 75 | publishFloatingIPMetric(fip) 76 | } 77 | 78 | return nil 79 | } 80 | 81 | // publishFloatingIPMetric extracts data from a floating ip and exposes the metrics via prometheus 82 | func publishFloatingIPMetric(fip floatingips.FloatingIP) { 83 | labels := []string{fip.ID, fip.FloatingIP, fip.FixedIP, fip.PortID} 84 | 85 | neutronFloatingIPCreated.WithLabelValues(labels...).Set(float64(fip.CreatedAt.Unix())) 86 | neutronFloatingIPUpdatedAt.WithLabelValues(labels...).Set(float64(fip.UpdatedAt.Unix())) 87 | 88 | for _, status := range floatingIpStatus { 89 | statusLabels := append(labels, status) 90 | neutronFloatingIPStatus.WithLabelValues(statusLabels...).Set(boolFloat64(fip.Status == status)) 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /pkg/metrics/fwaasv1.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "github.com/gophercloud/gophercloud" 7 | "github.com/gophercloud/gophercloud/openstack/common/extensions" 8 | "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/fwaas/firewalls" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | var ( 14 | firewallV1AdminStateUp *prometheus.GaugeVec 15 | firewallV1Status *prometheus.GaugeVec 16 | 17 | // according to the nl_constants from https://github.com/openstack/neutron-fwaas/blob/stable/ocata/neutron_fwaas/services/firewall/fwaas_plugin.py 18 | // we will get the following firewall states 19 | firewallV1States = []string{"ACTIVE", "DOWN", "ERROR", "INACTIVE", "PENDING_CREATE", "PENDING_UPDATE", "PENDING_DELETE"} 20 | 21 | firewallV1Labels = []string{"id", "name", "description", "policyID", "projectID"} 22 | ) 23 | 24 | func registerFWaaSV1Metrics() { 25 | firewallV1AdminStateUp = prometheus.NewGaugeVec( 26 | prometheus.GaugeOpts{ 27 | Name: generateName("firewall_v1_admin_state_up"), 28 | Help: "Firewall v1 status", 29 | }, 30 | firewallV1Labels, 31 | ) 32 | firewallV1Status = prometheus.NewGaugeVec( 33 | prometheus.GaugeOpts{ 34 | Name: generateName("firewall_v1_status"), 35 | Help: "Firewall v1 status", 36 | }, 37 | append(firewallV1Labels, "status"), 38 | ) 39 | 40 | prometheus.MustRegister(firewallV1AdminStateUp) 41 | prometheus.MustRegister(firewallV1Status) 42 | } 43 | 44 | // PublishFirewallV1Metrics makes the list request to the firewall api and 45 | // passes the result to a publish function. It only does that when the extension 46 | // is available in neutron. 47 | func PublishFirewallV1Metrics(client *gophercloud.ServiceClient, tenantID string) error { 48 | // check if Neutron extenstion FWaaS v1 is available. 49 | fwaasV1Extension := extensions.Get(client, "fwaas") 50 | if fwaasV1Extension.Body != nil { 51 | return publishFirewallV1Metrics(client, tenantID) 52 | } 53 | 54 | // reset metrics if fwaas v1 is not available to not publish them anymore 55 | resetFirewallV1Metrics() 56 | klog.Info("skipping Firewall metrics as FWaaS v1 is not enabled") 57 | return nil 58 | } 59 | 60 | func publishFirewallV1Metrics(client *gophercloud.ServiceClient, tenantID string) error { 61 | // first step: gather the data 62 | mc := newOpenStackMetric("firewall", "list") 63 | pages, err := firewalls.List(client, firewalls.ListOpts{}).AllPages() 64 | if mc.Observe(err) != nil { 65 | // only warn, maybe the next list will work. 66 | klog.Warningf("Unable to list firewalls: %v", err) 67 | return err 68 | } 69 | firewallsList, err := firewalls.ExtractFirewalls(pages) 70 | if err != nil { 71 | // only warn, maybe the next publish will work. 72 | klog.Warningf("Unable to extract firewalls: %v", err) 73 | return err 74 | } 75 | 76 | // second step: reset the old metrics 77 | resetFirewallV1Metrics() 78 | 79 | // third step: publish the metrics 80 | for _, fw := range firewallsList { 81 | publishFirewallV1Metric(fw) 82 | } 83 | 84 | return nil 85 | } 86 | 87 | // resetFirewallV1Metrics resets the firewall v1 metrics 88 | func resetFirewallV1Metrics() { 89 | firewallV1AdminStateUp.Reset() 90 | firewallV1Status.Reset() 91 | } 92 | 93 | // publishFirewallV1Metric extracts data from a firewall and exposes the metrics via prometheus 94 | func publishFirewallV1Metric(fw firewalls.Firewall) { 95 | labels := []string{fw.ID, fw.Name, fw.Description, fw.PolicyID, fw.ProjectID} 96 | firewallV1AdminStateUp.WithLabelValues(labels...).Set(boolFloat64(fw.AdminStateUp)) 97 | 98 | // create one metric per status 99 | for _, state := range firewallV1States { 100 | stateLabels := append(labels, state) 101 | firewallV1Status.WithLabelValues(stateLabels...).Set(boolFloat64(fw.Status == state)) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /pkg/metrics/fwaasv2.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "github.com/gophercloud/gophercloud" 7 | "github.com/gophercloud/gophercloud/openstack/common/extensions" 8 | "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/fwaas_v2/groups" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | var ( 14 | firewallV2GroupAdminStateUp *prometheus.GaugeVec 15 | firewallV2GroupStatus *prometheus.GaugeVec 16 | 17 | // according to the nl_constants from https://github.com/openstack/neutron-fwaas/blob/stable/ocata/neutron_fwaas/services/firewall/fwaas_plugin.py 18 | // we will get the following firewall states 19 | firewallV2States = []string{"ACTIVE", "DOWN", "ERROR", "INACTIVE", "PENDING_CREATE", "PENDING_DELETE", "PENDING_UPDATE"} 20 | 21 | firewallV2Labels = []string{"id", "name", "description", "ingressPolicyID", "egressPolicyID", "projectID"} 22 | ) 23 | 24 | func registerFWaaSV2Metrics() { 25 | firewallV2GroupAdminStateUp = prometheus.NewGaugeVec( 26 | prometheus.GaugeOpts{ 27 | Name: generateName("firewall_v2_group_admin_state_up"), 28 | Help: "Firewall v2 status", 29 | }, 30 | firewallV2Labels, 31 | ) 32 | firewallV2GroupStatus = prometheus.NewGaugeVec( 33 | prometheus.GaugeOpts{ 34 | Name: generateName("firewall_v2_group_status"), 35 | Help: "Firewall v2 status", 36 | }, 37 | append(firewallV2Labels, "status"), 38 | ) 39 | 40 | prometheus.MustRegister(firewallV2GroupAdminStateUp) 41 | prometheus.MustRegister(firewallV2GroupStatus) 42 | } 43 | 44 | // PublishFirewallV2Metrics makes the list request to the firewall api and 45 | // passes the result to a publish function. It only does that when the extension 46 | // is available in neutron. 47 | func PublishFirewallV2Metrics(client *gophercloud.ServiceClient, tenantID string) error { 48 | // check if Neutron extenstion FWaaS v2 is available. 49 | fwaasV2Extension := extensions.Get(client, "fwaas_v2") 50 | if fwaasV2Extension.Body != nil { 51 | return publishFirewallV2Metrics(client, tenantID) 52 | } 53 | 54 | // reset metrics if fwaas v2 is not available to not publish them anymore 55 | resetFirewallV2Metrics() 56 | klog.Info("skipping Firewall metrics as FWaaS v2 is not enabled") 57 | return nil 58 | } 59 | 60 | func publishFirewallV2Metrics(client *gophercloud.ServiceClient, tenantID string) error { 61 | // first step: gather the data 62 | mc := newOpenStackMetric("firewallv2_group", "list") 63 | pages, err := groups.List(client, groups.ListOpts{}).AllPages() 64 | if mc.Observe(err) != nil { 65 | // only warn, maybe the next list will work. 66 | klog.Warningf("Unable to list firewall groups: %v", err) 67 | return err 68 | } 69 | groupsList, err := groups.ExtractGroups(pages) 70 | if err != nil { 71 | // only warn, maybe the next publish will work. 72 | klog.Warningf("Unable to extract firewall groups: %v", err) 73 | return err 74 | } 75 | 76 | // second step: reset the old metrics 77 | resetFirewallV2Metrics() 78 | 79 | // third step: publish the metrics 80 | for _, group := range groupsList { 81 | if group.Name == "default" { 82 | continue 83 | } 84 | publishFirewallV2GroupMetric(group) 85 | } 86 | 87 | return nil 88 | } 89 | 90 | // resetFirewallV2Metrics resets the firewall v2 metrics 91 | func resetFirewallV2Metrics() { 92 | firewallV2GroupAdminStateUp.Reset() 93 | firewallV2GroupStatus.Reset() 94 | } 95 | 96 | // publishFirewallV2GroupMetric extracts data from a firewall and exposes the metrics via prometheus 97 | func publishFirewallV2GroupMetric(group groups.Group) { 98 | labels := []string{group.ID, group.Name, group.Description, group.IngressFirewallPolicyID, group.EgressFirewallPolicyID, group.ProjectID} 99 | firewallV2GroupAdminStateUp.WithLabelValues(labels...).Set(boolFloat64(group.AdminStateUp)) 100 | 101 | // create one metric per status 102 | for _, state := range firewallV2States { 103 | stateLabels := append(labels, state) 104 | firewallV2GroupStatus.WithLabelValues(stateLabels...).Set(boolFloat64(group.Status == state)) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # This project is no longer maintained and is archived. 3 | 4 | # Kosmoo 5 | 6 | [![Build Status](https://github.com/mercedes-benz/kosmoo/actions/workflows/ci.yml/badge.svg)](https://github.com/mercedes-benz/kosmoo/actions?query=.github%2Fworkflows%2Fci.yml) 7 | [![Release Status](https://github.com/mercedes-benz/kosmoo/workflows/release/badge.svg)](https://github.com/mercedes-benz/kosmoo/actions?query=workflow%3Arelease) 8 | 9 | *Kosmoo* exposes metrics about: 10 | * [Persistent Volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/) by queries to the Kubernetes API 11 | * [Cinder](https://docs.openstack.org/cinder/latest/) and its Disks by queries to the OpenStack API combined with data from the Kubernetes API 12 | * [Neutron Floating IPs](https://docs.openstack.org/api-ref/network/v2/index.html#floating-ips-floatingips) 13 | * [Load balancers](https://docs.openstack.org/api-ref/load-balancer/) 14 | 15 | ## Installation 16 | 17 | ### Building from source 18 | 19 | To build the exporter from the source code yourself you need to have a working Go environment with [version 1.12 or greater installed](https://golang.org/doc/install). 20 | 21 | ``` 22 | $ mkdir -p $GOPATH/src/github.com/mercedes-benz 23 | $ cd $GOPATH/src/github.com/mercedes-benz 24 | $ git clone https://github.com/mercedes-benz/kosmoo.git 25 | $ cd kosmoo 26 | $ make build 27 | ``` 28 | 29 | The Makefile provides several targets: 30 | 31 | * *build:* build the `kosmoo` binary 32 | * *docker:* build a docker container for the current `HEAD` 33 | * *fmt:* format the source code 34 | * *test:* runs the `vet`, `lint` and `fmtcheck` targets 35 | * *vet:* check the source code for common errors 36 | * *lint:* does source code linting 37 | * *fmtcheck:* check the source code for format findings 38 | * *version:* prints the version tag 39 | 40 | ## Usage 41 | 42 | ``` 43 | $ ./kosmoo -h 44 | Usage of ./kosmoo: 45 | -addr string 46 | Address to listen on (default ":9183") 47 | -alsologtostderr 48 | log to standard error as well as files 49 | -cloud-conf string 50 | path to the cloud.conf file. If this path is not set the scraper will use the usual OpenStack environment variables. 51 | -kubeconfig string 52 | Path to the kubeconfig file to use for CLI requests. (uses in-cluster config if empty) 53 | -log_backtrace_at value 54 | when logging hits line file:N, emit a stack trace 55 | -log_dir string 56 | If non-empty, write log files in this directory 57 | -log_file string 58 | If non-empty, use this log file 59 | -log_file_max_size uint 60 | Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) 61 | -logtostderr 62 | log to standard error instead of files (default true) 63 | -refresh-interval int 64 | Interval between scrapes to OpenStack API (default 120s) (default 120) 65 | -skip_headers 66 | If true, avoid header prefixes in the log messages 67 | -skip_log_headers 68 | If true, avoid headers when openning log files 69 | -stderrthreshold value 70 | logs at or above this threshold go to stderr (default 2) 71 | -v value 72 | number for the log level verbosity 73 | -vmodule value 74 | comma-separated list of pattern=N settings for file-filtered logging 75 | ``` 76 | 77 | ## Deployment to Kubernetes 78 | 79 | *kosmoo* can get deployed as a deployment. See the [instructions](kubernetes/) how to get started. 80 | You can also use the docker images under [packages](https://github.com/mercedes-benz/kosmoo/packages), 81 | see also [authenticating-to-github-package-registry](https://help.github.com/en/articles/configuring-docker-for-use-with-github-package-registry#authenticating-to-github-package-registry). 82 | 83 | 84 | 85 | ## Metrics 86 | 87 | Metrics will be made available on port 9183 by default, or you can pass the commandline flag `-addr` to override the port. 88 | An overview and example output of the metrics can be found in [metrics.md](docs/metrics.md). 89 | 90 | ## Alert Rules 91 | 92 | In combination with [Prometheus](https://prometheus.io/) it is possible to create alerts from the metrics exposed by the `kosmoo`. 93 | An example for some alerts can be found in [alerts.md](docs/alerts.md). 94 | 95 | ## Contributing 96 | 97 | We welcome any contributions. 98 | If you want to contribute to this project, please read the [contributing guide](CONTRIBUTING.md). 99 | 100 | ## License 101 | 102 | Full information on the license for this software is available in the [LICENSE](LICENSE) file. 103 | 104 | # Provider Information 105 | 106 | Please visit https://www.mercedes-benz-techinnovation.com/en/imprint/ for information on the provider. 107 | 108 | Notice: Before you use the program in productive use, please take all necessary precautions, e.g. testing and verifying the program with regard to your specific use. The program was tested solely for our own use cases, which might differ from yours. 109 | -------------------------------------------------------------------------------- /pkg/metrics/nova.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "github.com/gophercloud/gophercloud" 7 | "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/quotasets" 8 | "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | var ( 14 | computeQuotaCores *prometheus.GaugeVec 15 | computeQuotaFloatingIPs *prometheus.GaugeVec 16 | computeQuotaInstances *prometheus.GaugeVec 17 | computeQuotaRAM *prometheus.GaugeVec 18 | serverStatus *prometheus.GaugeVec 19 | serverVolumeAttachment *prometheus.GaugeVec 20 | serverVolumeAttachmentCount *prometheus.GaugeVec 21 | 22 | // possible server states, from https://github.com/openstack/nova/blob/master/nova/objects/fields.py#L949 23 | states = []string{"ACTIVE", "BUILDING", "PAUSED", "SUSPENDED", "STOPPED", "RESCUED", "RESIZED", "SOFT_DELETED", "DELETED", "ERROR", "SHELVED", "SHELVED_OFFLOADED"} 24 | 25 | serverLabels = []string{"id", "name"} 26 | ) 27 | 28 | func registerServerMetrics() { 29 | computeQuotaCores = prometheus.NewGaugeVec( 30 | prometheus.GaugeOpts{ 31 | Name: generateName("compute_quota_cores"), 32 | Help: "Number of instance cores allowed", 33 | }, 34 | []string{"quota_type"}, 35 | ) 36 | computeQuotaFloatingIPs = prometheus.NewGaugeVec( 37 | prometheus.GaugeOpts{ 38 | Name: generateName("compute_quota_floating_ips"), 39 | Help: "Number of floating IPs allowed", 40 | }, 41 | []string{"quota_type"}, 42 | ) 43 | computeQuotaInstances = prometheus.NewGaugeVec( 44 | prometheus.GaugeOpts{ 45 | Name: generateName("compute_quota_instances"), 46 | Help: "Number of instances (servers) allowed", 47 | }, 48 | []string{"quota_type"}, 49 | ) 50 | computeQuotaRAM = prometheus.NewGaugeVec( 51 | prometheus.GaugeOpts{ 52 | Name: generateName("compute_quota_ram_megabytes"), 53 | Help: "RAM (in MB) allowed", 54 | }, 55 | []string{"quota_type"}, 56 | ) 57 | serverStatus = prometheus.NewGaugeVec( 58 | prometheus.GaugeOpts{ 59 | Name: generateName("server_status"), 60 | Help: "Server status", 61 | }, 62 | append(serverLabels, "status"), 63 | ) 64 | serverVolumeAttachmentCount = prometheus.NewGaugeVec( 65 | prometheus.GaugeOpts{ 66 | Name: generateName("server_volume_attachment_count"), 67 | Help: "Server volume attachment count", 68 | }, 69 | serverLabels, 70 | ) 71 | serverVolumeAttachment = prometheus.NewGaugeVec( 72 | prometheus.GaugeOpts{ 73 | Name: generateName("server_volume_attachment"), 74 | Help: "Server volume attachment", 75 | }, 76 | append(serverLabels, "volume_id"), 77 | ) 78 | 79 | prometheus.MustRegister(computeQuotaCores) 80 | prometheus.MustRegister(computeQuotaFloatingIPs) 81 | prometheus.MustRegister(computeQuotaInstances) 82 | prometheus.MustRegister(computeQuotaRAM) 83 | prometheus.MustRegister(serverStatus) 84 | prometheus.MustRegister(serverVolumeAttachmentCount) 85 | prometheus.MustRegister(serverVolumeAttachment) 86 | } 87 | 88 | // PublishServerMetrics makes the list request to the server api and 89 | // passes the result to a publish function. 90 | func PublishServerMetrics(client *gophercloud.ServiceClient, tenantID string) error { 91 | // first step: gather the data 92 | mc := newOpenStackMetric("server", "list") 93 | pages, err := servers.List(client, servers.ListOpts{}).AllPages() 94 | if mc.Observe(err) != nil { 95 | // only warn, maybe the next list will work. 96 | klog.Warningf("Unable to list servers: %v", err) 97 | return err 98 | } 99 | serversList, err := servers.ExtractServers(pages) 100 | if err != nil { 101 | // only warn, maybe the next publish will work. 102 | klog.Warningf("Unable to extract servers: %v", err) 103 | return err 104 | } 105 | 106 | // second step: reset the old metrics 107 | serverStatus.Reset() 108 | serverVolumeAttachmentCount.Reset() 109 | serverVolumeAttachment.Reset() 110 | 111 | // third step: publish the metrics 112 | for _, srv := range serversList { 113 | publishServerMetric(srv) 114 | } 115 | 116 | // Get compute quotas from OpenStack. 117 | mc = newOpenStackMetric("compute_quotasets_detail", "get") 118 | quotas, err := quotasets.GetDetail(client, tenantID).Extract() 119 | if mc.Observe(err) != nil { 120 | // only warn, maybe the next get will work. 121 | klog.Warningf("Unable to get compute quotas: %v", err) 122 | return err 123 | } 124 | publishComputeQuotaMetrics(quotas) 125 | 126 | return nil 127 | } 128 | 129 | // publishServerMetric extracts data from a server and exposes the metrics via prometheus 130 | func publishServerMetric(srv servers.Server) { 131 | labels := []string{srv.ID, srv.Name} 132 | 133 | serverVolumeAttachmentCount.WithLabelValues(labels...).Set(float64(len(srv.AttachedVolumes))) 134 | for _, attachedVolumeID := range srv.AttachedVolumes { 135 | serverVolumeAttachment.WithLabelValues(append(labels, attachedVolumeID.ID)...).Set(1) 136 | } 137 | 138 | // create one metric per status 139 | for _, state := range states { 140 | stateLabels := append(labels, state) 141 | serverStatus.WithLabelValues(stateLabels...).Set(boolFloat64(srv.Status == state)) 142 | } 143 | } 144 | 145 | // publishComputeQuotaMetrics publishes all compute related quotas 146 | func publishComputeQuotaMetrics(q quotasets.QuotaDetailSet) { 147 | computeQuotaCores.WithLabelValues("in-use").Set(float64(q.Cores.InUse)) 148 | computeQuotaCores.WithLabelValues("reserved").Set(float64(q.Cores.Reserved)) 149 | computeQuotaCores.WithLabelValues("limit").Set(float64(q.Cores.Limit)) 150 | 151 | computeQuotaFloatingIPs.WithLabelValues("in-use").Set(float64(q.FloatingIPs.InUse)) 152 | computeQuotaFloatingIPs.WithLabelValues("reserved").Set(float64(q.FloatingIPs.Reserved)) 153 | computeQuotaFloatingIPs.WithLabelValues("limit").Set(float64(q.FloatingIPs.Limit)) 154 | 155 | computeQuotaInstances.WithLabelValues("in-use").Set(float64(q.Instances.InUse)) 156 | computeQuotaInstances.WithLabelValues("reserved").Set(float64(q.Instances.Reserved)) 157 | computeQuotaInstances.WithLabelValues("limit").Set(float64(q.Instances.Limit)) 158 | 159 | computeQuotaRAM.WithLabelValues("in-use").Set(float64(q.RAM.InUse)) 160 | computeQuotaRAM.WithLabelValues("reserved").Set(float64(q.RAM.Reserved)) 161 | computeQuotaRAM.WithLabelValues("limit").Set(float64(q.RAM.Limit)) 162 | } 163 | -------------------------------------------------------------------------------- /pkg/metrics/loadbalancer.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "github.com/gophercloud/gophercloud" 7 | "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers" 8 | "github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/pools" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | var ( 14 | loadbalancerAdminStateUp *prometheus.GaugeVec 15 | loadbalancerStatus *prometheus.GaugeVec 16 | loadbalancerPoolProvisioningStatus *prometheus.GaugeVec 17 | loadbalancerPoolMemberProvisioningStatus *prometheus.GaugeVec 18 | 19 | // possible load balancer provisioning states, from https://github.com/openstack/octavia-lib/blob/fe022cdf14604206af783c8a0887c008c48fd053/octavia_lib/common/constants.py#L169 20 | provisioningStates = []string{"ALLOCATED", "BOOTING", "READY", "ACTIVE", "PENDING_DELETE", "PENDING_UPDATE", "PENDING_CREATE", "DELETED", "ERROR"} 21 | 22 | // https://github.com/gophercloud/gophercloud/blob/0ffab06fc18e06ed9544fe10b385f0a59492fdb5/openstack/loadbalancer/v2/pools/results.go#L100 23 | // Possible states ACTIVE, PENDING_* or ERROR. 24 | poolProvisioningStates = []string{"ACTIVE", "PENDING_DELETE", "PENDING_CREATE", "PENDING_UPDATE", "ERROR"} 25 | 26 | loadBalancerLabels = []string{"id", "name", "vip_address", "provider", "port_id"} 27 | poolLabels = []string{"pool_id", "pool_name"} 28 | poolMemberLabels = []string{"member_id", "member_name"} 29 | ) 30 | 31 | func registerLoadBalancerMetrics() { 32 | loadbalancerAdminStateUp = prometheus.NewGaugeVec( 33 | prometheus.GaugeOpts{ 34 | Name: generateName("loadbalancer_admin_state_up"), 35 | Help: "Load balancer admin state up", 36 | }, 37 | loadBalancerLabels, 38 | ) 39 | loadbalancerStatus = prometheus.NewGaugeVec( 40 | prometheus.GaugeOpts{ 41 | Name: generateName("loadbalancer_provisioning_status"), 42 | Help: "Load balancer status", 43 | }, 44 | append(loadBalancerLabels, "provisioning_status"), 45 | ) 46 | 47 | loadbalancerPoolProvisioningStatus = prometheus.NewGaugeVec( 48 | prometheus.GaugeOpts{ 49 | Name: generateName("loadbalancer_pool_provisioning_status"), 50 | Help: "Load balancer pool provisioning status", 51 | }, 52 | append(append(loadBalancerLabels, poolLabels...), "pool_provisioning_status"), 53 | ) 54 | 55 | loadbalancerPoolMemberProvisioningStatus = prometheus.NewGaugeVec( 56 | prometheus.GaugeOpts{ 57 | Name: generateName("loadbalancer_pool_member_provisioning_status"), 58 | Help: "Load balancer pool member provisioning status", 59 | }, 60 | append(append(append(loadBalancerLabels, poolLabels...), poolMemberLabels...), "pool_member_provisioning_status"), 61 | ) 62 | 63 | prometheus.MustRegister(loadbalancerAdminStateUp) 64 | prometheus.MustRegister(loadbalancerStatus) 65 | prometheus.MustRegister(loadbalancerPoolProvisioningStatus) 66 | prometheus.MustRegister(loadbalancerPoolMemberProvisioningStatus) 67 | } 68 | 69 | // PublishLoadBalancerMetrics makes the list request to the load balancer api and 70 | // passes the result to a publish function. 71 | func PublishLoadBalancerMetrics(client *gophercloud.ServiceClient, tenantID string) error { 72 | // first step: gather the data 73 | mc := newOpenStackMetric("loadbalancer", "list") 74 | pages, err := loadbalancers.List(client, loadbalancers.ListOpts{}).AllPages() 75 | if mc.Observe(err) != nil { 76 | // only warn, maybe the next list will work. 77 | klog.Warningf("Unable to list load balancers: %v", err) 78 | return err 79 | } 80 | loadBalancerList, err := loadbalancers.ExtractLoadBalancers(pages) 81 | if err != nil { 82 | // only warn, maybe the next publish will work. 83 | klog.Warningf("Unable to extract load balancers: %v", err) 84 | return err 85 | } 86 | if len(loadBalancerList) == 0 { 87 | klog.Info("No load balancers found. Skipping load balancer metrics.") 88 | } 89 | 90 | // second step: reset the old metrics 91 | loadbalancerAdminStateUp.Reset() 92 | loadbalancerStatus.Reset() 93 | loadbalancerPoolProvisioningStatus.Reset() 94 | loadbalancerPoolMemberProvisioningStatus.Reset() 95 | 96 | // third step: publish the metrics 97 | for _, lb := range loadBalancerList { 98 | publishLoadBalancerMetric(lb) 99 | 100 | // for the pools associated with the loadbalancer 101 | for _, poolWithOnlyId := range lb.Pools { 102 | pool, err := pools.Get(client, poolWithOnlyId.ID).Extract() 103 | if err != nil { 104 | klog.Warningf("Unable to get pool %s: %v", poolWithOnlyId.ID, err) 105 | continue 106 | } 107 | publishPoolStatus(lb, *pool) 108 | 109 | // for the pool members 110 | for _, memberWithOnlyId := range pool.Members { 111 | member, err := pools.GetMember(client, pool.ID, memberWithOnlyId.ID).Extract() 112 | if err != nil { 113 | klog.Warningf("Unable to get member %s of pool %s: %v", memberWithOnlyId.ID, poolWithOnlyId.ID, err) 114 | continue 115 | } 116 | publishMemberStatus(lb, *pool, *member) 117 | } 118 | } 119 | } 120 | 121 | return nil 122 | } 123 | 124 | // publishLoadBalancerMetric extracts data from a load balancer and exposes the metrics via prometheus 125 | func publishLoadBalancerMetric(lb loadbalancers.LoadBalancer) { 126 | labels := []string{lb.ID, lb.Name, lb.VipAddress, lb.Provider, lb.VipPortID} 127 | 128 | loadbalancerAdminStateUp.WithLabelValues(labels...).Set(boolFloat64(lb.AdminStateUp)) 129 | 130 | // create one metric per provisioning status 131 | for _, state := range provisioningStates { 132 | stateLabels := append(labels, state) 133 | loadbalancerStatus.WithLabelValues(stateLabels...).Set(boolFloat64(lb.ProvisioningStatus == state)) 134 | } 135 | } 136 | 137 | func publishPoolStatus(lb loadbalancers.LoadBalancer, pool pools.Pool) { 138 | labels := []string{lb.ID, lb.Name, lb.VipAddress, lb.Provider, lb.VipPortID, pool.ID, pool.Name} 139 | 140 | for _, state := range poolProvisioningStates { 141 | stateLabels := append(labels, state) 142 | loadbalancerPoolProvisioningStatus.WithLabelValues(stateLabels...).Set(boolFloat64(pool.ProvisioningStatus == state)) 143 | } 144 | } 145 | 146 | func publishMemberStatus(lb loadbalancers.LoadBalancer, pool pools.Pool, member pools.Member) { 147 | labels := []string{lb.ID, lb.Name, lb.VipAddress, lb.Provider, lb.VipPortID, pool.ID, pool.Name, member.ID, member.Name} 148 | 149 | for _, state := range poolProvisioningStates { 150 | stateLabels := append(labels, state) 151 | loadbalancerPoolMemberProvisioningStatus.WithLabelValues(stateLabels...).Set(boolFloat64(member.ProvisioningStatus == state)) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /pkg/metrics/cinder.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package metrics 4 | 5 | import ( 6 | "github.com/gophercloud/gophercloud" 7 | "github.com/gophercloud/gophercloud/openstack/blockstorage/extensions/quotasets" 8 | "github.com/gophercloud/gophercloud/openstack/blockstorage/v2/volumes" 9 | "github.com/prometheus/client_golang/prometheus" 10 | corev1 "k8s.io/api/core/v1" 11 | "k8s.io/client-go/kubernetes" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | var ( 16 | // labels which get applied to the most metrics 17 | defaultLabels = []string{"id", "description", "name", "status", "cinder_availability_zone", "volume_type", "pvc_name", "pvc_namespace", "pv_name", "pv_storage_class", "pv_reclaim_policy", "pv_fs_type"} 18 | 19 | // possible cinder states, from https://github.com/openstack/cinder/blob/master/cinder/objects/fields.py#L168 20 | cinderStates = []string{"creating", "available", "deleting", "error", "error_deleting", "error_managing", "managing", "attaching", "in-use", "detaching", "maintenance", "restoring-backup", "error_restoring", "reserved", "awaiting-transfer", "backing-up", "error_backing-up", "error_extending", "downloading", "uploading", "retyping", "extending"} 21 | 22 | cinderQuotaVolumes *prometheus.GaugeVec 23 | cinderQuotaVolumesGigabyte *prometheus.GaugeVec 24 | cinderVolumeCreated *prometheus.GaugeVec 25 | cinderVolumeUpdatedAt *prometheus.GaugeVec 26 | cinderVolumeStatus *prometheus.GaugeVec 27 | cinderVolumeSize *prometheus.GaugeVec 28 | cinderVolumeAttachedAt *prometheus.GaugeVec 29 | ) 30 | 31 | func registerCinderMetrics() { 32 | cinderQuotaVolumes = prometheus.NewGaugeVec( 33 | prometheus.GaugeOpts{ 34 | Name: generateName("cinder_quota_volume_disks"), 35 | Help: "Cinder volume metric (number of volumes)", 36 | }, 37 | []string{"quota_type"}, 38 | ) 39 | cinderQuotaVolumesGigabyte = prometheus.NewGaugeVec( 40 | prometheus.GaugeOpts{ 41 | Name: generateName("cinder_quota_volume_disk_gigabytes"), 42 | Help: "Cinder volume metric (GB)", 43 | }, 44 | []string{"quota_type"}, 45 | ) 46 | 47 | cinderVolumeCreated = prometheus.NewGaugeVec( 48 | prometheus.GaugeOpts{ 49 | Name: generateName("cinder_volume_created_at"), 50 | Help: "Cinder volume created at", 51 | }, 52 | defaultLabels, 53 | ) 54 | cinderVolumeUpdatedAt = prometheus.NewGaugeVec( 55 | prometheus.GaugeOpts{ 56 | Name: generateName("cinder_volume_updated_at"), 57 | Help: "Cinder volume updated at", 58 | }, 59 | defaultLabels, 60 | ) 61 | cinderVolumeStatus = prometheus.NewGaugeVec( 62 | prometheus.GaugeOpts{ 63 | Name: generateName("cinder_volume_status"), 64 | Help: "Cinder volume status", 65 | }, 66 | defaultLabels, 67 | ) 68 | cinderVolumeSize = prometheus.NewGaugeVec( 69 | prometheus.GaugeOpts{ 70 | Name: generateName("cinder_volume_size"), 71 | Help: "Cinder volume size", 72 | }, 73 | defaultLabels, 74 | ) 75 | cinderVolumeAttachedAt = prometheus.NewGaugeVec( 76 | prometheus.GaugeOpts{ 77 | Name: generateName("cinder_volume_attached_at"), 78 | Help: "Cinder volume attached at", 79 | }, 80 | append(defaultLabels, "server_id", "device", "hostname"), 81 | ) 82 | 83 | prometheus.MustRegister(cinderQuotaVolumes) 84 | prometheus.MustRegister(cinderQuotaVolumesGigabyte) 85 | prometheus.MustRegister(cinderVolumeCreated) 86 | prometheus.MustRegister(cinderVolumeUpdatedAt) 87 | prometheus.MustRegister(cinderVolumeSize) 88 | prometheus.MustRegister(cinderVolumeStatus) 89 | prometheus.MustRegister(cinderVolumeAttachedAt) 90 | } 91 | 92 | // PublishCinderMetrics makes the list request to the blockstorage api and passes 93 | // the result to a publish function. 94 | func PublishCinderMetrics(client *gophercloud.ServiceClient, clientset *kubernetes.Clientset, tenantID string) error { 95 | // first step: gather the data 96 | 97 | // get the cinder pvs to add metadata 98 | pvs, err := getPVsByCinderID(clientset) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | // get all volumes from openstack 104 | mc := newOpenStackMetric("volume", "list") 105 | pages, err := volumes.List(client, volumes.ListOpts{}).AllPages() 106 | if mc.Observe(err) != nil { 107 | // only warn, maybe the next list will work. 108 | klog.Warningf("Unable to list volumes: %v", err) 109 | return err 110 | } 111 | volumesList, err := volumes.ExtractVolumes(pages) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | // get quotas form openstack 117 | mc = newOpenStackMetric("volume_quotasets_usage", "get") 118 | quotas, err := quotasets.GetUsage(client, tenantID).Extract() 119 | if mc.Observe(err) != nil { 120 | // only warn, maybe the next get will work. 121 | klog.Warningf("Unable to get volume quotas: %v", err) 122 | return err 123 | } 124 | 125 | // second step: reset the old metrics 126 | // cinderQuotaVolumes and CinderQuotaVolumesGigabytes are not dynamic and do not need to be reset 127 | cinderVolumeCreated.Reset() 128 | cinderVolumeUpdatedAt.Reset() 129 | cinderVolumeStatus.Reset() 130 | cinderVolumeSize.Reset() 131 | cinderVolumeAttachedAt.Reset() 132 | 133 | // third step: publish the metrics 134 | publishVolumes(volumesList, pvs) 135 | 136 | publishCinderQuotas(quotas) 137 | return nil 138 | } 139 | 140 | // publishCinderQuotas publishes all cinder related quotas 141 | func publishCinderQuotas(q quotasets.QuotaUsageSet) { 142 | cinderQuotaVolumes.WithLabelValues("in-use").Set(float64(q.Volumes.InUse)) 143 | cinderQuotaVolumes.WithLabelValues("reserved").Set(float64(q.Volumes.Reserved)) 144 | cinderQuotaVolumes.WithLabelValues("limit").Set(float64(q.Volumes.Limit)) 145 | cinderQuotaVolumes.WithLabelValues("allocated").Set(float64(q.Volumes.Allocated)) 146 | 147 | cinderQuotaVolumesGigabyte.WithLabelValues("in-use").Set(float64(q.Gigabytes.InUse)) 148 | cinderQuotaVolumesGigabyte.WithLabelValues("reserved").Set(float64(q.Gigabytes.Reserved)) 149 | cinderQuotaVolumesGigabyte.WithLabelValues("limit").Set(float64(q.Gigabytes.Limit)) 150 | cinderQuotaVolumesGigabyte.WithLabelValues("allocated").Set(float64(q.Gigabytes.Allocated)) 151 | } 152 | 153 | // publishVolumes iterates over a page, the result of a list request 154 | func publishVolumes(vList []volumes.Volume, pvs map[string]corev1.PersistentVolume) { 155 | for _, v := range vList { 156 | if pv, ok := pvs[v.ID]; ok { 157 | publishVolumeMetrics(v, &pv) 158 | } else { 159 | publishVolumeMetrics(v, nil) 160 | } 161 | } 162 | } 163 | 164 | // publishVolumeMetrics extracts data from a volume and exposes the metrics via prometheus 165 | func publishVolumeMetrics(v volumes.Volume, pv *corev1.PersistentVolume) { 166 | labels := []string{v.ID, v.Description, v.Name, v.Status, v.AvailabilityZone, v.VolumeType} 167 | 168 | k8sMetadata := extractK8sMetadata(pv) 169 | 170 | // add the k8s specific labels. 171 | // We do not need to check them, because the empty string is totally fine. 172 | labels = append(labels, k8sMetadata...) 173 | 174 | // set the volume-specific metrics 175 | cinderVolumeCreated.WithLabelValues(labels...).Set(float64(v.CreatedAt.Unix())) 176 | cinderVolumeUpdatedAt.WithLabelValues(labels...).Set(float64(v.UpdatedAt.Unix())) 177 | cinderVolumeSize.WithLabelValues(labels...).Set(float64(v.Size)) 178 | 179 | if len(v.Attachments) == 0 { 180 | l := append(labels, "", "", "") 181 | cinderVolumeAttachedAt.WithLabelValues(l...).Set(float64(0)) 182 | } else { 183 | // set the volume-attachment-specific labels 184 | for _, a := range v.Attachments { 185 | // TODO: hostname 186 | l := append(labels, a.ServerID, a.Device, "") 187 | cinderVolumeAttachedAt.WithLabelValues(l...).Set(float64(a.AttachedAt.Unix())) 188 | } 189 | } 190 | 191 | // create one metric per state. If it's the current state it's 1 192 | for _, status := range cinderStates { 193 | labels := []string{v.ID, v.Description, v.Name, status, v.AvailabilityZone, v.VolumeType} 194 | labels = append(labels, k8sMetadata...) 195 | cinderVolumeStatus.WithLabelValues(labels...).Set(boolFloat64(v.Status == status)) 196 | } 197 | } 198 | 199 | func boolFloat64(b bool) float64 { 200 | if b { 201 | return 1 202 | } 203 | return 0 204 | } 205 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: MIT 2 | 3 | package main 4 | 5 | import ( 6 | "flag" 7 | "fmt" 8 | "net/http" 9 | "os" 10 | "sync" 11 | "time" 12 | 13 | "github.com/gophercloud/gophercloud" 14 | "github.com/gophercloud/gophercloud/openstack" 15 | "github.com/prometheus/client_golang/prometheus" 16 | "github.com/prometheus/client_golang/prometheus/promhttp" 17 | "gopkg.in/ini.v1" 18 | "k8s.io/client-go/kubernetes" 19 | "k8s.io/client-go/rest" 20 | "k8s.io/client-go/tools/clientcmd" 21 | "k8s.io/klog/v2" 22 | 23 | "github.com/mercedes-benz/kosmoo/pkg/metrics" 24 | ) 25 | 26 | var ( 27 | refreshInterval = flag.Int64("refresh-interval", 120, "Interval between scrapes to OpenStack API (default 120s)") 28 | addr = flag.String("addr", ":9183", "Address to listen on") 29 | cloudConfFile = flag.String("cloud-conf", "", "path to the cloud.conf file. If this path is not set the scraper will use the usual OpenStack environment variables.") 30 | kubeconfig = flag.String("kubeconfig", os.Getenv("KUBECONFIG"), "Path to the kubeconfig file to use for CLI requests. (uses in-cluster config if empty)") 31 | metricsPrefix = flag.String("metrics-prefix", metrics.DefaultMetricsPrefix, "Prefix used for all metrics") 32 | ) 33 | 34 | var ( 35 | scrapeDuration *prometheus.GaugeVec 36 | scrapedAt *prometheus.GaugeVec 37 | scrapedStatus *prometheus.GaugeVec 38 | ) 39 | 40 | var ( 41 | clientset *kubernetes.Clientset 42 | backoffSleep = time.Second 43 | maxBackoffSleep = time.Hour 44 | ) 45 | 46 | func registerMetrics(prefix string) { 47 | scrapeDuration = prometheus.NewGaugeVec( 48 | prometheus.GaugeOpts{ 49 | Name: metrics.AddPrefix("scrape_duration", prefix), 50 | Help: "Time in seconds needed for the last scrape", 51 | }, 52 | []string{"refresh_interval"}, 53 | ) 54 | scrapedAt = prometheus.NewGaugeVec( 55 | prometheus.GaugeOpts{ 56 | Name: metrics.AddPrefix("scraped_at", prefix), 57 | Help: "Timestamp when last scrape started", 58 | }, 59 | []string{"refresh_interval"}, 60 | ) 61 | scrapedStatus = prometheus.NewGaugeVec( 62 | prometheus.GaugeOpts{ 63 | Name: metrics.AddPrefix("scrape_status_succeeded", prefix), 64 | Help: "Scrape status succeeded", 65 | }, 66 | []string{"refresh_interval"}, 67 | ) 68 | 69 | prometheus.MustRegister(scrapeDuration) 70 | prometheus.MustRegister(scrapedAt) 71 | prometheus.MustRegister(scrapedStatus) 72 | } 73 | 74 | // metrixMutex locks to prevent race-conditions between scraping the metrics 75 | // endpoint and updating the metrics 76 | var metricsMutex sync.Mutex 77 | 78 | func main() { 79 | klog.InitFlags(nil) 80 | flag.Parse() 81 | 82 | klog.Infof("starting kosmoo at %s", *addr) 83 | 84 | registerMetrics(*metricsPrefix) 85 | metrics.RegisterMetrics(*metricsPrefix) 86 | 87 | // start prometheus metrics endpoint 88 | go func() { 89 | // klog.Info().Str("addr", *addr).Msg("starting prometheus http endpoint") 90 | klog.Infof("starting prometheus http endpoint at %s", *addr) 91 | metricsMux := http.NewServeMux() 92 | promHandler := promhttp.Handler() 93 | metricsMux.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { 94 | // metricsMutex ensures that we do not drop the metrics during promHandler.ServeHTTP call 95 | metricsMutex.Lock() 96 | defer metricsMutex.Unlock() 97 | promHandler.ServeHTTP(w, r) 98 | }) 99 | 100 | metricsMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { 101 | w.WriteHeader(http.StatusOK) 102 | if _, err := w.Write([]byte(http.StatusText(http.StatusOK))); err != nil { 103 | klog.Warningf("error handling /healthz: %v", err) 104 | } 105 | }) 106 | 107 | err := http.ListenAndServe(*addr, metricsMux) 108 | klog.Fatalf("prometheus http.ListenAndServe failed: %v", err) 109 | }() 110 | 111 | for { 112 | if run() != nil { 113 | klog.Errorf("error during run - sleeping %s", backoffSleep) 114 | time.Sleep(backoffSleep) 115 | backoffSleep = min(2*backoffSleep, maxBackoffSleep) 116 | } 117 | } 118 | } 119 | 120 | // run does the initialization of the operational exporter and also the metrics scraping 121 | func run() error { 122 | var authOpts gophercloud.AuthOptions 123 | var endpointOpts gophercloud.EndpointOpts 124 | var err error 125 | 126 | // get OpenStack credentials 127 | if *cloudConfFile != "" { 128 | authOpts, endpointOpts, err = authOptsFromCloudConf(*cloudConfFile) 129 | if err != nil { 130 | return logError("unable to read OpenStack credentials from cloud.conf: %v", err) 131 | } 132 | klog.Infof("OpenStack credentials read from cloud.conf file at %s", *cloudConfFile) 133 | } else { 134 | authOpts, err = openstack.AuthOptionsFromEnv() 135 | if err != nil { 136 | return logError("unable to get authentication credentials from environment: %v", err) 137 | } 138 | endpointOpts = gophercloud.EndpointOpts{ 139 | Region: os.Getenv("OS_REGION_NAME"), 140 | } 141 | if endpointOpts.Region == "" { 142 | endpointOpts.Region = "nova" 143 | } 144 | klog.Info("OpenStack credentials read from environment") 145 | } 146 | 147 | // get kubernetes clientset 148 | var config *rest.Config 149 | config, err = clientcmd.BuildConfigFromFlags("", *kubeconfig) 150 | if err != nil { 151 | return logError("unable to get kubernetes config: %v", err) 152 | } 153 | clientset, err = kubernetes.NewForConfig(config) 154 | if err != nil { 155 | return logError("error creating kubernetes Clientset: %v", err) 156 | } 157 | 158 | // authenticate to OpenStack 159 | provider, err := openstack.AuthenticatedClient(authOpts) 160 | if err != nil { 161 | return logError("unable to authenticate to OpenStack: %v", err) 162 | } 163 | _ = provider 164 | 165 | klog.Infof("OpenStack authentication was successful: username=%s, tenant-id=%s, tenant-name=%s", authOpts.Username, authOpts.TenantID, authOpts.TenantName) 166 | 167 | // start scraping loop 168 | for { 169 | err := updateMetrics(provider, endpointOpts, clientset, authOpts.TenantID) 170 | if err != nil { 171 | return err 172 | } 173 | time.Sleep(time.Second * time.Duration(*refreshInterval)) 174 | } 175 | } 176 | 177 | // original struct is in 178 | // "github.com/kubernetes/kubernetes/pkg/cloudprovider/providers/openstack" 179 | // and read by 180 | // "gopkg.in/gcfg.v1" 181 | // maybe use that to be type-safe 182 | 183 | // For Reference, the layout of the [Global] section in cloud.conf 184 | // Global struct { 185 | // AuthURL string `gcfg:"auth-url"` 186 | // Username string 187 | // UserID string `gcfg:"user-id"` 188 | // Password string 189 | // TenantID string `gcfg:"tenant-id"` 190 | // TenantName string `gcfg:"tenant-name"` 191 | // TrustID string `gcfg:"trust-id"` 192 | // DomainID string `gcfg:"domain-id"` 193 | // DomainName string `gcfg:"domain-name"` 194 | // Region string 195 | // CAFile string `gcfg:"ca-file"` 196 | // } 197 | 198 | // authOptsFromCloudConf reads the cloud.conf from `path` and returns the read AuthOptions 199 | func authOptsFromCloudConf(path string) (gophercloud.AuthOptions, gophercloud.EndpointOpts, error) { 200 | cfg, err := ini.Load(path) 201 | if err != nil { 202 | return gophercloud.AuthOptions{}, gophercloud.EndpointOpts{}, fmt.Errorf("unable to read cloud.conf content: %v", err) 203 | } 204 | 205 | global, err := cfg.GetSection("Global") 206 | if err != nil { 207 | return gophercloud.AuthOptions{}, gophercloud.EndpointOpts{}, fmt.Errorf("unable get Global section: %v", err) 208 | } 209 | 210 | ao := gophercloud.AuthOptions{ 211 | IdentityEndpoint: global.Key("auth-url").String(), 212 | Username: global.Key("username").String(), 213 | UserID: global.Key("user-id").String(), 214 | Password: global.Key("password").String(), 215 | DomainID: global.Key("domain-id").String(), 216 | DomainName: global.Key("domain-name").String(), 217 | TenantID: global.Key("tenant-id").String(), 218 | TenantName: global.Key("tenant-name").String(), 219 | AllowReauth: true, 220 | } 221 | eo := gophercloud.EndpointOpts{ 222 | Region: global.Key("region").String(), 223 | } 224 | 225 | return ao, eo, nil 226 | } 227 | 228 | func updateMetrics(provider *gophercloud.ProviderClient, eo gophercloud.EndpointOpts, clientset *kubernetes.Clientset, tenantID string) error { 229 | metricsMutex.Lock() 230 | defer metricsMutex.Unlock() 231 | 232 | var errs []error 233 | scrapeStart := time.Now() 234 | 235 | cinderClient, neutronClient, loadbalancerClient, computeClient, err := getClients(provider, eo) 236 | if err != nil { 237 | err := logError("creating openstack clients failed: %v", err) 238 | errs = append(errs, err) 239 | } else { 240 | if err := metrics.PublishCinderMetrics(cinderClient, clientset, tenantID); err != nil { 241 | err := logError("scraping cinder metrics failed: %v", err) 242 | errs = append(errs, err) 243 | } 244 | 245 | if err := metrics.PublishNeutronMetrics(neutronClient, tenantID); err != nil { 246 | err := logError("scraping neutron metrics failed: %v", err) 247 | errs = append(errs, err) 248 | } 249 | 250 | if err := metrics.PublishLoadBalancerMetrics(loadbalancerClient, tenantID); err != nil { 251 | err := logError("scraping load balancer metrics failed: %v", err) 252 | errs = append(errs, err) 253 | } 254 | 255 | if err := metrics.PublishServerMetrics(computeClient, tenantID); err != nil { 256 | err := logError("scraping server metrics failed: %v", err) 257 | errs = append(errs, err) 258 | } 259 | 260 | if err := metrics.PublishFirewallV1Metrics(neutronClient, tenantID); err != nil { 261 | err := logError("scraping firewall v1 metrics failed: %v", err) 262 | errs = append(errs, err) 263 | } 264 | 265 | if err := metrics.PublishFirewallV2Metrics(neutronClient, tenantID); err != nil { 266 | err := logError("scraping firewall v1 metrics failed: %v", err) 267 | errs = append(errs, err) 268 | } 269 | } 270 | 271 | duration := time.Since(scrapeStart) 272 | scrapeDuration.WithLabelValues( 273 | fmt.Sprintf("%d", *refreshInterval), 274 | ).Set(duration.Seconds()) 275 | 276 | scrapedAt.WithLabelValues( 277 | fmt.Sprintf("%d", *refreshInterval), 278 | ).Set(float64(scrapeStart.Unix())) 279 | 280 | if len(errs) > 0 { 281 | scrapedStatus.WithLabelValues( 282 | fmt.Sprintf("%d", *refreshInterval), 283 | ).Set(0) 284 | return fmt.Errorf("errors during scrape loop") 285 | } 286 | 287 | scrapedStatus.WithLabelValues( 288 | fmt.Sprintf("%d", *refreshInterval), 289 | ).Set(1) 290 | 291 | // reset backoff after successful scrape 292 | backoffSleep = time.Second 293 | 294 | return nil 295 | } 296 | 297 | func getClients(provider *gophercloud.ProviderClient, endpointOpts gophercloud.EndpointOpts) (cinder, neutron, loadbalancer, compute *gophercloud.ServiceClient, err error) { 298 | cinderClient, err := openstack.NewBlockStorageV3(provider, endpointOpts) 299 | if err != nil { 300 | return nil, nil, nil, nil, fmt.Errorf("unable to get cinder client: %v", err) 301 | } 302 | 303 | neutronClient, err := openstack.NewNetworkV2(provider, endpointOpts) 304 | if err != nil { 305 | return nil, nil, nil, nil, fmt.Errorf("unable to get neutron client: %v", err) 306 | } 307 | 308 | computeClient, err := openstack.NewComputeV2(provider, endpointOpts) 309 | if err != nil { 310 | return nil, nil, nil, nil, fmt.Errorf("unable to get compute client: %v", err) 311 | } 312 | 313 | if _, err := provider.EndpointLocator(gophercloud.EndpointOpts{Type: "load-balancer", Availability: gophercloud.AvailabilityPublic}); err != nil { 314 | // we can use the neutron client to access lbaas because no octavia is available 315 | return cinderClient, neutronClient, neutronClient, computeClient, nil 316 | } 317 | 318 | loadbalancerClient, err := openstack.NewLoadBalancerV2(provider, endpointOpts) 319 | if err != nil { 320 | return nil, nil, nil, nil, fmt.Errorf("failed to create loadbalancer service client: %v", err) 321 | } 322 | return cinderClient, neutronClient, loadbalancerClient, computeClient, nil 323 | } 324 | 325 | func min(a, b time.Duration) time.Duration { 326 | if a < b { 327 | return a 328 | } 329 | return b 330 | } 331 | 332 | func logError(format string, a ...interface{}) error { 333 | err := fmt.Errorf(format, a...) 334 | klog.Error(err) 335 | return err 336 | } 337 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | 2 | # Metrics overview 3 | 4 | As addition to the following metrics the exporter exposes some basic `go_*`, `process_` and `promhttp_*` metrics. 5 | 6 | 7 | ``` 8 | # HELP kos_cinder_quota_volume_disk_gigabytes Cinder volume metric (GB) 9 | # TYPE kos_cinder_quota_volume_disk_gigabytes gauge 10 | # HELP kos_cinder_quota_volume_disks Cinder volume metric (number of volumes) 11 | # TYPE kos_cinder_quota_volume_disks gauge 12 | # HELP kos_cinder_volume_attached_at Cinder volume attached at 13 | # TYPE kos_cinder_volume_attached_at gauge 14 | # HELP kos_cinder_volume_created_at Cinder volume created at 15 | # TYPE kos_cinder_volume_created_at gauge 16 | # HELP kos_cinder_volume_size Cinder volume size 17 | # TYPE kos_cinder_volume_size gauge 18 | # HELP kos_cinder_volume_status Cinder volume status 19 | # TYPE kos_cinder_volume_status gauge 20 | # HELP kos_cinder_volume_updated_at Cinder volume updated at 21 | # TYPE kos_cinder_volume_updated_at gauge 22 | # HELP kos_compute_quota_cores Number of instance cores allowed 23 | # TYPE kos_compute_quota_cores gauge 24 | # HELP kos_compute_quota_floating_ips Number of floating IPs allowed 25 | # TYPE kos_compute_quota_floating_ips gauge 26 | # HELP kos_compute_quota_instances Number of instances (servers) allowed 27 | # TYPE kos_compute_quota_instances gauge 28 | # HELP kos_compute_quota_ram_megabytes RAM (in MB) allowed 29 | # TYPE kos_compute_quota_ram_megabytes gauge 30 | # HELP kos_firewall_v1_admin_state_up Firewall v1 status 31 | # TYPE kos_firewall_v1_admin_state_up gauge 32 | # HELP kos_firewall_v1_status Firewall v1 status 33 | # TYPE kos_firewall_v1_status gauge 34 | # HELP kos_firewall_v2_group_admin_state_up Firewall v2 status 35 | # TYPE kos_firewall_v2_group_admin_state_up gauge 36 | # HELP kos_firewall_v2_group_status Firewall v2 status 37 | # TYPE kos_firewall_v2_group_status gauge 38 | # HELP kos_loadbalancer_admin_state_up Load balancer admin state up 39 | # TYPE kos_loadbalancer_admin_state_up gauge 40 | # HELP kos_loadbalancer_provisioning_status Load balancer status 41 | # TYPE kos_loadbalancer_provisioning_status gauge 42 | # HELP kos_neutron_floating_ip_status Neutron floating ip status 43 | # TYPE kos_neutron_floating_ip_status gauge 44 | # HELP kos_neutron_floatingip_created_at Neutron floating ip created at 45 | # TYPE kos_neutron_floatingip_created_at gauge 46 | # HELP kos_neutron_floatingip_updated_at Neutron floating ip updated at 47 | # TYPE kos_neutron_floatingip_updated_at gauge 48 | # HELP kos_openstack_api_request_duration_seconds Latency of an OpenStack API call 49 | # TYPE kos_openstack_api_request_duration_seconds histogram 50 | # HELP kos_openstack_api_requests_total Total number of OpenStack API calls 51 | # TYPE kos_openstack_api_requests_total counter 52 | # HELP kos_scrape_duration Time in seconds needed for the last scrape 53 | # TYPE kos_scrape_duration gauge 54 | # HELP kos_scrape_status_succeeded Scrape status succeeded 55 | # TYPE kos_scrape_status_succeeded gauge 56 | # HELP kos_scraped_at Timestamp when last scrape started 57 | # TYPE kos_scraped_at gauge 58 | # HELP kos_server_status Server status 59 | # TYPE kos_server_status gauge 60 | # HELP kos_server_volume_attachment Server volume attachment 61 | # TYPE kos_server_volume_attachment gauge 62 | # HELP kos_server_volume_attachment_count Server volume attachment count 63 | # TYPE kos_server_volume_attachment_count gauge 64 | ``` 65 | 66 | # Example Metrics 67 | 68 | 69 | ``` 70 | kos_cinder_quota_volume_disk_gigabytes{quota_type="allocated"} 0 71 | kos_cinder_quota_volume_disk_gigabytes{quota_type="in-use"} 8 72 | kos_cinder_quota_volume_disk_gigabytes{quota_type="limit"} 768 73 | kos_cinder_quota_volume_disk_gigabytes{quota_type="reserved"} 0 74 | kos_cinder_quota_volume_disks{quota_type="allocated"} 0 75 | kos_cinder_quota_volume_disks{quota_type="in-use"} 8 76 | kos_cinder_quota_volume_disks{quota_type="limit"} -1 77 | kos_cinder_quota_volume_disks{quota_type="reserved"} 0 78 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="",hostname="",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",server_id="",status="available",volume_type="novassd"} 0 79 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="",hostname="",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",server_id="",status="available",volume_type="novassd"} 0 80 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="/dev/sdb",hostname="",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",server_id="234162a9-ecaf-4f96-9fc9-78d2324e5e9b",status="in-use",volume_type="novassd"} 1.598575568e+09 81 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="/dev/sdb",hostname="",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",server_id="d1913d51-9588-471a-bc0d-94ad3a2e5ff2",status="in-use",volume_type="novassd"} 1.598575568e+09 82 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="/dev/sdb",hostname="",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",server_id="298e787c-5c1e-4ca3-a8db-c50c4d4c3bb8",status="in-use",volume_type="novassd"} 1.598575593e+09 83 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="/dev/sdc",hostname="",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",server_id="234162a9-ecaf-4f96-9fc9-78d2324e5e9b",status="in-use",volume_type="novassd"} 1.598575594e+09 84 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="/dev/sdc",hostname="",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",server_id="298e787c-5c1e-4ca3-a8db-c50c4d4c3bb8",status="in-use",volume_type="novassd"} 1.598575606e+09 85 | kos_cinder_volume_attached_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",device="/dev/sdc",hostname="",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",server_id="d1913d51-9588-471a-bc0d-94ad3a2e5ff2",status="in-use",volume_type="novassd"} 1.598575605e+09 86 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="available",volume_type="novassd"} 1.598575561e+09 87 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="available",volume_type="novassd"} 1.598575561e+09 88 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575588e+09 89 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575561e+09 90 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575561e+09 91 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575587e+09 92 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.5985756e+09 93 | kos_cinder_volume_created_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575599e+09 94 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="available",volume_type="novassd"} 1 95 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="available",volume_type="novassd"} 1 96 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 97 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 98 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 99 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 100 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 101 | kos_cinder_volume_size{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 102 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 103 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="available",volume_type="novassd"} 1 104 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 105 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 106 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="creating",volume_type="novassd"} 0 107 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 108 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 109 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 110 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="error",volume_type="novassd"} 0 111 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 112 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 113 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 114 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 115 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 116 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="extending",volume_type="novassd"} 0 117 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="in-use",volume_type="novassd"} 0 118 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 119 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="managing",volume_type="novassd"} 0 120 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 121 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 122 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 123 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 124 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 125 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="available",volume_type="novassd"} 1 126 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 127 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 128 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="creating",volume_type="novassd"} 0 129 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 130 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 131 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 132 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="error",volume_type="novassd"} 0 133 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 134 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 135 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 136 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 137 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 138 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="extending",volume_type="novassd"} 0 139 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="in-use",volume_type="novassd"} 0 140 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 141 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="managing",volume_type="novassd"} 0 142 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 143 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 144 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 145 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 146 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 147 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="available",volume_type="novassd"} 0 148 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 149 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 150 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="creating",volume_type="novassd"} 0 151 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 152 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 153 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 154 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="error",volume_type="novassd"} 0 155 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 156 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 157 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 158 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 159 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 160 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="extending",volume_type="novassd"} 0 161 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 162 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 163 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="managing",volume_type="novassd"} 0 164 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 165 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 166 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 167 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 168 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 169 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="available",volume_type="novassd"} 0 170 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 171 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 172 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="creating",volume_type="novassd"} 0 173 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 174 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 175 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 176 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="error",volume_type="novassd"} 0 177 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 178 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 179 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 180 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 181 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 182 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="extending",volume_type="novassd"} 0 183 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 184 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 185 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="managing",volume_type="novassd"} 0 186 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 187 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 188 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 189 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 190 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 191 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="available",volume_type="novassd"} 0 192 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 193 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 194 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="creating",volume_type="novassd"} 0 195 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 196 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 197 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 198 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="error",volume_type="novassd"} 0 199 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 200 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 201 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 202 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 203 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 204 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="extending",volume_type="novassd"} 0 205 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 206 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 207 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="managing",volume_type="novassd"} 0 208 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 209 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 210 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 211 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 212 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 213 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="available",volume_type="novassd"} 0 214 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 215 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 216 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="creating",volume_type="novassd"} 0 217 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 218 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 219 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 220 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="error",volume_type="novassd"} 0 221 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 222 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 223 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 224 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 225 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 226 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="extending",volume_type="novassd"} 0 227 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 228 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 229 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="managing",volume_type="novassd"} 0 230 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 231 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 232 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 233 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 234 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 235 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="available",volume_type="novassd"} 0 236 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 237 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 238 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="creating",volume_type="novassd"} 0 239 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 240 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 241 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 242 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="error",volume_type="novassd"} 0 243 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 244 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 245 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 246 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 247 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 248 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="extending",volume_type="novassd"} 0 249 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 250 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 251 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="managing",volume_type="novassd"} 0 252 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 253 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 254 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 255 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 256 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="attaching",volume_type="novassd"} 0 257 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="available",volume_type="novassd"} 0 258 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="awaiting-transfer",volume_type="novassd"} 0 259 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="backing-up",volume_type="novassd"} 0 260 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="creating",volume_type="novassd"} 0 261 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="deleting",volume_type="novassd"} 0 262 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="detaching",volume_type="novassd"} 0 263 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="downloading",volume_type="novassd"} 0 264 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="error",volume_type="novassd"} 0 265 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="error_backing-up",volume_type="novassd"} 0 266 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="error_deleting",volume_type="novassd"} 0 267 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="error_extending",volume_type="novassd"} 0 268 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="error_managing",volume_type="novassd"} 0 269 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="error_restoring",volume_type="novassd"} 0 270 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="extending",volume_type="novassd"} 0 271 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1 272 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="maintenance",volume_type="novassd"} 0 273 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="managing",volume_type="novassd"} 0 274 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="reserved",volume_type="novassd"} 0 275 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="restoring-backup",volume_type="novassd"} 0 276 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="retyping",volume_type="novassd"} 0 277 | kos_cinder_volume_status{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="uploading",volume_type="novassd"} 0 278 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="25fe54b8-ea60-4b7f-9529-4757089f2814",name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_fs_type="xfs",pv_name="pvc-b9465fd3-dd4c-4e7a-afd4-300a721e583e",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="pvc",pvc_namespace="default",status="available",volume_type="novassd"} 1.598575573e+09 279 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="4607f294-739c-4595-8cd0-4dba40c451b0",name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_fs_type="xfs",pv_name="pvc-38e1097c-b020-4773-bd72-3102ea703653",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="pvc-retain",pvc_namespace="default",status="available",volume_type="novassd"} 1.598575573e+09 280 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="67653dda-d1b4-4280-92ec-1397c8a32da0",name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_fs_type="xfs",pv_name="pvc-6993c3ad-34fa-43d0-9556-1465a0ba72b9",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575596e+09 281 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="9a74156a-2233-4b54-aeca-18ba72eb3f0b",name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_fs_type="xfs",pv_name="pvc-f15c30a5-94b5-447e-862c-50e98750961d",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.59857557e+09 282 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="a4a843b9-4021-4a7c-abd4-6830a5482023",name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_fs_type="xfs",pv_name="pvc-dee6655a-546d-4cdf-bed0-b37c9e3af23b",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-0",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575569e+09 283 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="d553ffc0-30a7-40ed-ba79-40f21dfcaf76",name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_fs_type="xfs",pv_name="pvc-d05b7c8b-231e-47ef-99d4-d8f0a2bc58d4",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-1",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575594e+09 284 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="e010c879-0b53-4204-a723-1b26a8ad5e3f",name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_fs_type="xfs",pv_name="pvc-6fe7433a-5205-4b8b-acff-e17416a98515",pv_reclaim_policy="Delete",pv_storage_class="cinder",pvc_name="vol-sts-pvc-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575608e+09 285 | kos_cinder_volume_updated_at{cinder_availability_zone="nova",description="Created by OpenStack Cinder CSI driver",id="fd2de264-1a01-44e5-88ed-a63f26336142",name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_fs_type="xfs",pv_name="pvc-d6fc37f5-b3ef-4ef9-abf7-ef0391262f12",pv_reclaim_policy="Retain",pv_storage_class="cinder-retain",pvc_name="vol-sts-pvc-retain-2",pvc_namespace="default",status="in-use",volume_type="novassd"} 1.598575606e+09 286 | kos_compute_quota_cores{quota_type="in-use"} 0 287 | kos_compute_quota_cores{quota_type="limit"} 80 288 | kos_compute_quota_cores{quota_type="reserved"} 0 289 | kos_compute_quota_floating_ips{quota_type="in-use"} 0 290 | kos_compute_quota_floating_ips{quota_type="limit"} 10 291 | kos_compute_quota_floating_ips{quota_type="reserved"} 0 292 | kos_compute_quota_instances{quota_type="in-use"} 0 293 | kos_compute_quota_instances{quota_type="limit"} 40 294 | kos_compute_quota_instances{quota_type="reserved"} 0 295 | kos_compute_quota_ram_megabytes{quota_type="in-use"} 0 296 | kos_compute_quota_ram_megabytes{quota_type="limit"} 512000 297 | kos_compute_quota_ram_megabytes{quota_type="reserved"} 0 298 | kos_firewall_v1_admin_state_up{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200"} 1 299 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="ACTIVE"} 1 300 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="DOWN"} 0 301 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="ERROR"} 0 302 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="INACTIVE"} 0 303 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="PENDING_CREATE"} 0 304 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="PENDING_DELETE"} 0 305 | kos_firewall_v1_status{description="",id="cb4a31f4-f20f-4a67-9945-21ba0c5a3d21",name="my-firewall",policyID="fc73d805-324b-4415-92ae-b8a4c4232abe",projectID="5af4393c0a4b4eb98b2c0b61393f1200",status="PENDING_UPDATE"} 0 306 | kos_firewall_v2_group_admin_state_up{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f"} 1 307 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="ACTIVE"} 1 308 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="DOWN"} 0 309 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="ERROR"} 0 310 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="INACTIVE"} 0 311 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="PENDING_CREATE"} 0 312 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="PENDING_DELETE"} 0 313 | kos_firewall_v2_group_status{description="",egressPolicyID="cbb204f5-3a92-427f-b082-c34a004856df",id="f4d10c07-4b29-4b91-a57c-7a4762634f90",ingressPolicyID="ca06e59e-5d49-4944-80a6-f14e01edc01c",name="my-firewall",projectID="1b08db53fdec4c498a15ad7d027eac4f",status="PENDING_UPDATE"} 0 314 | kos_loadbalancer_admin_state_up{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",vip_address="10.6.0.8"} 1 315 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="ACTIVE",vip_address="10.6.0.8"} 1 316 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="ALLOCATED",vip_address="10.6.0.8"} 0 317 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="BOOTING",vip_address="10.6.0.8"} 0 318 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="DELETED",vip_address="10.6.0.8"} 0 319 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="ERROR",vip_address="10.6.0.8"} 0 320 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="PENDING_CREATE",vip_address="10.6.0.8"} 0 321 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="PENDING_DELETE",vip_address="10.6.0.8"} 0 322 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="PENDING_UPDATE",vip_address="10.6.0.8"} 0 323 | kos_loadbalancer_provisioning_status{id="059e7a29-5229-409e-870d-6ceb9c1059a9",name="foo",port_id="10902a71-299d-4666-b086-1a0725288dac",provider="vmwareedge",provisioning_status="READY",vip_address="10.6.0.8"} 0 324 | kos_neutron_floating_ip_status{fixed_ip="",floating_ip="172.17.0.161",id="aa8ae8f5-fd6e-49fc-a755-93cccc74ae11",port_id=""} 1 325 | kos_neutron_floating_ip_status{fixed_ip="",floating_ip="172.17.0.174",id="ed22b15f-05fd-401d-8bb1-9745c7b8e93d",port_id=""} 1 326 | kos_neutron_floating_ip_status{fixed_ip="10.6.0.8",floating_ip="172.17.0.173",id="81a46647-ff4e-4697-9162-f802a15e88a5",port_id="10902a71-299d-4666-b086-1a0725288dac"} 1 327 | kos_neutron_floatingip_created_at{fixed_ip="",floating_ip="172.17.0.161",id="aa8ae8f5-fd6e-49fc-a755-93cccc74ae11",port_id=""} 1.598578483e+09 328 | kos_neutron_floatingip_created_at{fixed_ip="",floating_ip="172.17.0.174",id="ed22b15f-05fd-401d-8bb1-9745c7b8e93d",port_id=""} 1.598574125e+09 329 | kos_neutron_floatingip_created_at{fixed_ip="10.6.0.8",floating_ip="172.17.0.173",id="81a46647-ff4e-4697-9162-f802a15e88a5",port_id="10902a71-299d-4666-b086-1a0725288dac"} 1.5985741e+09 330 | kos_neutron_floatingip_updated_at{fixed_ip="",floating_ip="172.17.0.161",id="aa8ae8f5-fd6e-49fc-a755-93cccc74ae11",port_id=""} 1.598578483e+09 331 | kos_neutron_floatingip_updated_at{fixed_ip="",floating_ip="172.17.0.174",id="ed22b15f-05fd-401d-8bb1-9745c7b8e93d",port_id=""} 1.598574125e+09 332 | kos_neutron_floatingip_updated_at{fixed_ip="10.6.0.8",floating_ip="172.17.0.173",id="81a46647-ff4e-4697-9162-f802a15e88a5",port_id="10902a71-299d-4666-b086-1a0725288dac"} 1.5985741e+09 333 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.005"} 0 334 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.01"} 0 335 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.025"} 0 336 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.05"} 0 337 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.1"} 0 338 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.25"} 1 339 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="0.5"} 1 340 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="1"} 1 341 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="2.5"} 1 342 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="5"} 1 343 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="10"} 1 344 | kos_openstack_api_request_duration_seconds_bucket{request="compute_quotasets_detail_get",le="+Inf"} 1 345 | kos_openstack_api_request_duration_seconds_sum{request="compute_quotasets_detail_get"} 0.148614815 346 | kos_openstack_api_request_duration_seconds_count{request="compute_quotasets_detail_get"} 1 347 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.005"} 0 348 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.01"} 0 349 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.025"} 0 350 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.05"} 0 351 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.1"} 0 352 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.25"} 0 353 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="0.5"} 1 354 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="1"} 1 355 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="2.5"} 1 356 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="5"} 1 357 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="10"} 1 358 | kos_openstack_api_request_duration_seconds_bucket{request="floating_ip_list",le="+Inf"} 1 359 | kos_openstack_api_request_duration_seconds_sum{request="floating_ip_list"} 0.467078935 360 | kos_openstack_api_request_duration_seconds_count{request="floating_ip_list"} 1 361 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.005"} 0 362 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.01"} 0 363 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.025"} 0 364 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.05"} 0 365 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.1"} 0 366 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.25"} 1 367 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="0.5"} 1 368 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="1"} 1 369 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="2.5"} 1 370 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="5"} 1 371 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="10"} 1 372 | kos_openstack_api_request_duration_seconds_bucket{request="loadbalancer_list",le="+Inf"} 1 373 | kos_openstack_api_request_duration_seconds_sum{request="loadbalancer_list"} 0.107812617 374 | kos_openstack_api_request_duration_seconds_count{request="loadbalancer_list"} 1 375 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.005"} 0 376 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.01"} 0 377 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.025"} 0 378 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.05"} 0 379 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.1"} 0 380 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.25"} 0 381 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="0.5"} 0 382 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="1"} 1 383 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="2.5"} 1 384 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="5"} 1 385 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="10"} 1 386 | kos_openstack_api_request_duration_seconds_bucket{request="volume_list",le="+Inf"} 1 387 | kos_openstack_api_request_duration_seconds_sum{request="volume_list"} 0.738951285 388 | kos_openstack_api_request_duration_seconds_count{request="volume_list"} 1 389 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.005"} 0 390 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.01"} 0 391 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.025"} 0 392 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.05"} 0 393 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.1"} 0 394 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.25"} 1 395 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="0.5"} 1 396 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="1"} 1 397 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="2.5"} 1 398 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="5"} 1 399 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="10"} 1 400 | kos_openstack_api_request_duration_seconds_bucket{request="volume_quotasets_usage_get",le="+Inf"} 1 401 | kos_openstack_api_request_duration_seconds_sum{request="volume_quotasets_usage_get"} 0.135694331 402 | kos_openstack_api_request_duration_seconds_count{request="volume_quotasets_usage_get"} 1 403 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.005"} 0 404 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.01"} 0 405 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.025"} 0 406 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.05"} 0 407 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.1"} 1 408 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.25"} 1 409 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="0.5"} 1 410 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="1"} 1 411 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="2.5"} 1 412 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="5"} 1 413 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="10"} 1 414 | kos_openstack_api_request_duration_seconds_bucket{request="firewallv2_group_list",le="+Inf"} 1 415 | kos_openstack_api_request_duration_seconds_sum{request="firewallv2_group_list"} 0.061682213 416 | kos_openstack_api_request_duration_seconds_count{request="firewallv2_group_list"} 1 417 | kos_openstack_api_requests_total{request="firewallv2_group_list"} 1 418 | kos_openstack_api_requests_total{request="firewall_list"} 1 419 | kos_openstack_api_requests_total{request="floating_ip_list"} 1 420 | kos_openstack_api_requests_total{request="loadbalancer_list"} 1 421 | kos_openstack_api_requests_total{request="volume_list"} 1 422 | kos_openstack_api_requests_total{request="volume_quotasets_usage_get"} 1 423 | kos_scrape_duration{refresh_interval="120"} 1.957081635 424 | kos_scrape_status_succeeded{refresh_interval="120"} 1 425 | kos_scraped_at{refresh_interval="120"} 1.598625922e+09 426 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="ACTIVE"} 1 427 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="BUILDING"} 0 428 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="DELETED"} 0 429 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="ERROR"} 0 430 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="PAUSED"} 0 431 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="RESCUED"} 0 432 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="RESIZED"} 0 433 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="SHELVED"} 0 434 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="SHELVED_OFFLOADED"} 0 435 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="SOFT_DELETED"} 0 436 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="STOPPED"} 0 437 | kos_server_status{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",status="SUSPENDED"} 0 438 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="ACTIVE"} 1 439 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="BUILDING"} 0 440 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="DELETED"} 0 441 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="ERROR"} 0 442 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="PAUSED"} 0 443 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="RESCUED"} 0 444 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="RESIZED"} 0 445 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="SHELVED"} 0 446 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="SHELVED_OFFLOADED"} 0 447 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="SOFT_DELETED"} 0 448 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="STOPPED"} 0 449 | kos_server_status{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",status="SUSPENDED"} 0 450 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="ACTIVE"} 1 451 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="BUILDING"} 0 452 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="DELETED"} 0 453 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="ERROR"} 0 454 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="PAUSED"} 0 455 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="RESCUED"} 0 456 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="RESIZED"} 0 457 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="SHELVED"} 0 458 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="SHELVED_OFFLOADED"} 0 459 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="SOFT_DELETED"} 0 460 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="STOPPED"} 0 461 | kos_server_status{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",status="SUSPENDED"} 0 462 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="ACTIVE"} 1 463 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="BUILDING"} 0 464 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="DELETED"} 0 465 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="ERROR"} 0 466 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="PAUSED"} 0 467 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="RESCUED"} 0 468 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="RESIZED"} 0 469 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="SHELVED"} 0 470 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="SHELVED_OFFLOADED"} 0 471 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="SOFT_DELETED"} 0 472 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="STOPPED"} 0 473 | kos_server_status{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01",status="SUSPENDED"} 0 474 | kos_server_volume_attachment{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",volume_id="b3d989e7-3f28-4224-bc30-62c198bf56d0"} 1 475 | kos_server_volume_attachment{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01",volume_id="f37d89a4-a8fd-46bb-9c08-fba5b3e22398"} 1 476 | kos_server_volume_attachment{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",volume_id="726ca281-e21c-41c2-ab2e-bb44c2894db1"} 1 477 | kos_server_volume_attachment{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02",volume_id="9806bff3-86f9-402b-98f6-70af7fe524f1"} 1 478 | kos_server_volume_attachment{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",volume_id="9b672d6a-8231-4b10-90a4-90e481bf31d4"} 1 479 | kos_server_volume_attachment{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03",volume_id="7f6d127f-5e53-41db-b710-57515bfe61ee"} 1 480 | kos_server_volume_attachment_count{id="5e10188d-214d-4949-857c-171c83b80ad6",name="k8s-kube-node01"} 2 481 | kos_server_volume_attachment_count{id="a373e367-c661-454c-83c9-3f9186c7a55a",name="k8s-kube-node02"} 2 482 | kos_server_volume_attachment_count{id="c20db019-dff7-457f-9076-943a669cd464",name="k8s-kube-node03"} 2 483 | kos_server_volume_attachment_count{id="f98bccd0-8627-4075-9d31-e6fc9ea99ce3",name="k8s-kube-master01"} 0 484 | ``` 485 | --------------------------------------------------------------------------------