├── .dockerignore ├── .editorconfig ├── .github ├── dependabot.yml └── workflows │ ├── build-docker.yaml │ ├── ci-docker.yaml │ ├── release-assets.yaml │ ├── release-docker.yaml │ └── schedule-docker.yaml ├── .gitignore ├── .golangci.yaml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── common.logger.go ├── common.system.go ├── config ├── consts.go └── opts.go ├── go.mod ├── go.sum ├── main.go ├── metrics ├── azure.policy.go ├── insights.general.go ├── insights.go ├── insights.subscription.go ├── insights.target.go ├── metrics.go ├── misc.go ├── prober.go ├── servicediscovery.go └── settings.go ├── misc.go ├── probe_metrics_list.go ├── probe_metrics_resource.go ├── probe_metrics_resourcegraph.go ├── probe_metrics_scrape.go ├── probe_metrics_subscription.go └── templates └── query.html /.dockerignore: -------------------------------------------------------------------------------- 1 | /azure-metrics-exporter 2 | /release-assets 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | [*] 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | end_of_line = lf 10 | insert_final_newline = true 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [Makefile] 15 | indent_style = tab 16 | 17 | [{*.yml,*.yaml}] 18 | indent_size = 2 19 | 20 | [*.conf] 21 | indent_size = 2 22 | 23 | [*.go] 24 | indent_size = 4 25 | indent_style = tab 26 | ij_continuation_indent_size = 4 27 | ij_go_GROUP_CURRENT_PROJECT_IMPORTS = true 28 | ij_go_add_leading_space_to_comments = true 29 | ij_go_add_parentheses_for_single_import = true 30 | ij_go_call_parameters_new_line_after_left_paren = true 31 | ij_go_call_parameters_right_paren_on_new_line = true 32 | ij_go_call_parameters_wrap = off 33 | ij_go_fill_paragraph_width = 80 34 | ij_go_group_stdlib_imports = true 35 | ij_go_import_sorting = goimports 36 | ij_go_keep_indents_on_empty_lines = false 37 | ij_go_local_group_mode = project 38 | ij_go_move_all_imports_in_one_declaration = true 39 | ij_go_move_all_stdlib_imports_in_one_group = true 40 | ij_go_remove_redundant_import_aliases = false 41 | ij_go_run_go_fmt_on_reformat = true 42 | ij_go_use_back_quotes_for_imports = false 43 | ij_go_wrap_comp_lit = off 44 | ij_go_wrap_comp_lit_newline_after_lbrace = true 45 | ij_go_wrap_comp_lit_newline_before_rbrace = true 46 | ij_go_wrap_func_params = off 47 | ij_go_wrap_func_params_newline_after_lparen = true 48 | ij_go_wrap_func_params_newline_before_rparen = true 49 | ij_go_wrap_func_result = off 50 | ij_go_wrap_func_result_newline_after_lparen = true 51 | ij_go_wrap_func_result_newline_before_rparen = true 52 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | groups: 8 | all-github-actions: 9 | patterns: [ "*" ] 10 | 11 | - package-ecosystem: "docker" 12 | directory: "/" 13 | schedule: 14 | interval: "weekly" 15 | groups: 16 | all-docker-versions: 17 | patterns: [ "*" ] 18 | 19 | - package-ecosystem: "gomod" 20 | directory: "/" 21 | schedule: 22 | interval: "weekly" 23 | groups: 24 | all-go-mod-patch-and-minor: 25 | patterns: [ "*" ] 26 | update-types: [ "patch", "minor" ] 27 | -------------------------------------------------------------------------------- /.github/workflows/build-docker.yaml: -------------------------------------------------------------------------------- 1 | name: build/docker 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | publish: 7 | required: true 8 | type: boolean 9 | 10 | jobs: 11 | lint: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Set Swap Space 17 | uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c 18 | with: 19 | swap-size-gb: 12 20 | 21 | - uses: actions/setup-go@v5 22 | with: 23 | go-version-file: 'go.mod' 24 | cache-dependency-path: "go.sum" 25 | check-latest: true 26 | 27 | - name: Run Golangci lint 28 | uses: golangci/golangci-lint-action@v7 29 | with: 30 | version: latest 31 | args: --print-resources-usage 32 | 33 | build: 34 | name: "build ${{ matrix.Dockerfile }}:${{ matrix.target }}" 35 | needs: lint 36 | strategy: 37 | fail-fast: false 38 | matrix: 39 | include: 40 | - Dockerfile: Dockerfile 41 | target: "final-static" 42 | suffix: "" 43 | latest: "auto" 44 | 45 | runs-on: ubuntu-latest 46 | steps: 47 | - uses: actions/checkout@v4 48 | 49 | - name: Set Swap Space 50 | uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c 51 | with: 52 | swap-size-gb: 12 53 | 54 | - uses: actions/setup-go@v5 55 | with: 56 | go-version-file: 'go.mod' 57 | cache-dependency-path: "go.sum" 58 | check-latest: true 59 | 60 | - name: Docker meta 61 | id: docker_meta 62 | uses: docker/metadata-action@v5 63 | with: 64 | images: | 65 | ${{ github.repository }} 66 | quay.io/${{ github.repository }} 67 | labels: | 68 | io.artifacthub.package.readme-url=https://raw.githubusercontent.com/${{ github.repository }}/${{ github.event.repository.default_branch }}/README.md 69 | flavor: | 70 | latest=${{ matrix.latest }} 71 | suffix=${{ matrix.suffix }} 72 | 73 | - name: Set up QEMU 74 | uses: docker/setup-qemu-action@v3 75 | 76 | - name: Set up Docker Buildx 77 | uses: docker/setup-buildx-action@v3 78 | 79 | - name: Login to DockerHub 80 | uses: docker/login-action@v3 81 | if: ${{ inputs.publish }} 82 | with: 83 | username: ${{ secrets.DOCKERHUB_USERNAME }} 84 | password: ${{ secrets.DOCKERHUB_TOKEN }} 85 | 86 | - name: Login to Quay 87 | uses: docker/login-action@v3 88 | if: ${{ inputs.publish }} 89 | with: 90 | registry: quay.io 91 | username: ${{ secrets.QUAY_USERNAME }} 92 | password: ${{ secrets.QUAY_TOKEN }} 93 | 94 | - name: ${{ inputs.publish && 'Build and push' || 'Build' }} 95 | uses: docker/build-push-action@v6 96 | with: 97 | context: . 98 | file: ./${{ matrix.Dockerfile }} 99 | target: ${{ matrix.target }} 100 | platforms: linux/amd64,linux/arm64 101 | push: ${{ inputs.publish }} 102 | tags: ${{ steps.docker_meta.outputs.tags }} 103 | labels: ${{ steps.docker_meta.outputs.labels }} 104 | -------------------------------------------------------------------------------- /.github/workflows/ci-docker.yaml: -------------------------------------------------------------------------------- 1 | name: "ci/docker" 2 | 3 | on: [pull_request, workflow_dispatch] 4 | 5 | jobs: 6 | build: 7 | uses: ./.github/workflows/build-docker.yaml 8 | secrets: inherit 9 | with: 10 | publish: false 11 | -------------------------------------------------------------------------------- /.github/workflows/release-assets.yaml: -------------------------------------------------------------------------------- 1 | name: "release/assets" 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | release: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | 13 | - name: Set Swap Space 14 | uses: pierotofy/set-swap-space@49819abfb41bd9b44fb781159c033dba90353a7c 15 | with: 16 | swap-size-gb: 12 17 | 18 | - uses: actions/setup-go@v5 19 | with: 20 | go-version-file: 'go.mod' 21 | cache-dependency-path: "go.sum" 22 | check-latest: true 23 | 24 | - name: Build 25 | run: | 26 | make release-assets 27 | 28 | - name: Upload assets to release 29 | uses: svenstaro/upload-release-action@v2 30 | with: 31 | repo_token: ${{ secrets.GITHUB_TOKEN }} 32 | file: ./release-assets/* 33 | tag: ${{ github.ref }} 34 | overwrite: true 35 | file_glob: true 36 | -------------------------------------------------------------------------------- /.github/workflows/release-docker.yaml: -------------------------------------------------------------------------------- 1 | name: "release/docker" 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | - 'feature-**' 8 | - 'bugfix-**' 9 | tags: 10 | - '*.*.*' 11 | 12 | jobs: 13 | release: 14 | uses: ./.github/workflows/build-docker.yaml 15 | secrets: inherit 16 | with: 17 | publish: ${{ github.event_name != 'pull_request' }} 18 | -------------------------------------------------------------------------------- /.github/workflows/schedule-docker.yaml: -------------------------------------------------------------------------------- 1 | name: "schedule/docker" 2 | 3 | on: 4 | schedule: 5 | - cron: '45 6 * * 1' 6 | 7 | jobs: 8 | schedule: 9 | uses: ./.github/workflows/build-docker.yaml 10 | secrets: inherit 11 | with: 12 | publish: true 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | /azure-metrics-exporter 3 | /release-assets 4 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | linters: 3 | enable: 4 | - asciicheck 5 | - bidichk 6 | - bodyclose 7 | - copyloopvar 8 | - errorlint 9 | - gomodguard 10 | - gosec 11 | settings: 12 | gomodguard: 13 | blocked: 14 | modules: 15 | - github.com/Azure/go-autorest/autorest/azure/auth: 16 | reason: deprecated 17 | gosec: 18 | confidence: low 19 | config: 20 | global: 21 | audit: true 22 | exclusions: 23 | generated: lax 24 | presets: 25 | - comments 26 | - common-false-positives 27 | - legacy 28 | - std-error-handling 29 | paths: 30 | - third_party$ 31 | - builtin$ 32 | - examples$ 33 | formatters: 34 | enable: 35 | - gofmt 36 | - goimports 37 | exclusions: 38 | generated: lax 39 | paths: 40 | - third_party$ 41 | - builtin$ 42 | - examples$ 43 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ############################################# 2 | # Build 3 | ############################################# 4 | FROM --platform=$BUILDPLATFORM golang:1.24-alpine AS build 5 | 6 | RUN apk upgrade --no-cache --force 7 | RUN apk add --update build-base make git 8 | 9 | WORKDIR /go/src/github.com/webdevops/azure-metrics-exporter 10 | 11 | # Dependencies 12 | COPY go.mod go.sum . 13 | RUN go mod download 14 | 15 | # Compile 16 | COPY . . 17 | RUN make test 18 | ARG TARGETOS TARGETARCH 19 | RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make build 20 | 21 | ############################################# 22 | # Test 23 | ############################################# 24 | FROM gcr.io/distroless/static AS test 25 | USER 0:0 26 | WORKDIR /app 27 | COPY --from=build /go/src/github.com/webdevops/azure-metrics-exporter/azure-metrics-exporter . 28 | RUN ["./azure-metrics-exporter", "--help"] 29 | 30 | ############################################# 31 | # final-static 32 | ############################################# 33 | FROM gcr.io/distroless/static AS final-static 34 | ENV LOG_JSON=1 35 | WORKDIR / 36 | COPY --from=test /app . 37 | USER 1000:1000 38 | ENTRYPOINT ["/azure-metrics-exporter"] 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019-2021 WebDevOps 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT_NAME := $(shell basename $(CURDIR)) 2 | GIT_TAG := $(shell git describe --dirty --tags --always) 3 | GIT_COMMIT := $(shell git rev-parse --short HEAD) 4 | LDFLAGS := -X "main.gitTag=$(GIT_TAG)" -X "main.gitCommit=$(GIT_COMMIT)" -extldflags "-static" -s -w 5 | 6 | FIRST_GOPATH := $(firstword $(subst :, ,$(shell go env GOPATH))) 7 | GOLANGCI_LINT_BIN := $(FIRST_GOPATH)/bin/golangci-lint 8 | 9 | .PHONY: all 10 | all: vendor build 11 | 12 | .PHONY: clean 13 | clean: 14 | git clean -Xfd . 15 | 16 | ####################################### 17 | # builds 18 | ####################################### 19 | 20 | .PHONY: vendor 21 | vendor: 22 | go mod tidy 23 | go mod vendor 24 | go mod verify 25 | 26 | .PHONY: build-all 27 | build-all: 28 | GOOS=linux GOARCH=${GOARCH} CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o '$(PROJECT_NAME)' . 29 | GOOS=darwin GOARCH=${GOARCH} CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o '$(PROJECT_NAME).darwin' . 30 | GOOS=windows GOARCH=${GOARCH} CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o '$(PROJECT_NAME).exe' . 31 | 32 | .PHONY: build 33 | build: 34 | GOOS=${GOOS} GOARCH=${GOARCH} CGO_ENABLED=0 go build -ldflags '$(LDFLAGS)' -o $(PROJECT_NAME) . 35 | 36 | .PHONY: image 37 | image: image 38 | docker build -t $(PROJECT_NAME):$(GIT_TAG) . 39 | 40 | .PHONY: build-push-development 41 | build-push-development: 42 | docker buildx create --use 43 | docker buildx build -t webdevops/$(PROJECT_NAME):development --platform linux/amd64,linux/arm,linux/arm64 --push . 44 | 45 | ####################################### 46 | # quality checks 47 | ####################################### 48 | 49 | .PHONY: check 50 | check: vendor lint test 51 | 52 | .PHONY: test 53 | test: 54 | time go test ./... 55 | 56 | .PHONY: lint 57 | lint: $(GOLANGCI_LINT_BIN) 58 | time $(GOLANGCI_LINT_BIN) run --verbose --print-resources-usage 59 | 60 | $(GOLANGCI_LINT_BIN): 61 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(FIRST_GOPATH)/bin 62 | 63 | ####################################### 64 | # release assets 65 | ####################################### 66 | 67 | RELEASE_ASSETS = \ 68 | $(foreach GOARCH,amd64 arm64,\ 69 | $(foreach GOOS,linux darwin windows,\ 70 | release-assets/$(GOOS).$(GOARCH))) \ 71 | 72 | word-dot = $(word $2,$(subst ., ,$1)) 73 | 74 | .PHONY: release-assets 75 | release-assets: clean-release-assets vendor $(RELEASE_ASSETS) 76 | 77 | .PHONY: clean-release-assets 78 | clean-release-assets: 79 | rm -rf ./release-assets 80 | mkdir -p ./release-assets 81 | 82 | release-assets/windows.%: $(SOURCE) 83 | echo 'build release-assets for windows/$(call word-dot,$*,2)' 84 | GOOS=windows \ 85 | GOARCH=$(call word-dot,$*,1) \ 86 | CGO_ENABLED=0 \ 87 | time go build -ldflags '$(LDFLAGS)' -o './release-assets/$(PROJECT_NAME).windows.$(call word-dot,$*,1).exe' . 88 | 89 | release-assets/%: $(SOURCE) 90 | echo 'build release-assets for $(call word-dot,$*,1)/$(call word-dot,$*,2)' 91 | GOOS=$(call word-dot,$*,1) \ 92 | GOARCH=$(call word-dot,$*,2) \ 93 | CGO_ENABLED=0 \ 94 | time go build -ldflags '$(LDFLAGS)' -o './release-assets/$(PROJECT_NAME).$(call word-dot,$*,1).$(call word-dot,$*,2)' . 95 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Monitor metrics exporter 2 | 3 | [![license](https://img.shields.io/github/license/webdevops/azure-metrics-exporter.svg)](https://github.com/webdevops/azure-metrics-exporter/blob/master/LICENSE) 4 | [![DockerHub](https://img.shields.io/badge/DockerHub-webdevops%2Fazure--metrics--exporter-blue)](https://hub.docker.com/r/webdevops/azure-metrics-exporter/) 5 | [![Quay.io](https://img.shields.io/badge/Quay.io-webdevops%2Fazure--metrics--exporter-blue)](https://quay.io/repository/webdevops/azure-metrics-exporter) 6 | [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/azure-metrics-exporter)](https://artifacthub.io/packages/search?repo=azure-metrics-exporter) 7 | 8 | Prometheus exporter for Azure Monitor metrics. 9 | Supports metrics fetching from all resource with one scrape (automatic service discovery), custom metric names with template system, full dimensions support and caching. 10 | 11 | Configuration (except Azure connection) of this exporter is made entirely in Prometheus instead of a seperate configuration file, see examples below. 12 | 13 | TOC: 14 | * [Features](#Features) 15 | * [Configuration](#configuration) 16 | * [Metrics](#metrics) 17 | + [Azuretracing metrics](#azuretracing-metrics) 18 | + [Metric name and help template system](#metric-name-and-help-template-system) 19 | - [default template](#default-template) 20 | - [template `{name}_{metric}_{unit}`](#template-name_metric_unit) 21 | - [template `{name}_{metric}_{aggregation}_{unit}`](#template-name_metric_aggregation_unit) 22 | * [HTTP Endpoints](#http-endpoints) 23 | + [/probe/metrics parameters](#probemetrics-parameters) 24 | + [/probe/metrics/resource parameters](#probemetricsresource-parameters) 25 | + [/probe/metrics/list parameters](#probemetricslist-parameters) 26 | + [/probe/metrics/scrape parameters](#probemetricsscrape-parameters) 27 | * [Prometheus configuration examples](#prometheus-configuration-examples) 28 | * [Redis](#Redis) 29 | * [VirtualNetworkGateways](#virtualnetworkgateways) 30 | * [virtualNetworkGateway connections (dimension support)](#virtualnetworkgateway-connections-dimension-support) 31 | * [StorageAccount (metric namespace and dimension support)](#storageaccount-metric-namespace-and-dimension-support) 32 | * [Development and testing query webui](#development-and-testing-query-webui) 33 | 34 | ## Features 35 | 36 | - Uses of official [Azure SDK for go](https://github.com/Azure/azure-sdk-for-go) 37 | - Supports all Azure environments (Azure public cloud, Azure governmant cloud, Azure china cloud, ...) via Azure SDK configuration 38 | - Caching of Azure ServiceDiscovery to reduce Azure API calls 39 | - Caching of fetched metrics (no need to request every minute from Azure Monitor API; you can keep scrape time of `30s` for metrics) 40 | - Customizable metric names (with [template system with metric information](#metric-name-template-system)) 41 | - Ability to fetch metrics from one or more resources via `target` parameter (see `/probe/metrics/resource`) 42 | - Ability to fetch metrics from resources found with ServiceDiscovery via [Azure resources API based on $filter](https://docs.microsoft.com/en-us/rest/api/resources/resources/list) (see `/probe/metrics/list`) 43 | - Ability to fetch metrics from resources found with ServiceDiscovery via [Azure resources API based on $filter](https://docs.microsoft.com/en-us/rest/api/resources/resources/list) with configuration inside Azure resource tags (see `/probe/metrics/scrape`) 44 | - Ability to fetch metrics from resources found with ServiceDiscovery via [Azure ResourceGraph API based on Kusto query](https://docs.microsoft.com/en-us/azure/governance/resource-graph/overview) (see `/probe/metrics/resourcegraph`) 45 | - Configuration based on Prometheus scraping config or ServiceMonitor manifest (Prometheus operator) 46 | - Metric manipulation (adding, removing, updating or filtering of labels or metrics) can be done in scraping config (eg [`metric_relabel_configs`](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs)) 47 | - Full metric [dimension support](#virtualnetworkgateway-connections-dimension-support) 48 | - Docker image is based on [Google's distroless](https://github.com/GoogleContainerTools/distroless) static image to reduce attack surface (no shell, no other binaries inside image) 49 | - Available via Docker Hub and Quay (see badges on top) 50 | - Can run non-root and with readonly root filesystem, doesn't need any capabilities (you can safely use `drop: ["All"]`) 51 | - Publishes Azure API rate limit metrics (when exporter sends Azure API requests, available via `/metrics`) 52 | 53 | useful with additional exporters: 54 | 55 | - [azure-resourcegraph-exporter](https://github.com/webdevops/azure-resourcegraph-exporter) for exporting Azure resource information from Azure ResourceGraph API with custom Kusto queries (get the tags from resources and ResourceGroups with this exporter) 56 | - [azure-resourcemanager-exporter](https://github.com/webdevops/azure-resourcemanager-exporter) for exporting Azure subscription information (eg ratelimit, subscription quotas, ServicePrincipal expiry, RoleAssignments, resource health, ...) 57 | - [azure-keyvault-exporter](https://github.com/webdevops/azure-keyvault-exporter) for exporting Azure KeyVault information (eg expiry date for secrets, certificates and keys) 58 | - [azure-loganalytics-exporter](https://github.com/webdevops/azure-loganalytics-exporter) for exporting Azure LogAnalytics workspace information with custom Kusto queries (eg ingestion rate or application error count) 59 | 60 | ## Configuration 61 | 62 | Normally no configuration is needed but can be customized using environment variables. 63 | 64 | ``` 65 | Usage: 66 | azure-metrics-exporter [OPTIONS] 67 | 68 | Application Options: 69 | --log.debug debug mode [$LOG_DEBUG] 70 | --log.devel development mode [$LOG_DEVEL] 71 | --log.json Switch log output to json format [$LOG_JSON] 72 | --azure-environment= Azure environment name (default: AZUREPUBLICCLOUD) [$AZURE_ENVIRONMENT] 73 | --azure-ad-resource-url= Specifies the AAD resource ID to use. If not set, it defaults to ResourceManagerEndpoint for operations with Azure Resource Manager 74 | [$AZURE_AD_RESOURCE] 75 | --azure.servicediscovery.cache= Duration for caching Azure ServiceDiscovery of workspaces to reduce API calls (time.Duration) (default: 30m) 76 | [$AZURE_SERVICEDISCOVERY_CACHE] 77 | --azure.resource-tag= Azure Resource tags (space delimiter) (default: owner) [$AZURE_RESOURCE_TAG] 78 | --metrics.template= Template for metric name (default: {name}) [$METRIC_TEMPLATE] 79 | --metrics.help= Metric help (with template support) (default: Azure monitor insight metric) [$METRIC_HELP] 80 | --metrics.dimensions.lowercase Lowercase dimension values [$METRIC_DIMENSIONS_LOWERCASE] 81 | --concurrency.subscription= Concurrent subscription fetches (default: 5) [$CONCURRENCY_SUBSCRIPTION] 82 | --concurrency.subscription.resource= Concurrent requests per resource (inside subscription requests) (default: 10) [$CONCURRENCY_SUBSCRIPTION_RESOURCE] 83 | --enable-caching Enable internal caching [$ENABLE_CACHING] 84 | --server.bind= Server address (default: :8080) [$SERVER_BIND] 85 | --server.timeout.read= Server read timeout (default: 5s) [$SERVER_TIMEOUT_READ] 86 | --server.timeout.write= Server write timeout (default: 10s) [$SERVER_TIMEOUT_WRITE] 87 | 88 | Help Options: 89 | -h, --help Show this help message 90 | ``` 91 | 92 | for Azure API authentication (using ENV vars) see following documentations: 93 | - https://github.com/webdevops/go-common/blob/main/azuresdk/README.md 94 | - https://docs.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication 95 | 96 | ## How to test 97 | 98 | Enable the webui (`--development.webui`) to get a basic web frontend to query the exporter which helps you to find 99 | the right settings for your configuration. 100 | 101 | webui is available under url `/query` 102 | 103 | ## Metrics 104 | 105 | | Metric | Description | 106 | |------------------------------------------|-------------------------------------------------------------------------------------------------| 107 | | `azurerm_stats_metric_collecttime` | General exporter stats | 108 | | `azurerm_stats_metric_requests` | Counter of resource metric requests with result (error, success) | 109 | | `azurerm_resource_metric` (customizable) | Resource metrics exported by probes (can be changed using `name` parameter and template system) | 110 | | `azurerm_api_ratelimit` | Azure ratelimit metrics (only on /metrics, resets after query) | 111 | | `azurerm_api_request_*` | Azure request count and latency as histogram | 112 | 113 | ### ResourceTags handling 114 | 115 | see [armclient tagmanager documentation](https://github.com/webdevops/go-common/blob/main/azuresdk/README.md#tag-manager) 116 | 117 | ### AzureTracing metrics 118 | 119 | see [armclient tracing documentation](https://github.com/webdevops/go-common/blob/main/azuresdk/README.md#azuretracing-metrics) 120 | | 121 | 122 | ### Metric name and help template system 123 | 124 | (with 21.5.3 and later) 125 | 126 | By default Azure monitor metrics are generated with the name specified in the request (see parameter `name`). 127 | This can be modified via environment variable `$METRIC_TEMPLATE` or as request parameter `template`. 128 | 129 | HINT: Used templates are removed from labels! 130 | 131 | Metric name recommendation: `{name}_{metric}_{aggregation}_{unit}` 132 | 133 | Help recommendation: `Azure metrics for {metric} with aggregation {aggregation} as {unit}` 134 | 135 | Following templates are available: 136 | 137 | | Template | Description | 138 | |-----------------|-------------------------------------------------------------------------------------------| 139 | | `{name}` | Name of template specified by request parameter `name` | 140 | | `{type}` | The ResourceType or MetricNamespace specified in the request (not applicable to all APIs) | 141 | | `{metric}` | Name of Azure monitor metric | 142 | | `{dimension}` | Dimension value of Azure monitor metric (if dimension is used) | 143 | | `{unit}` | Unit name of Azure monitor metric (eg `count`, `percent`, ...) | 144 | | `{aggregation}` | Aggregation of Azure monitor metric (eg `total`, `average`) | 145 | | `{interval}` | Interval of requested Azure monitor metric | 146 | | `{timespan}` | Timespan of requested Azure monitor metric | 147 | 148 | #### default template 149 | 150 | Prometheus config: 151 | ```yaml 152 | - job_name: azure-metrics-keyvault 153 | scrape_interval: 1m 154 | metrics_path: /probe/metrics/list 155 | params: 156 | name: ["azure_metric_keyvault"] 157 | subscription: 158 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 159 | filter: ["resourceType eq 'Microsoft.KeyVault/vaults'"] 160 | metric: 161 | - Availability 162 | - ServiceApiHit 163 | - ServiceApiLatency 164 | interval: ["PT15M"] 165 | timespan: ["PT15M"] 166 | aggregation: 167 | - average 168 | - total 169 | static_configs: 170 | - targets: ["azure-metrics:8080"] 171 | ``` 172 | 173 | generated metrics: 174 | ``` 175 | # HELP azure_metric_keyvault Azure monitor insight metric 176 | # TYPE azure_metric_keyvault gauge 177 | azure_metric_keyvault{aggregation="average",dimension="",interval="PT12H",metric="Availability",resourceID="/subscriptions/...",timespan="PT12H",unit="Percent"} 100 178 | azure_metric_keyvault{aggregation="average",dimension="",interval="PT12H",metric="Availability",resourceID="/subscriptions/...",timespan="PT12H",unit="Percent"} 100 179 | azure_metric_keyvault{aggregation="average",dimension="",interval="PT12H",metric="ServiceApiHit",resourceID="/subscriptions/...",timespan="PT12H",unit="Count"} 0 180 | azure_metric_keyvault{aggregation="average",dimension="",interval="PT12H",metric="ServiceApiHit",resourceID="/subscriptions/...",timespan="PT12H",unit="Count"} 0 181 | azure_metric_keyvault{aggregation="total",dimension="",interval="PT12H",metric="ServiceApiHit",resourceID="/subscriptions/...",timespan="PT12H",unit="Count"} 0 182 | azure_metric_keyvault{aggregation="total",dimension="",interval="PT12H",metric="ServiceApiHit",resourceID="/subscriptions/...",timespan="PT12H",unit="Count"} 0 183 | # HELP azurerm_ratelimit Azure ResourceManager ratelimit 184 | # TYPE azurerm_ratelimit gauge 185 | azurerm_ratelimit{scope="subscription",subscriptionID="...",type="read"} 11997 186 | ``` 187 | 188 | 189 | #### template `{name}_{metric}_{unit}` 190 | 191 | Prometheus config: 192 | ```yaml 193 | - job_name: azure-metrics-keyvault 194 | scrape_interval: 1m 195 | metrics_path: /probe/metrics/list 196 | params: 197 | name: ["azure_metric_keyvault"] 198 | template: ["{name}_{metric}_{unit}"] 199 | help: ["Custom help with {metric}"] 200 | subscription: 201 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 202 | filter: ["resourceType eq 'Microsoft.KeyVault/vaults'"] 203 | metric: 204 | - Availability 205 | - ServiceApiHit 206 | - ServiceApiLatency 207 | interval: ["PT15M"] 208 | timespan: ["PT15M"] 209 | aggregation: 210 | - average 211 | - total 212 | static_configs: 213 | - targets: ["azure-metrics:8080"] 214 | ``` 215 | 216 | generated metrics: 217 | ``` 218 | # HELP azure_metric_keyvault_availability_percent Custom help with availability 219 | # TYPE azure_metric_keyvault_availability_percent gauge 220 | azure_metric_keyvault_availability_percent{aggregation="average",dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 100 221 | azure_metric_keyvault_availability_percent{aggregation="average",dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 100 222 | 223 | # HELP azure_metric_keyvault_serviceapihit_count Custom help with serviceapihit 224 | # TYPE azure_metric_keyvault_serviceapihit_count gauge 225 | azure_metric_keyvault_serviceapihit_count{aggregation="average",dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 0 226 | azure_metric_keyvault_serviceapihit_count{aggregation="average",dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 0 227 | azure_metric_keyvault_serviceapihit_count{aggregation="total",dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 0 228 | azure_metric_keyvault_serviceapihit_count{aggregation="total",dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 0 229 | 230 | # HELP azurerm_ratelimit Azure ResourceManager ratelimit 231 | # TYPE azurerm_ratelimit gauge 232 | azurerm_ratelimit{scope="subscription",subscriptionID="...",type="read"} 11996 233 | ``` 234 | 235 | #### template `{name}_{metric}_{aggregation}_{unit}` 236 | 237 | Prometheus config: 238 | ```yaml 239 | - job_name: azure-metrics-keyvault 240 | scrape_interval: 1m 241 | metrics_path: /probe/metrics/list 242 | params: 243 | name: ["azure_metric_keyvault"] 244 | template: ["{name}_{metric}_{aggregation}_{unit}"] 245 | subscription: 246 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 247 | filter: ["resourceType eq 'Microsoft.KeyVault/vaults'"] 248 | metric: 249 | - Availability 250 | - ServiceApiHit 251 | - ServiceApiLatency 252 | interval: ["PT15M"] 253 | timespan: ["PT15M"] 254 | aggregation: 255 | - average 256 | - total 257 | static_configs: 258 | - targets: ["azure-metrics:8080"] 259 | ``` 260 | 261 | generated metrics: 262 | ``` 263 | # HELP azure_metric_keyvault_availability_average_percent Azure monitor insight metric 264 | # TYPE azure_metric_keyvault_availability_average_percent gauge 265 | azure_metric_keyvault_availability_average_percent{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 100 266 | azure_metric_keyvault_availability_average_percent{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 100 267 | # HELP azure_metric_keyvault_availability_total_percent Azure monitor insight metric 268 | # TYPE azure_metric_keyvault_availability_total_percent gauge 269 | azure_metric_keyvault_availability_total_percent{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 9 270 | # HELP azure_metric_keyvault_serviceapihit_average_count Azure monitor insight metric 271 | # TYPE azure_metric_keyvault_serviceapihit_average_count gauge 272 | azure_metric_keyvault_serviceapihit_average_count{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 0 273 | azure_metric_keyvault_serviceapihit_average_count{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 1 274 | # HELP azure_metric_keyvault_serviceapihit_total_count Azure monitor insight metric 275 | # TYPE azure_metric_keyvault_serviceapihit_total_count gauge 276 | azure_metric_keyvault_serviceapihit_total_count{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 0 277 | azure_metric_keyvault_serviceapihit_total_count{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 9 278 | # HELP azure_metric_keyvault_serviceapilatency_average_milliseconds Azure monitor insight metric 279 | # TYPE azure_metric_keyvault_serviceapilatency_average_milliseconds gauge 280 | azure_metric_keyvault_serviceapilatency_average_milliseconds{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 38.666666666666664 281 | # HELP azure_metric_keyvault_serviceapilatency_total_milliseconds Azure monitor insight metric 282 | # TYPE azure_metric_keyvault_serviceapilatency_total_milliseconds gauge 283 | azure_metric_keyvault_serviceapilatency_total_milliseconds{dimension="",interval="PT12H",resourceID="/subscriptions/...",timespan="PT12H"} 348 284 | # HELP azurerm_ratelimit Azure ResourceManager ratelimit 285 | # TYPE azurerm_ratelimit gauge 286 | azurerm_ratelimit{scope="subscription",subscriptionID="...",type="read"} 11999 287 | ``` 288 | 289 | ## HTTP Endpoints 290 | 291 | | Endpoint | Description | 292 | |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------| 293 | | `/metrics` | Default prometheus golang metrics | 294 | | `/probe/metrics` | Probe metrics by subscription and region, split by resource (one query per subscription and region; see `azurerm_resource_metric`) | 295 | | `/probe/metrics/resource` | Probe metrics for one resource (one query per resource; see `azurerm_resource_metric`) | 296 | | `/probe/metrics/list` | Probe metrics for list of resources (sone query per resource; see `azurerm_resource_metric`) | 297 | | `/probe/metrics/scrape` | Probe metrics for list of resources and config on resource by tag name (one query per resource; see `azurerm_resource_metric`) | 298 | | `/probe/metrics/resourcegraph` | Probe metrics for list of resources based on a kusto query and the resource graph API (one query per resource) | 299 | 300 | ### /probe/metrics parameters 301 | 302 | one metric request per subscription and region 303 | 304 | | GET parameter | Default | Required | Multiple | Description | 305 | |----------------------|---------------------------|----------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------| 306 | | `subscription` | | **yes** | **yes** | Azure Subscription ID | 307 | | `region` | | no | **yes** | Azure Regions (eg. `westeurope`, `northeurope`). If omit, ResourceGrapth will be used to discover regions | 308 | | `resourceType` | | **yes** | no | Azure Resource type | 309 | | `timespan` | `PT1M` | no | no | Metric timespan | 310 | | `interval` | | no | no | Metric timespan | 311 | | `metricNamespace` | | no | no | Metric namespace | 312 | | `metric` | | no | **yes** | Metric name | 313 | | `aggregation` | | no | **yes** | Metric aggregation (`minimum`, `maximum`, `average`, `total`, `count`, multiple possible separated with `,`) | 314 | | `name` | `azurerm_resource_metric` | no | no | Prometheus metric name | 315 | | `metricFilter` | | no | no | Prometheus metric filter (dimension support; supports only 2 filters in subscription query mode as the first filter is used to split by resource id) | 316 | | `metricTop` | | no | no | Prometheus metric dimension count (dimension support) | 317 | | `metricOrderBy` | | no | no | Prometheus metric order by (dimension support) | 318 | | `validateDimensions` | `true` | no | no | When set to false, invalid filter parameter values will be ignored. | 319 | | `cache` | (same as timespan) | no | no | Use of internal metrics caching | 320 | | `template` | set to `$METRIC_TEMPLATE` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 321 | | `help` | set to `$METRIC_HELP` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 322 | 323 | *Hint: Multiple values can be specified multiple times or with a comma in a single value.* 324 | 325 | ### /probe/metrics/resource parameters 326 | 327 | metrics are requested per resource in chunks of 20 metric names (35 metric names = 2 requests per resource) 328 | 329 | | GET parameter | Default | Required | Multiple | Description | 330 | |----------------------|---------------------------|----------|----------|--------------------------------------------------------------------------------------------------------------| 331 | | `subscription` | | **yes** | **yes** | Azure Subscription ID | 332 | | `target` | | **yes** | **yes** | Azure Resource URI | 333 | | `timespan` | `PT1M` | no | no | Metric timespan | 334 | | `interval` | | no | no | Metric timespan | 335 | | `metricNamespace` | | no | **yes** | Metric namespace | 336 | | `metric` | | no | **yes** | Metric name | 337 | | `aggregation` | | no | **yes** | Metric aggregation (`minimum`, `maximum`, `average`, `total`, `count`, multiple possible separated with `,`) | 338 | | `name` | `azurerm_resource_metric` | no | no | Prometheus metric name | 339 | | `metricFilter` | | no | no | Prometheus metric filter (dimension support) | 340 | | `metricTop` | | no | no | Prometheus metric dimension count (dimension support) | 341 | | `metricOrderBy` | | no | no | Prometheus metric order by (dimension support) | 342 | | `validateDimensions` | `true` | no | no | When set to false, invalid filter parameter values will be ignored. | 343 | | `cache` | (same as timespan) | no | no | Use of internal metrics caching | 344 | | `template` | set to `$METRIC_TEMPLATE` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 345 | | `help` | set to `$METRIC_HELP` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 346 | 347 | *Hint: Multiple values can be specified multiple times or with a comma in a single value.* 348 | 349 | ### /probe/metrics/list parameters 350 | 351 | metrics are requested per resource in chunks of 20 metric names (35 metric names = 2 requests per resource) 352 | 353 | HINT: service discovery information is cached for duration set by `$AZURE_SERVICEDISCOVERY_CACHE` (set to `0` to disable) 354 | 355 | | GET parameter | Default | Required | Multiple | Description | 356 | |----------------------------|---------------------------|----------|----------|--------------------------------------------------------------------------------------------------------------| 357 | | `subscription` | | **yes** | **yes** | Azure Subscription ID (or multiple separate by comma) | 358 | | `resourceType` or `filter` | | **yes** | no | Azure Resource type or filter query (https://docs.microsoft.com/en-us/rest/api/resources/resources/list) | 359 | | `timespan` | `PT1M` | no | no | Metric timespan | 360 | | `interval` | | no | no | Metric timespan | 361 | | `metricNamespace` | | no | **yes** | Metric namespace | 362 | | `metric` | | no | **yes** | Metric name | 363 | | `aggregation` | | no | **yes** | Metric aggregation (`minimum`, `maximum`, `average`, `total`, `count`, multiple possible separated with `,`) | 364 | | `name` | `azurerm_resource_metric` | no | no | Prometheus metric name | 365 | | `metricFilter` | | no | no | Prometheus metric filter (dimension support) | 366 | | `metricTop` | | no | no | Prometheus metric dimension count (dimension support) | 367 | | `metricOrderBy` | | no | no | Prometheus metric order by (dimension support) | 368 | | `validateDimensions` | `true` | no | no | When set to false, invalid filter parameter values will be ignored. | 369 | | `cache` | (same as timespan) | no | no | Use of internal metrics caching | 370 | | `template` | set to `$METRIC_TEMPLATE` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 371 | | `help` | set to `$METRIC_HELP` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 372 | 373 | *Hint: Multiple values can be specified multiple times or with a comma in a single value.* 374 | 375 | ### /probe/metrics/scrape parameters 376 | 377 | HINT: service discovery information is cached for duration set by `$AZURE_SERVICEDISCOVERY_CACHE` (set to `0` to disable) 378 | 379 | | GET parameter | Default | Required | Multiple | Description | 380 | |----------------------------|---------------------------|----------|----------|----------------------------------------------------------------------------------------------------------| 381 | | `subscription` | | **yes** | **yes** | Azure Subscription ID (or multiple separate by comma) | 382 | | `resourceType` or `filter` | | **yes** | no | Azure Resource type or filter query (https://docs.microsoft.com/en-us/rest/api/resources/resources/list) | 383 | | `metricTagName` | | **yes** | no | Resource tag name for getting "metrics" list | 384 | | `aggregationTagName` | | **yes** | no | Resource tag name for getting "aggregations" list | 385 | | `timespan` | `PT1M` | no | no | Metric timespan | 386 | | `interval` | | no | no | Metric timespan | 387 | | `metricNamespace` | | no | **yes** | Metric namespace | 388 | | `metric` | | no | **yes** | Metric name | 389 | | `aggregation` | | no | **yes** | Metric aggregation (`minimum`, `maximum`, `average`, `total`, multiple possible separated with `,`) | 390 | | `name` | `azurerm_resource_metric` | no | no | Prometheus metric name | 391 | | `metricFilter` | | no | no | Prometheus metric filter (dimension support) | 392 | | `metricTop` | | no | no | Prometheus metric dimension count (integer, dimension support) | 393 | | `metricOrderBy` | | no | no | Prometheus metric order by (dimension support) | 394 | | `validateDimensions` | `true` | no | no | When set to false, invalid filter parameter values will be ignored. | 395 | | `cache` | (same as timespan) | no | no | Use of internal metrics caching | 396 | | `template` | set to `$METRIC_TEMPLATE` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 397 | | `help` | set to `$METRIC_HELP` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 398 | 399 | *Hint: Multiple values can be specified multiple times or with a comma in a single value.* 400 | 401 | 402 | ### /probe/metrics/resourcegraph parameters 403 | 404 | This endpoint is using Azure ResoruceGraph API for servicediscovery (with 21.9.0 and later) 405 | 406 | metrics are requested per resource in chunks of 20 metric names (35 metric names = 2 requests per resource) 407 | 408 | HINT: service discovery information is cached for duration set by `$AZURE_SERVICEDISCOVERY_CACHE` (set to `0` to disable) 409 | 410 | | GET parameter | Default | Required | Multiple | Description | 411 | |----------------------|---------------------------|----------|----------|--------------------------------------------------------------------------------------------------------------| 412 | | `subscription` | | **yes** | **yes** | Azure Subscription ID (or multiple separate by comma) | 413 | | `resourceType` | | **yes** | no | Azure Resource type | 414 | | `filter` | | no | no | Additional Kusto query part (eg. `where id contains "/xzy/"`) | 415 | | `timespan` | `PT1M` | no | no | Metric timespan | 416 | | `interval` | | no | no | Metric timespan | 417 | | `metricNamespace` | | no | **yes** | Metric namespace | 418 | | `metric` | | no | **yes** | Metric name | 419 | | `aggregation` | | no | **yes** | Metric aggregation (`minimum`, `maximum`, `average`, `total`, `count`, multiple possible separated with `,`) | 420 | | `name` | `azurerm_resource_metric` | no | no | Prometheus metric name | 421 | | `metricFilter` | | no | no | Prometheus metric filter (dimension support) | 422 | | `metricTop` | | no | no | Prometheus metric dimension count (dimension support) | 423 | | `metricOrderBy` | | no | no | Prometheus metric order by (dimension support) | 424 | | `validateDimensions` | `true` | no | no | When set to false, invalid filter parameter values will be ignored. | 425 | | `cache` | (same as timespan) | no | no | Use of internal metrics caching | 426 | | `template` | set to `$METRIC_TEMPLATE` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 427 | | `help` | set to `$METRIC_HELP` | no | no | see [metric name and help template system](#metric-name-and-help-template-system) | 428 | 429 | *Hint: Multiple values can be specified multiple times or with a comma in a single value.* 430 | 431 | ## Prometheus configuration examples 432 | 433 | ### Redis 434 | 435 | using target (single instances): 436 | 437 | ```yaml 438 | - job_name: azure-metrics-redis 439 | scrape_interval: 1m 440 | metrics_path: /probe/metrics/resource 441 | params: 442 | name: ["my_own_metric_name"] 443 | subscription: 444 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 445 | target: 446 | - /subscriptions/.../resourceGroups/.../providers/Microsoft.Cache/Redis/... 447 | - /subscriptions/.../resourceGroups/.../providers/Microsoft.Cache/Redis/... 448 | - /subscriptions/.../resourceGroups/.../providers/Microsoft.Cache/Redis/... 449 | - /subscriptions/.../resourceGroups/.../providers/Microsoft.Cache/Redis/... 450 | metric: 451 | - connectedclients 452 | - totalcommandsprocessed 453 | - cachehits 454 | - cachemisses 455 | - getcommands 456 | - setcommands 457 | - operationsPerSecond 458 | - evictedkeys 459 | - totalkeys 460 | - expiredkeys 461 | - usedmemory 462 | - usedmemorypercentage 463 | - usedmemoryRss 464 | - serverLoad 465 | - cacheWrite 466 | - cacheRead 467 | - percentProcessorTime 468 | - cacheLatency 469 | - errors 470 | interval: ["PT1M"] 471 | timespan: ["PT1M"] 472 | aggregation: 473 | - average 474 | - total 475 | static_configs: 476 | - targets: ["azure-metrics:8080"] 477 | ``` 478 | 479 | using ServiceDiscovery: 480 | ```yaml 481 | - job_name: azure-metrics-redis 482 | scrape_interval: 1m 483 | metrics_path: /probe/metrics/list 484 | params: 485 | name: ["my_own_metric_name"] 486 | subscription: 487 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 488 | resourceType: ["Microsoft.Cache/Redis"] 489 | metric: 490 | - connectedclients 491 | - totalcommandsprocessed 492 | - cachehits 493 | - cachemisses 494 | - getcommands 495 | - setcommands 496 | - operationsPerSecond 497 | - evictedkeys 498 | - totalkeys 499 | - expiredkeys 500 | - usedmemory 501 | - usedmemorypercentage 502 | - usedmemoryRss 503 | - serverLoad 504 | - cacheWrite 505 | - cacheRead 506 | - percentProcessorTime 507 | - cacheLatency 508 | - errors 509 | interval: ["PT1M"] 510 | timespan: ["PT1M"] 511 | aggregation: 512 | - average 513 | - total 514 | static_configs: 515 | - targets: ["azure-metrics:8080"] 516 | ``` 517 | 518 | using ServiceDiscovery with custom resource filter query: 519 | ```yaml 520 | - job_name: azure-metrics-redis 521 | scrape_interval: 1m 522 | metrics_path: /probe/metrics/list 523 | params: 524 | name: ["my_own_metric_name"] 525 | subscription: 526 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 527 | filter: ["resourceType eq 'Microsoft.Cache/Redis'"] 528 | metric: 529 | - connectedclients 530 | - totalcommandsprocessed 531 | - cachehits 532 | - cachemisses 533 | - getcommands 534 | - setcommands 535 | - operationsPerSecond 536 | - evictedkeys 537 | - totalkeys 538 | - expiredkeys 539 | - usedmemory 540 | - usedmemorypercentage 541 | - usedmemoryRss 542 | - serverLoad 543 | - cacheWrite 544 | - cacheRead 545 | - percentProcessorTime 546 | - cacheLatency 547 | - errors 548 | interval: ["PT1M"] 549 | timespan: ["PT1M"] 550 | aggregation: 551 | - average 552 | - total 553 | static_configs: 554 | - targets: ["azure-metrics:8080"] 555 | ``` 556 | ### VirtualNetworkGateways 557 | 558 | ```yaml 559 | - job_name: azure-metrics-virtualNetworkGateways 560 | scrape_interval: 1m 561 | metrics_path: /probe/metrics/list 562 | params: 563 | name: ["my_own_metric_name"] 564 | subscription: 565 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 566 | resourceType: ["Microsoft.Network/virtualNetworkGateways"] 567 | metric: 568 | - AverageBandwidth 569 | - P2SBandwidth 570 | - P2SConnectionCount 571 | - TunnelAverageBandwidth 572 | - TunnelEgressBytes 573 | - TunnelIngressBytes 574 | - TunnelEgressPackets 575 | - TunnelIngressPackets 576 | - TunnelEgressPacketDropTSMismatch 577 | - TunnelIngressPacketDropTSMismatch 578 | interval: ["PT5M"] 579 | timespan: ["PT5M"] 580 | aggregation: 581 | - average 582 | - total 583 | static_configs: 584 | - targets: ["azure-metrics:8080"] 585 | ``` 586 | 587 | ### virtualNetworkGateway connections (dimension support) 588 | 589 | Virtual Gateway connection metrics (dimension support) 590 | ```yaml 591 | - job_name: azure-metrics-virtualNetworkGateways-connections 592 | scrape_interval: 1m 593 | metrics_path: /probe/metrics/list 594 | params: 595 | name: ["my_own_metric_name"] 596 | subscription: 597 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 598 | resourceType: ["Microsoft.Network/virtualNetworkGateways"] 599 | metric: 600 | - TunnelAverageBandwidth 601 | - TunnelEgressBytes 602 | - TunnelIngressBytes 603 | - TunnelEgressPackets 604 | - TunnelIngressPackets 605 | - TunnelEgressPacketDropTSMismatch 606 | - TunnelIngressPacketDropTSMismatch 607 | interval: ["PT5M"] 608 | timespan: ["PT5M"] 609 | aggregation: 610 | - average 611 | - total 612 | # by connection (dimension support) 613 | metricFilter: ["ConnectionName eq '*'"] 614 | metricTop: ["10"] 615 | static_configs: 616 | - targets: ["azure-metrics:8080"] 617 | ``` 618 | 619 | ### StorageAccount (metric namespace and dimension support) 620 | 621 | ```yaml 622 | - job_name: azure-metrics-virtualNetworkGateways-connections 623 | scrape_interval: 1m 624 | metrics_path: /probe/metrics/list 625 | params: 626 | name: ["my_own_metric_name"] 627 | subscription: 628 | - xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 629 | resourceType: ["Microsoft.Storage/storageAccounts"] 630 | metricNamespace: ["Microsoft.Storage/storageAccounts/blobServices"] 631 | metric: 632 | - BlobCapacity 633 | interval: ["PT1H"] 634 | timespan: ["PT1H"] 635 | aggregation: 636 | - average 637 | - count 638 | # by blobtype (dimension support) 639 | metricFilter: ["BlobType eq '*'"] 640 | metricTop: ["10"] 641 | static_configs: 642 | - targets: ["azure-metrics:8080"] 643 | ``` 644 | 645 | In these examples all metrics are published with metric name `my_own_metric_name`. 646 | 647 | The [List of supported metrics](https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported) is available in the Microsoft Azure docs. 648 | 649 | ### Development and testing query webui 650 | 651 | azure-metrics-exporter provides a query webui at `http://url-to-exporter/query` where you can 652 | test different query settings and endpoints. the query webui also generates an example prometheus scrape_config. 653 | -------------------------------------------------------------------------------- /common.logger.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "go.uber.org/zap" 5 | "go.uber.org/zap/zapcore" 6 | ) 7 | 8 | var ( 9 | logger *zap.SugaredLogger 10 | ) 11 | 12 | func initLogger() *zap.SugaredLogger { 13 | var config zap.Config 14 | if Opts.Logger.Development { 15 | config = zap.NewDevelopmentConfig() 16 | config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder 17 | } else { 18 | config = zap.NewProductionConfig() 19 | } 20 | 21 | config.Encoding = "console" 22 | config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder 23 | 24 | // debug level 25 | if Opts.Logger.Debug { 26 | config.Level = zap.NewAtomicLevelAt(zapcore.DebugLevel) 27 | } 28 | 29 | // json log format 30 | if Opts.Logger.Json { 31 | config.Encoding = "json" 32 | 33 | // if running in containers, logs already enriched with timestamp by the container runtime 34 | config.EncoderConfig.TimeKey = "" 35 | } 36 | 37 | // build logger 38 | log, err := config.Build() 39 | if err != nil { 40 | panic(err) 41 | } 42 | 43 | logger = log.Sugar() 44 | 45 | return logger 46 | } 47 | -------------------------------------------------------------------------------- /common.system.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/webdevops/go-common/system" 5 | ) 6 | 7 | func initSystem() { 8 | system.AutoProcMemLimit(logger) 9 | } 10 | -------------------------------------------------------------------------------- /config/consts.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | const ( 4 | MetricsUrl = "/metrics" 5 | 6 | ProbeMetricsResourceUrl = "/probe/metrics/resource" 7 | ProbeMetricsResourceTimeoutDefault = 10 8 | 9 | ProbeMetricsListUrl = "/probe/metrics/list" 10 | ProbeMetricsListTimeoutDefault = 120 11 | 12 | ProbeMetricsSubscriptionUrl = "/probe/metrics" 13 | ProbeMetricsSubscriptionTimeoutDefault = 120 14 | 15 | ProbeMetricsScrapeUrl = "/probe/metrics/scrape" 16 | ProbeMetricsScrapeTimeoutDefault = 120 17 | 18 | ProbeMetricsResourceGraphUrl = "/probe/metrics/resourcegraph" 19 | ProbeMetricsResourceGraphTimeoutDefault = 120 20 | ) 21 | -------------------------------------------------------------------------------- /config/opts.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | ) 7 | 8 | type ( 9 | Opts struct { 10 | // logger 11 | Logger struct { 12 | Debug bool `long:"log.debug" env:"LOG_DEBUG" description:"debug mode"` 13 | Development bool `long:"log.devel" env:"LOG_DEVEL" description:"development mode"` 14 | Json bool `long:"log.json" env:"LOG_JSON" description:"Switch log output to json format"` 15 | } 16 | 17 | // azure 18 | Azure struct { 19 | Environment *string `long:"azure-environment" env:"AZURE_ENVIRONMENT" description:"Azure environment name" default:"AZUREPUBLICCLOUD"` 20 | AdResourceUrl *string `long:"azure-ad-resource-url" env:"AZURE_AD_RESOURCE" description:"Specifies the AAD resource ID to use. If not set, it defaults to ResourceManagerEndpoint for operations with Azure Resource Manager"` 21 | ServiceDiscovery struct { 22 | CacheDuration *time.Duration `long:"azure.servicediscovery.cache" env:"AZURE_SERVICEDISCOVERY_CACHE" description:"Duration for caching Azure ServiceDiscovery of workspaces to reduce API calls (time.Duration)" default:"30m"` 23 | } 24 | ResourceTags []string `long:"azure.resource-tag" env:"AZURE_RESOURCE_TAG" env-delim:" " description:"Azure Resource tags (space delimiter)" default:"owner"` 25 | } 26 | 27 | Metrics struct { 28 | Template string `long:"metrics.template" env:"METRIC_TEMPLATE" description:"Template for metric name" default:"{name}"` 29 | Help string `long:"metrics.help" env:"METRIC_HELP" description:"Metric help (with template support)" default:"Azure monitor insight metric"` 30 | Dimensions struct { 31 | Lowercase bool `long:"metrics.dimensions.lowercase" env:"METRIC_DIMENSIONS_LOWERCASE" description:"Lowercase dimension values"` 32 | } 33 | } 34 | 35 | // Prober settings 36 | Prober struct { 37 | ConcurrencySubscription int `long:"concurrency.subscription" env:"CONCURRENCY_SUBSCRIPTION" description:"Concurrent subscription fetches" default:"5"` 38 | ConcurrencySubscriptionResource int `long:"concurrency.subscription.resource" env:"CONCURRENCY_SUBSCRIPTION_RESOURCE" description:"Concurrent requests per resource (inside subscription requests)" default:"10"` 39 | Cache bool `long:"enable-caching" env:"ENABLE_CACHING" description:"Enable internal caching"` 40 | } 41 | 42 | // general options 43 | Server struct { 44 | // general options 45 | Bind string `long:"server.bind" env:"SERVER_BIND" description:"Server address" default:":8080"` 46 | ReadTimeout time.Duration `long:"server.timeout.read" env:"SERVER_TIMEOUT_READ" description:"Server read timeout" default:"5s"` 47 | WriteTimeout time.Duration `long:"server.timeout.write" env:"SERVER_TIMEOUT_WRITE" description:"Server write timeout" default:"10s"` 48 | } 49 | } 50 | ) 51 | 52 | func (o *Opts) GetJson() []byte { 53 | jsonBytes, err := json.Marshal(o) 54 | if err != nil { 55 | panic(err) 56 | } 57 | return jsonBytes 58 | } 59 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/webdevops/azure-metrics-exporter 2 | 3 | go 1.24.0 4 | 5 | toolchain go1.24.2 6 | 7 | require ( 8 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 9 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 10 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0 11 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 12 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 13 | github.com/Azure/go-autorest/autorest v0.11.30 14 | github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 15 | github.com/google/uuid v1.6.0 16 | github.com/jessevdk/go-flags v1.6.1 17 | github.com/patrickmn/go-cache v2.1.0+incompatible 18 | github.com/prometheus/client_golang v1.22.0 19 | github.com/remeh/sizedwaitgroup v1.0.0 20 | github.com/webdevops/go-common v0.0.0-20250501164923-7cab87d11d0f 21 | go.uber.org/zap v1.27.0 22 | ) 23 | 24 | require ( 25 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 // indirect 26 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect 27 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 28 | github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect 29 | github.com/Azure/go-autorest/autorest/date v0.3.1 // indirect 30 | github.com/Azure/go-autorest/logger v0.2.2 // indirect 31 | github.com/Azure/go-autorest/tracing v0.6.1 // indirect 32 | github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect 33 | github.com/KimMachineGun/automemlimit v0.7.1 // indirect 34 | github.com/beorn7/perks v1.0.1 // indirect 35 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 36 | github.com/dustin/go-humanize v1.0.1 // indirect 37 | github.com/go-logr/logr v1.4.2 // indirect 38 | github.com/golang-jwt/jwt/v4 v4.5.2 // indirect 39 | github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 40 | github.com/kylelemons/godebug v1.1.0 // indirect 41 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 42 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 43 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect 44 | github.com/prometheus/client_model v0.6.2 // indirect 45 | github.com/prometheus/common v0.63.0 // indirect 46 | github.com/prometheus/procfs v0.16.1 // indirect 47 | go.uber.org/automaxprocs v1.6.0 // indirect 48 | go.uber.org/multierr v1.11.0 // indirect 49 | go.uber.org/zap/exp v0.3.0 // indirect 50 | golang.org/x/crypto v0.37.0 // indirect 51 | golang.org/x/net v0.39.0 // indirect 52 | golang.org/x/sys v0.32.0 // indirect 53 | golang.org/x/text v0.24.0 // indirect 54 | google.golang.org/protobuf v1.36.6 // indirect 55 | k8s.io/apimachinery v0.33.0 // indirect 56 | k8s.io/klog/v2 v2.130.1 // indirect 57 | k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e // indirect 58 | ) 59 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= 2 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= 3 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= 4 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= 5 | github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= 6 | github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= 7 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= 8 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= 9 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= 10 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= 11 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= 12 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= 13 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg= 14 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0/go.mod h1:jj6P8ybImR+5topJ+eH6fgcemSFBmU6/6bFF8KkwuDI= 15 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0 h1:zLzoX5+W2l95UJoVwiyNS4dX8vHyQ6x2xRLoBBL9wMk= 16 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.9.0/go.mod h1:wVEOJfGTj0oPAUGA1JuRAvz/lxXQsWW16axmHPP47Bk= 17 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= 18 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= 19 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= 20 | github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= 21 | github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= 22 | github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= 23 | github.com/Azure/go-autorest/autorest v0.11.30 h1:iaZ1RGz/ALZtN5eq4Nr1SOFSlf2E4pDI3Tcsl+dZPVE= 24 | github.com/Azure/go-autorest/autorest v0.11.30/go.mod h1:t1kpPIOpIVX7annvothKvb0stsrXa37i7b+xpmBW8Fs= 25 | github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= 26 | github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= 27 | github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= 28 | github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= 29 | github.com/Azure/go-autorest/autorest/date v0.3.1 h1:o9Z8Jyt+VJJTCZ/UORishuHOusBwolhjokt9s5k8I4w= 30 | github.com/Azure/go-autorest/autorest/date v0.3.1/go.mod h1:Dz/RDmXlfiFFS/eW+b/xMUSFs1tboPVy6UjgADToWDM= 31 | github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= 32 | github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= 33 | github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= 34 | github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= 35 | github.com/Azure/go-autorest/logger v0.2.2 h1:hYqBsEBywrrOSW24kkOCXRcKfKhK76OzLTfF+MYDE2o= 36 | github.com/Azure/go-autorest/logger v0.2.2/go.mod h1:I5fg9K52o+iuydlWfa9T5K6WFos9XYr9dYTFzpqgibw= 37 | github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= 38 | github.com/Azure/go-autorest/tracing v0.6.1 h1:YUMSrC/CeD1ZnnXcNYU4a/fzsO35u2Fsful9L/2nyR0= 39 | github.com/Azure/go-autorest/tracing v0.6.1/go.mod h1:/3EgjbsjraOqiicERAeu3m7/z0x1TzjQGAwDrJrXGkc= 40 | github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= 41 | github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= 42 | github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= 43 | github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= 44 | github.com/KimMachineGun/automemlimit v0.7.1 h1:QcG/0iCOLChjfUweIMC3YL5Xy9C3VBeNmCZHrZfJMBw= 45 | github.com/KimMachineGun/automemlimit v0.7.1/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= 46 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 47 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 48 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 49 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 50 | github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 h1:o64h9XF42kVEUuhuer2ehqrlX8rZmvQSU0+Vpj1rF6Q= 51 | github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61/go.mod h1:Rp8e0DCtEKwXFOC6JPJQVTz8tuGoGvw6Xfexggh/ed0= 52 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 53 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 54 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 55 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 56 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 57 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 58 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 59 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 60 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 61 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 62 | github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= 63 | github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= 64 | github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= 65 | github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= 66 | github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= 67 | github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= 68 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 69 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 70 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 71 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 72 | github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= 73 | github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= 74 | github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= 75 | github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= 76 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 77 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 78 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 79 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 80 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 81 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 82 | github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= 83 | github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= 84 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= 85 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= 86 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= 87 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= 88 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 89 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 90 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 91 | github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= 92 | github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= 93 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 94 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 95 | github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= 96 | github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= 97 | github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= 98 | github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= 99 | github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= 100 | github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= 101 | github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= 102 | github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= 103 | github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= 104 | github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= 105 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 106 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 107 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 108 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 109 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 110 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 111 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 112 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 113 | github.com/webdevops/go-common v0.0.0-20250501164923-7cab87d11d0f h1:gbTwG6Cp4tYTFXp5FKxThUGKmd+Hi9qHIfrRy8m7dEI= 114 | github.com/webdevops/go-common v0.0.0-20250501164923-7cab87d11d0f/go.mod h1:GzD/xLtTZ5Vh3aHTi02g0OlfDUoiDx44OHeUnqWO2CI= 115 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 116 | go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= 117 | go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= 118 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 119 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 120 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 121 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 122 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 123 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 124 | go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= 125 | go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= 126 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 127 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 128 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 129 | golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= 130 | golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= 131 | golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= 132 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 133 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 134 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 135 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 136 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 137 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 138 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 139 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 140 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= 141 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= 142 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 143 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 144 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 145 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 146 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 147 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 148 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 149 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 150 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 151 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 152 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 153 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 154 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 155 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 156 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 157 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 158 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 159 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 160 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 161 | golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= 162 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 163 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 164 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 165 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 166 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 167 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 168 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 169 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 170 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 171 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 172 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 173 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 174 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 175 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 176 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 177 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 178 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 179 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 180 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 181 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 182 | k8s.io/apimachinery v0.33.0 h1:1a6kHrJxb2hs4t8EE5wuR/WxKDwGN1FKH3JvDtA0CIQ= 183 | k8s.io/apimachinery v0.33.0/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= 184 | k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= 185 | k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= 186 | k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e h1:KqK5c/ghOm8xkHYhlodbp6i6+r+ChV2vuAuVRdFbLro= 187 | k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 188 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "embed" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "html/template" 9 | "net/http" 10 | "os" 11 | "runtime" 12 | "time" 13 | 14 | "github.com/google/uuid" 15 | "github.com/jessevdk/go-flags" 16 | "github.com/patrickmn/go-cache" 17 | "github.com/prometheus/client_golang/prometheus" 18 | "github.com/prometheus/client_golang/prometheus/promhttp" 19 | "github.com/webdevops/go-common/azuresdk/armclient" 20 | "github.com/webdevops/go-common/azuresdk/azidentity" 21 | "github.com/webdevops/go-common/azuresdk/prometheus/tracing" 22 | 23 | "github.com/webdevops/azure-metrics-exporter/config" 24 | ) 25 | 26 | const ( 27 | Author = "webdevops.io" 28 | 29 | UserAgent = "azure-metrics-exporter/" 30 | ) 31 | 32 | var ( 33 | argparser *flags.Parser 34 | Opts config.Opts 35 | 36 | AzureClient *armclient.ArmClient 37 | AzureResourceTagManager *armclient.ResourceTagManager 38 | 39 | prometheusCollectTime *prometheus.SummaryVec 40 | prometheusMetricRequests *prometheus.CounterVec 41 | 42 | metricsCache *cache.Cache 43 | azureCache *cache.Cache 44 | 45 | //go:embed templates/*.html 46 | templates embed.FS 47 | 48 | // Git version information 49 | gitCommit = "" 50 | gitTag = "" 51 | ) 52 | 53 | func main() { 54 | initArgparser() 55 | initLogger() 56 | 57 | logger.Infof("starting azure-metrics-exporter v%s (%s; %s; by %v)", gitTag, gitCommit, runtime.Version(), Author) 58 | logger.Info(string(Opts.GetJson())) 59 | initSystem() 60 | metricsCache = cache.New(1*time.Minute, 1*time.Minute) 61 | azureCache = cache.New(1*time.Minute, 1*time.Minute) 62 | 63 | logger.Infof("init Azure connection") 64 | initAzureConnection() 65 | initMetricCollector() 66 | 67 | logger.Infof("starting http server on %s", Opts.Server.Bind) 68 | startHttpServer() 69 | } 70 | 71 | func initArgparser() { 72 | argparser = flags.NewParser(&Opts, flags.Default) 73 | _, err := argparser.Parse() 74 | 75 | // check if there is an parse error 76 | if err != nil { 77 | var flagsErr *flags.Error 78 | if ok := errors.As(err, &flagsErr); ok && flagsErr.Type == flags.ErrHelp { 79 | os.Exit(0) 80 | } else { 81 | fmt.Println() 82 | argparser.WriteHelp(os.Stdout) 83 | os.Exit(1) 84 | } 85 | } 86 | } 87 | 88 | func initAzureConnection() { 89 | var err error 90 | 91 | if Opts.Azure.Environment != nil { 92 | if err := os.Setenv(azidentity.EnvAzureEnvironment, *Opts.Azure.Environment); err != nil { 93 | logger.Warnf(`unable to set envvar "%s": %v`, azidentity.EnvAzureEnvironment, err.Error()) 94 | } 95 | } 96 | 97 | AzureClient, err = armclient.NewArmClientFromEnvironment(logger) 98 | if err != nil { 99 | logger.Fatal(err.Error()) 100 | } 101 | AzureClient.SetUserAgent(UserAgent + gitTag) 102 | 103 | if err := AzureClient.Connect(); err != nil { 104 | logger.Fatal(err.Error()) 105 | } 106 | 107 | AzureResourceTagManager, err = AzureClient.TagManager.ParseTagConfig(Opts.Azure.ResourceTags) 108 | if err != nil { 109 | logger.Fatalf(`unable to parse resourceTag configuration "%s": %v"`, Opts.Azure.ResourceTags, err.Error()) 110 | } 111 | } 112 | 113 | // start and handle prometheus handler 114 | func startHttpServer() { 115 | mux := http.NewServeMux() 116 | 117 | // healthz 118 | mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { 119 | if _, err := fmt.Fprint(w, "Ok"); err != nil { 120 | logger.Error(err) 121 | } 122 | }) 123 | 124 | // readyz 125 | mux.HandleFunc("/readyz", func(w http.ResponseWriter, r *http.Request) { 126 | if _, err := fmt.Fprint(w, "Ok"); err != nil { 127 | logger.Error(err) 128 | } 129 | }) 130 | 131 | mux.Handle(config.MetricsUrl, tracing.RegisterAzureMetricAutoClean(promhttp.Handler())) 132 | 133 | mux.HandleFunc(config.ProbeMetricsResourceUrl, probeMetricsResourceHandler) 134 | 135 | mux.HandleFunc(config.ProbeMetricsListUrl, probeMetricsListHandler) 136 | 137 | mux.HandleFunc(config.ProbeMetricsSubscriptionUrl, probeMetricsSubscriptionHandler) 138 | 139 | mux.HandleFunc(config.ProbeMetricsScrapeUrl, probeMetricsScrapeHandler) 140 | 141 | mux.HandleFunc(config.ProbeMetricsResourceGraphUrl, probeMetricsResourceGraphHandler) 142 | 143 | // report 144 | tmpl := template.Must(template.ParseFS(templates, "templates/*.html")) 145 | mux.HandleFunc("/query", func(w http.ResponseWriter, r *http.Request) { 146 | cspNonce := base64.StdEncoding.EncodeToString([]byte(uuid.New().String())) 147 | 148 | w.Header().Add("Content-Type", "text/html") 149 | w.Header().Add("Referrer-Policy", "same-origin") 150 | w.Header().Add("X-Frame-Options", "DENY") 151 | w.Header().Add("X-XSS-Protection", "1; mode=block") 152 | w.Header().Add("X-Content-Type-Options", "nosniff") 153 | w.Header().Add("Content-Security-Policy", 154 | fmt.Sprintf( 155 | "default-src 'self'; script-src 'nonce-%[1]s'; style-src 'nonce-%[1]s'; img-src 'self' data:", 156 | cspNonce, 157 | ), 158 | ) 159 | 160 | templatePayload := struct { 161 | Nonce string 162 | }{ 163 | Nonce: cspNonce, 164 | } 165 | 166 | if err := tmpl.ExecuteTemplate(w, "query.html", templatePayload); err != nil { 167 | logger.Error(err) 168 | } 169 | }) 170 | 171 | srv := &http.Server{ 172 | Addr: Opts.Server.Bind, 173 | Handler: mux, 174 | ReadTimeout: Opts.Server.ReadTimeout, 175 | WriteTimeout: Opts.Server.WriteTimeout, 176 | } 177 | logger.Fatal(srv.ListenAndServe()) 178 | } 179 | 180 | func initMetricCollector() { 181 | prometheusCollectTime = prometheus.NewSummaryVec( 182 | prometheus.SummaryOpts{ 183 | Name: "azurerm_stats_metric_collecttime", 184 | Help: "Azure Insights stats collecttime", 185 | }, 186 | []string{ 187 | "subscriptionID", 188 | "handler", 189 | "filter", 190 | }, 191 | ) 192 | prometheus.MustRegister(prometheusCollectTime) 193 | 194 | prometheusMetricRequests = prometheus.NewCounterVec( 195 | prometheus.CounterOpts{ 196 | Name: "azurerm_stats_metric_requests", 197 | Help: "Azure Insights resource requests", 198 | }, 199 | []string{ 200 | "subscriptionID", 201 | "handler", 202 | "filter", 203 | "result", 204 | }, 205 | ) 206 | prometheus.MustRegister(prometheusMetricRequests) 207 | } 208 | -------------------------------------------------------------------------------- /metrics/azure.policy.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | 7 | "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" 8 | ) 9 | 10 | type noCachePolicy struct{} 11 | 12 | func (p noCachePolicy) Do(req *policy.Request) (*http.Response, error) { 13 | // Mutate/process request. 14 | req.Raw().Header.Set("cache-control", "no-cache") 15 | 16 | // replace encoded %2C to , 17 | req.Raw().URL.RawQuery = strings.ReplaceAll(req.Raw().URL.RawQuery, "%2C", ",") 18 | 19 | // Forward the request to the next policy in the pipeline. 20 | return req.Next() 21 | } 22 | -------------------------------------------------------------------------------- /metrics/insights.general.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/prometheus/client_golang/prometheus" 7 | ) 8 | 9 | type ( 10 | AzureInsightBaseMetricsResult struct { 11 | prober *MetricProber 12 | } 13 | ) 14 | 15 | func (r *AzureInsightBaseMetricsResult) buildMetric(labels prometheus.Labels, value float64) (metric PrometheusMetricResult) { 16 | // copy map to ensure we don't keep references 17 | metricLabels := prometheus.Labels{} 18 | for labelName, labelValue := range labels { 19 | metricLabels[labelName] = labelValue 20 | } 21 | 22 | metric = PrometheusMetricResult{ 23 | Name: r.prober.settings.MetricTemplate, 24 | Labels: metricLabels, 25 | Value: value, 26 | } 27 | 28 | // fallback if template is empty (should not be) 29 | if r.prober.settings.MetricTemplate == "" { 30 | metric.Name = r.prober.settings.Name 31 | } 32 | 33 | resourceType := r.prober.settings.ResourceType 34 | // MetricNamespace is more descriptive than type 35 | if r.prober.settings.MetricNamespace != "" { 36 | resourceType = r.prober.settings.MetricNamespace 37 | } 38 | 39 | // set help 40 | metric.Help = r.prober.settings.HelpTemplate 41 | if metricNamePlaceholders.MatchString(metric.Help) { 42 | metric.Help = metricNamePlaceholders.ReplaceAllStringFunc( 43 | metric.Help, 44 | func(fieldName string) string { 45 | fieldName = strings.Trim(fieldName, "{}") 46 | switch fieldName { 47 | case "name": 48 | return r.prober.settings.Name 49 | case "type": 50 | return resourceType 51 | default: 52 | if fieldValue, exists := metric.Labels[fieldName]; exists { 53 | return fieldValue 54 | } 55 | } 56 | return "" 57 | }, 58 | ) 59 | } 60 | 61 | if metricNamePlaceholders.MatchString(metric.Name) { 62 | metric.Name = metricNamePlaceholders.ReplaceAllStringFunc( 63 | metric.Name, 64 | func(fieldName string) string { 65 | fieldName = strings.Trim(fieldName, "{}") 66 | switch fieldName { 67 | case "name": 68 | return r.prober.settings.Name 69 | case "type": 70 | return resourceType 71 | default: 72 | if fieldValue, exists := metric.Labels[fieldName]; exists { 73 | // remove label, when we add it to metric name 74 | delete(metric.Labels, fieldName) 75 | return fieldValue 76 | } 77 | } 78 | return "" 79 | }, 80 | ) 81 | } 82 | 83 | // sanitize metric name 84 | metric.Name = metricNameReplacer.Replace(metric.Name) 85 | metric.Name = strings.ToLower(metric.Name) 86 | metric.Name = metricNameNotAllowedChars.ReplaceAllString(metric.Name, "") 87 | 88 | return 89 | } 90 | -------------------------------------------------------------------------------- /metrics/insights.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/webdevops/go-common/utils/to" 11 | ) 12 | 13 | var ( 14 | metricNamePlaceholders = regexp.MustCompile(`{([^}]+)}`) 15 | metricNameNotAllowedChars = regexp.MustCompile(`[^a-zA-Z0-9_:]`) 16 | metricLabelNotAllowedChars = regexp.MustCompile(`[^a-zA-Z0-9_]`) 17 | metricNameReplacer = strings.NewReplacer("-", "_", " ", "_", "/", "_", ".", "_") 18 | ) 19 | 20 | type ( 21 | PrometheusMetricResult struct { 22 | Name string 23 | Labels prometheus.Labels 24 | Value float64 25 | Help string 26 | } 27 | ) 28 | 29 | func (p *MetricProber) MetricsClient(subscriptionId string) (*armmonitor.MetricsClient, error) { 30 | clientOpts := p.AzureClient.NewArmClientOptions() 31 | clientOpts.PerCallPolicies = append( 32 | clientOpts.PerCallPolicies, 33 | noCachePolicy{}, 34 | ) 35 | return armmonitor.NewMetricsClient(subscriptionId, p.AzureClient.GetCred(), clientOpts) 36 | } 37 | 38 | func (p *MetricProber) FetchMetricsFromTarget(client *armmonitor.MetricsClient, target MetricProbeTarget, metrics, aggregations []string) (AzureInsightMetricsResult, error) { 39 | ret := AzureInsightMetricsResult{ 40 | AzureInsightBaseMetricsResult: AzureInsightBaseMetricsResult{ 41 | prober: p, 42 | }, 43 | target: &target, 44 | } 45 | 46 | resultType := armmonitor.ResultTypeData 47 | opts := armmonitor.MetricsClientListOptions{ 48 | Interval: p.settings.Interval, 49 | ResultType: &resultType, 50 | Timespan: to.StringPtr(p.settings.Timespan), 51 | Metricnames: to.StringPtr(strings.Join(metrics, ",")), 52 | Top: p.settings.MetricTop, 53 | AutoAdjustTimegrain: to.BoolPtr(true), 54 | ValidateDimensions: to.BoolPtr(p.settings.ValidateDimensions), 55 | } 56 | 57 | if len(aggregations) >= 1 { 58 | opts.Aggregation = to.StringPtr(strings.Join(aggregations, ",")) 59 | } 60 | 61 | if len(p.settings.MetricFilter) >= 1 { 62 | opts.Filter = to.StringPtr(p.settings.MetricFilter) 63 | } 64 | 65 | if len(p.settings.MetricNamespace) >= 1 { 66 | opts.Metricnamespace = to.StringPtr(p.settings.MetricNamespace) 67 | } 68 | 69 | if len(p.settings.MetricOrderBy) >= 1 { 70 | opts.Orderby = to.StringPtr(p.settings.MetricOrderBy) 71 | } 72 | 73 | resourceURI := target.ResourceId 74 | if strings.HasPrefix(strings.ToLower(p.settings.MetricNamespace), "microsoft.storage/storageaccounts/") { 75 | splitNamespace := strings.Split(p.settings.MetricNamespace, "/") 76 | // Storage accounts have an extra requirement that their ResourceURI include /default 77 | storageAccountType := splitNamespace[len(splitNamespace)-1] 78 | resourceURI = resourceURI + fmt.Sprintf("/%s/default", storageAccountType) 79 | } 80 | 81 | result, err := client.List( 82 | p.ctx, 83 | resourceURI, 84 | &opts, 85 | ) 86 | 87 | if err == nil { 88 | ret.Result = &result 89 | } 90 | 91 | return ret, err 92 | } 93 | -------------------------------------------------------------------------------- /metrics/insights.subscription.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" 7 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/webdevops/go-common/azuresdk/armclient" 10 | stringsCommon "github.com/webdevops/go-common/strings" 11 | "github.com/webdevops/go-common/utils/to" 12 | ) 13 | 14 | type ( 15 | AzureInsightSubscriptionMetricsResult struct { 16 | AzureInsightBaseMetricsResult 17 | 18 | subscription *armsubscriptions.Subscription 19 | Result *armmonitor.MetricsClientListAtSubscriptionScopeResponse 20 | } 21 | ) 22 | 23 | func (r *AzureInsightSubscriptionMetricsResult) SendMetricToChannel(channel chan<- PrometheusMetricResult) { 24 | if r.Result.Value != nil { 25 | // DEBUGGING 26 | // data, _ := json.Marshal(r.Result) 27 | // fmt.Println(string(data)) 28 | 29 | for _, metric := range r.Result.Value { 30 | if metric.Timeseries != nil { 31 | for _, timeseries := range metric.Timeseries { 32 | if timeseries.Data != nil { 33 | // get dimension name (optional) 34 | dimensions := map[string]string{} 35 | resourceId := "" 36 | if timeseries.Metadatavalues != nil { 37 | for _, dimensionRow := range timeseries.Metadatavalues { 38 | dimensionRowName := to.String(dimensionRow.Name.Value) 39 | dimensionRowValue := to.String(dimensionRow.Value) 40 | 41 | if r.prober.settings.DimensionLowercase { 42 | dimensionRowValue = strings.ToLower(dimensionRowValue) 43 | } 44 | 45 | if strings.EqualFold(dimensionRowName, "microsoft.resourceid") { 46 | resourceId = dimensionRowValue 47 | } else { 48 | dimensions[dimensionRowName] = dimensionRowValue 49 | } 50 | } 51 | } 52 | 53 | azureResource, _ := armclient.ParseResourceId(resourceId) 54 | 55 | metricUnit := "" 56 | if metric.Unit != nil { 57 | metricUnit = string(*metric.Unit) 58 | } 59 | 60 | metricLabels := prometheus.Labels{ 61 | "resourceID": strings.ToLower(resourceId), 62 | "subscriptionID": azureResource.Subscription, 63 | "subscriptionName": to.String(r.subscription.DisplayName), 64 | "resourceGroup": azureResource.ResourceGroup, 65 | "resourceName": azureResource.ResourceName, 66 | "metric": to.String(metric.Name.Value), 67 | "unit": metricUnit, 68 | "interval": to.String(r.prober.settings.Interval), 69 | "timespan": r.prober.settings.Timespan, 70 | "aggregation": "", 71 | } 72 | 73 | // add resource tags as labels 74 | metricLabels = r.prober.AzureResourceTagManager.AddResourceTagsToPrometheusLabels(r.prober.ctx, metricLabels, resourceId) 75 | 76 | if len(dimensions) == 1 { 77 | // we have only one dimension 78 | // add one dimension="foobar" label (backward compatibility) 79 | for _, dimensionValue := range dimensions { 80 | metricLabels["dimension"] = dimensionValue 81 | } 82 | } else if len(dimensions) >= 2 { 83 | // we have multiple dimensions 84 | // add each dimension as dimensionXzy="foobar" label 85 | for dimensionName, dimensionValue := range dimensions { 86 | labelName := "dimension" + stringsCommon.UppercaseFirst(dimensionName) 87 | labelName = metricLabelNotAllowedChars.ReplaceAllString(labelName, "") 88 | metricLabels[labelName] = dimensionValue 89 | } 90 | } 91 | 92 | for _, timeseriesData := range timeseries.Data { 93 | if timeseriesData.Total != nil { 94 | metricLabels["aggregation"] = "total" 95 | channel <- r.buildMetric( 96 | metricLabels, 97 | *timeseriesData.Total, 98 | ) 99 | } 100 | 101 | if timeseriesData.Minimum != nil { 102 | metricLabels["aggregation"] = "minimum" 103 | channel <- r.buildMetric( 104 | metricLabels, 105 | *timeseriesData.Minimum, 106 | ) 107 | } 108 | 109 | if timeseriesData.Maximum != nil { 110 | metricLabels["aggregation"] = "maximum" 111 | channel <- r.buildMetric( 112 | metricLabels, 113 | *timeseriesData.Maximum, 114 | ) 115 | } 116 | 117 | if timeseriesData.Average != nil { 118 | metricLabels["aggregation"] = "average" 119 | channel <- r.buildMetric( 120 | metricLabels, 121 | *timeseriesData.Average, 122 | ) 123 | } 124 | 125 | if timeseriesData.Count != nil { 126 | metricLabels["aggregation"] = "count" 127 | channel <- r.buildMetric( 128 | metricLabels, 129 | *timeseriesData.Count, 130 | ) 131 | } 132 | } 133 | } 134 | } 135 | } 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /metrics/insights.target.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/webdevops/go-common/azuresdk/armclient" 9 | stringsCommon "github.com/webdevops/go-common/strings" 10 | "github.com/webdevops/go-common/utils/to" 11 | ) 12 | 13 | type ( 14 | AzureInsightMetricsResult struct { 15 | AzureInsightBaseMetricsResult 16 | 17 | target *MetricProbeTarget 18 | Result *armmonitor.MetricsClientListResponse 19 | } 20 | ) 21 | 22 | func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- PrometheusMetricResult) { 23 | if r.Result.Value != nil { 24 | // DEBUGGING 25 | // data, _ := json.Marshal(r.Result) 26 | // fmt.Println(string(data)) 27 | 28 | for _, metric := range r.Result.Value { 29 | if metric.Timeseries != nil { 30 | for _, timeseries := range metric.Timeseries { 31 | if timeseries.Data != nil { 32 | // get dimension name (optional) 33 | dimensions := map[string]string{} 34 | if timeseries.Metadatavalues != nil { 35 | for _, dimensionRow := range timeseries.Metadatavalues { 36 | dimensionValue := to.String(dimensionRow.Value) 37 | if r.prober.settings.DimensionLowercase { 38 | dimensionValue = strings.ToLower(dimensionValue) 39 | } 40 | dimensions[to.String(dimensionRow.Name.Value)] = dimensionValue 41 | } 42 | } 43 | 44 | resourceId := r.target.ResourceId 45 | azureResource, _ := armclient.ParseResourceId(resourceId) 46 | 47 | metricUnit := "" 48 | if metric.Unit != nil { 49 | metricUnit = string(*metric.Unit) 50 | } 51 | 52 | subscriptionName := "" 53 | if subscription, err := r.prober.AzureClient.GetCachedSubscription(r.prober.ctx, azureResource.Subscription); err == nil && subscription != nil { 54 | subscriptionName = to.String(subscription.DisplayName) 55 | } 56 | 57 | metricLabels := prometheus.Labels{ 58 | "resourceID": strings.ToLower(resourceId), 59 | "subscriptionID": azureResource.Subscription, 60 | "subscriptionName": subscriptionName, 61 | "resourceGroup": azureResource.ResourceGroup, 62 | "resourceName": azureResource.ResourceName, 63 | "metric": to.String(metric.Name.Value), 64 | "unit": metricUnit, 65 | "interval": to.String(r.prober.settings.Interval), 66 | "timespan": r.prober.settings.Timespan, 67 | "aggregation": "", 68 | } 69 | 70 | // add resource tags as labels 71 | metricLabels = r.prober.AzureResourceTagManager.AddResourceTagsToPrometheusLabels(r.prober.ctx, metricLabels, resourceId) 72 | 73 | if len(dimensions) == 1 { 74 | // we have only one dimension 75 | // add one dimension="foobar" label (backward compatibility) 76 | for _, dimensionValue := range dimensions { 77 | metricLabels["dimension"] = dimensionValue 78 | } 79 | } else if len(dimensions) >= 2 { 80 | // we have multiple dimensions 81 | // add each dimension as dimensionXzy="foobar" label 82 | for dimensionName, dimensionValue := range dimensions { 83 | labelName := "dimension" + stringsCommon.UppercaseFirst(dimensionName) 84 | labelName = metricLabelNotAllowedChars.ReplaceAllString(labelName, "") 85 | metricLabels[labelName] = dimensionValue 86 | } 87 | } 88 | 89 | for _, timeseriesData := range timeseries.Data { 90 | if timeseriesData.Total != nil { 91 | metricLabels["aggregation"] = "total" 92 | channel <- r.buildMetric( 93 | metricLabels, 94 | *timeseriesData.Total, 95 | ) 96 | } 97 | 98 | if timeseriesData.Minimum != nil { 99 | metricLabels["aggregation"] = "minimum" 100 | channel <- r.buildMetric( 101 | metricLabels, 102 | *timeseriesData.Minimum, 103 | ) 104 | } 105 | 106 | if timeseriesData.Maximum != nil { 107 | metricLabels["aggregation"] = "maximum" 108 | channel <- r.buildMetric( 109 | metricLabels, 110 | *timeseriesData.Maximum, 111 | ) 112 | } 113 | 114 | if timeseriesData.Average != nil { 115 | metricLabels["aggregation"] = "average" 116 | channel <- r.buildMetric( 117 | metricLabels, 118 | *timeseriesData.Average, 119 | ) 120 | } 121 | 122 | if timeseriesData.Count != nil { 123 | metricLabels["aggregation"] = "count" 124 | channel <- r.buildMetric( 125 | metricLabels, 126 | *timeseriesData.Count, 127 | ) 128 | } 129 | } 130 | } 131 | } 132 | } 133 | } 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | ) 6 | 7 | const ( 8 | MetricHelpDefault = "Azure monitor insight metric" 9 | ) 10 | 11 | type ( 12 | MetricList struct { 13 | List map[string][]MetricRow 14 | Help map[string]string 15 | } 16 | 17 | MetricRow struct { 18 | Labels prometheus.Labels 19 | Value float64 20 | } 21 | ) 22 | 23 | func NewMetricList() *MetricList { 24 | list := MetricList{} 25 | list.List = map[string][]MetricRow{} 26 | list.Help = map[string]string{} 27 | return &list 28 | } 29 | 30 | func (l *MetricList) Add(name string, metric ...MetricRow) { 31 | if _, ok := l.List[name]; !ok { 32 | l.List[name] = []MetricRow{} 33 | } 34 | 35 | l.List[name] = append(l.List[name], metric...) 36 | } 37 | 38 | func (l *MetricList) GetMetricNames() (list []string) { 39 | for name := range l.List { 40 | list = append(list, name) 41 | } 42 | return 43 | } 44 | 45 | func (l *MetricList) SetMetricHelp(name, help string) { 46 | l.Help[name] = help 47 | } 48 | 49 | func (l *MetricList) GetMetricHelp(name string) string { 50 | if val, ok := l.Help[name]; ok { 51 | return val 52 | } 53 | return MetricHelpDefault 54 | } 55 | 56 | func (l *MetricList) GetMetricList(name string) []MetricRow { 57 | return l.List[name] 58 | } 59 | 60 | func (l *MetricList) GetMetricLabelNames(name string) []string { 61 | var list []string 62 | uniqueLabelMap := map[string]string{} 63 | 64 | for _, row := range l.List[name] { 65 | for labelName := range row.Labels { 66 | uniqueLabelMap[labelName] = labelName 67 | } 68 | } 69 | 70 | for labelName := range uniqueLabelMap { 71 | list = append(list, labelName) 72 | } 73 | 74 | return list 75 | } 76 | -------------------------------------------------------------------------------- /metrics/misc.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strings" 7 | ) 8 | 9 | func paramsGetWithDefault(params url.Values, name, defaultValue string) (value string) { 10 | value = params.Get(name) 11 | if value == "" { 12 | value = defaultValue 13 | } 14 | return 15 | } 16 | 17 | func paramsGetList(params url.Values, name string) (list []string, err error) { 18 | for _, v := range params[name] { 19 | list = append(list, stringToStringList(v, ",")...) 20 | } 21 | return 22 | } 23 | 24 | func paramsGetListRequired(params url.Values, name string) (list []string, err error) { 25 | list, err = paramsGetList(params, name) 26 | 27 | if len(list) == 0 { 28 | err = fmt.Errorf("parameter \"%v\" is missing", name) 29 | return 30 | } 31 | 32 | return 33 | } 34 | 35 | func stringToStringList(v string, sep string) (list []string) { 36 | for _, v := range strings.Split(v, sep) { 37 | list = append(list, strings.TrimSpace(v)) 38 | } 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /metrics/prober.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "strings" 8 | "time" 9 | 10 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" 11 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" 12 | "github.com/Azure/go-autorest/autorest/azure" 13 | "github.com/patrickmn/go-cache" 14 | "github.com/prometheus/client_golang/prometheus" 15 | "github.com/remeh/sizedwaitgroup" 16 | "github.com/webdevops/go-common/azuresdk/armclient" 17 | "github.com/webdevops/go-common/utils/to" 18 | "go.uber.org/zap" 19 | 20 | "github.com/webdevops/azure-metrics-exporter/config" 21 | ) 22 | 23 | const ( 24 | AzureMetricApiMaxMetricNumber = 20 25 | ) 26 | 27 | type ( 28 | MetricProber struct { 29 | Conf config.Opts 30 | 31 | AzureClient *armclient.ArmClient 32 | AzureResourceTagManager *armclient.ResourceTagManager 33 | 34 | userAgent string 35 | 36 | settings *RequestMetricSettings 37 | 38 | response http.ResponseWriter 39 | 40 | ctx context.Context 41 | 42 | logger *zap.SugaredLogger 43 | 44 | metricsCache struct { 45 | cache *cache.Cache 46 | cacheKey *string 47 | cacheDuration *time.Duration 48 | } 49 | 50 | serviceDiscoveryCache struct { 51 | cache *cache.Cache 52 | cacheDuration *time.Duration 53 | } 54 | 55 | targets map[string][]MetricProbeTarget 56 | 57 | metricList *MetricList 58 | 59 | prometheus struct { 60 | registry *prometheus.Registry 61 | } 62 | 63 | callbackSubscriptionFishish func(subscriptionId string) 64 | 65 | ServiceDiscovery AzureServiceDiscovery 66 | } 67 | 68 | MetricProbeTarget struct { 69 | ResourceId string 70 | Metrics []string 71 | Aggregations []string 72 | Tags map[string]string 73 | } 74 | ) 75 | 76 | func NewMetricProber(ctx context.Context, logger *zap.SugaredLogger, w http.ResponseWriter, settings *RequestMetricSettings, conf config.Opts) *MetricProber { 77 | prober := MetricProber{} 78 | prober.ctx = ctx 79 | prober.response = w 80 | prober.logger = logger 81 | prober.settings = settings 82 | prober.Conf = conf 83 | prober.ServiceDiscovery = AzureServiceDiscovery{prober: &prober} 84 | prober.Init() 85 | return &prober 86 | } 87 | 88 | func (p *MetricProber) Init() { 89 | p.targets = map[string][]MetricProbeTarget{} 90 | 91 | p.metricList = NewMetricList() 92 | } 93 | func (p *MetricProber) RegisterSubscriptionCollectFinishCallback(callback func(subscriptionId string)) { 94 | p.callbackSubscriptionFishish = callback 95 | } 96 | 97 | func (p *MetricProber) SetUserAgent(value string) { 98 | p.userAgent = value 99 | } 100 | 101 | func (p *MetricProber) SetPrometheusRegistry(registry *prometheus.Registry) { 102 | p.prometheus.registry = registry 103 | } 104 | 105 | func (p *MetricProber) SetAzureClient(client *armclient.ArmClient) { 106 | p.AzureClient = client 107 | } 108 | 109 | func (p *MetricProber) SetAzureResourceTagManager(client *armclient.ResourceTagManager) { 110 | p.AzureResourceTagManager = client 111 | } 112 | 113 | func (p *MetricProber) EnableMetricsCache(cache *cache.Cache, cacheKey string, cacheDuration *time.Duration) { 114 | p.metricsCache.cache = cache 115 | p.metricsCache.cacheKey = &cacheKey 116 | p.metricsCache.cacheDuration = cacheDuration 117 | } 118 | 119 | func (p *MetricProber) EnableServiceDiscoveryCache(cache *cache.Cache, cacheDuration *time.Duration) { 120 | p.serviceDiscoveryCache.cache = cache 121 | p.serviceDiscoveryCache.cacheDuration = cacheDuration 122 | } 123 | 124 | func (p *MetricProber) AddTarget(targets ...MetricProbeTarget) { 125 | for _, target := range targets { 126 | resourceInfo, err := azure.ParseResourceID(target.ResourceId) 127 | if err != nil { 128 | p.logger.Warnf("unable to parse resource id: %s", err.Error()) 129 | continue 130 | } 131 | 132 | subscriptionId := resourceInfo.SubscriptionID 133 | if _, exists := p.targets[subscriptionId]; !exists { 134 | p.targets[subscriptionId] = []MetricProbeTarget{} 135 | } 136 | 137 | p.targets[subscriptionId] = append(p.targets[subscriptionId], target) 138 | } 139 | } 140 | 141 | func (p *MetricProber) FetchFromCache() bool { 142 | if p.metricsCache.cache == nil { 143 | return false 144 | } 145 | 146 | if val, ok := p.metricsCache.cache.Get(*p.metricsCache.cacheKey); ok { 147 | p.metricList = val.(*MetricList) 148 | p.publishMetricList() 149 | return true 150 | } 151 | 152 | return false 153 | } 154 | 155 | func (p *MetricProber) SaveToCache() { 156 | if p.metricsCache.cache == nil { 157 | return 158 | } 159 | 160 | if p.metricsCache.cacheDuration != nil { 161 | _ = p.metricsCache.cache.Add(*p.metricsCache.cacheKey, p.metricList, *p.metricsCache.cacheDuration) 162 | p.response.Header().Add("X-metrics-cached-until", time.Now().Add(*p.metricsCache.cacheDuration).Format(time.RFC3339)) 163 | } 164 | } 165 | 166 | func (p *MetricProber) Run() { 167 | p.collectMetricsFromTargets() 168 | p.SaveToCache() 169 | p.publishMetricList() 170 | } 171 | 172 | func (p *MetricProber) RunOnSubscriptionScope() { 173 | p.collectMetricsFromSubscriptions() 174 | p.SaveToCache() 175 | p.publishMetricList() 176 | } 177 | 178 | func (p *MetricProber) collectMetricsFromSubscriptions() { 179 | metricsChannel := make(chan PrometheusMetricResult) 180 | 181 | subscriptionIterator := armclient.NewSubscriptionIterator(p.AzureClient, p.settings.Subscriptions...) 182 | subscriptionIterator.SetConcurrency(p.Conf.Prober.ConcurrencySubscription) 183 | 184 | go func() { 185 | regions, err := p.discoverResourceRegions() 186 | if err != nil { 187 | p.logger.Error(fmt.Errorf("error getting subscription locations: %w", err)) 188 | return 189 | } 190 | 191 | err = subscriptionIterator.ForEachAsync(p.logger, func(subscription *armsubscriptions.Subscription, logger *zap.SugaredLogger) { 192 | subscriptionRegions := regions[*subscription.SubscriptionID] 193 | 194 | for _, region := range subscriptionRegions { 195 | client, err := p.MetricsClient(*subscription.SubscriptionID) 196 | if err != nil { 197 | // FIXME: find a better way to report errors 198 | p.logger.Error(err) 199 | return 200 | } 201 | 202 | // request metrics in 20 metrics chunks (azure metric api limitation) 203 | for i := 0; i < len(p.settings.Metrics); i += AzureMetricApiMaxMetricNumber { 204 | end := i + AzureMetricApiMaxMetricNumber 205 | if end > len(p.settings.Metrics) { 206 | end = len(p.settings.Metrics) 207 | } 208 | metricList := p.settings.Metrics[i:end] 209 | 210 | resultType := armmonitor.MetricResultTypeData 211 | opts := armmonitor.MetricsClientListAtSubscriptionScopeOptions{ 212 | Interval: p.settings.Interval, 213 | Timespan: to.StringPtr(p.settings.Timespan), 214 | Metricnames: to.StringPtr(strings.Join(metricList, ",")), 215 | Metricnamespace: to.StringPtr(p.settings.ResourceType), 216 | Top: p.settings.MetricTop, 217 | AutoAdjustTimegrain: to.BoolPtr(true), 218 | ResultType: &resultType, 219 | ValidateDimensions: to.BoolPtr(p.settings.ValidateDimensions), 220 | Filter: to.StringPtr(`Microsoft.ResourceId eq '*'`), 221 | } 222 | 223 | if len(p.settings.Aggregations) >= 1 { 224 | opts.Aggregation = to.StringPtr(strings.Join(p.settings.Aggregations, ",")) 225 | } 226 | 227 | if len(p.settings.MetricFilter) >= 1 { 228 | opts.Filter = to.StringPtr(*opts.Filter + " and " + p.settings.MetricFilter) 229 | } 230 | 231 | if len(p.settings.MetricOrderBy) >= 1 { 232 | opts.Orderby = to.StringPtr(p.settings.MetricOrderBy) 233 | } 234 | 235 | if len(p.settings.MetricNamespace) >= 1 { 236 | opts.Metricnamespace = to.StringPtr(p.settings.MetricNamespace) 237 | } 238 | 239 | response, err := client.ListAtSubscriptionScope(p.ctx, region, &opts) 240 | if err != nil { 241 | // FIXME: find a better way to report errors 242 | p.logger.Error(err) 243 | return 244 | } 245 | 246 | result := AzureInsightSubscriptionMetricsResult{ 247 | AzureInsightBaseMetricsResult: AzureInsightBaseMetricsResult{ 248 | prober: p, 249 | }, 250 | subscription: subscription, 251 | Result: &response} 252 | result.SendMetricToChannel(metricsChannel) 253 | } 254 | 255 | if p.callbackSubscriptionFishish != nil { 256 | p.callbackSubscriptionFishish(*subscription.SubscriptionID) 257 | } 258 | } 259 | }) 260 | if err != nil { 261 | // FIXME: find a better way to report errors 262 | p.logger.Error(err) 263 | } 264 | 265 | close(metricsChannel) 266 | }() 267 | 268 | for result := range metricsChannel { 269 | metric := MetricRow{ 270 | Labels: result.Labels, 271 | Value: result.Value, 272 | } 273 | p.metricList.Add(result.Name, metric) 274 | p.metricList.SetMetricHelp(result.Name, result.Help) 275 | } 276 | } 277 | 278 | func (p *MetricProber) discoverResourceRegions() (map[string][]string, error) { 279 | regions := map[string][]string{} 280 | 281 | for _, subscriptionId := range p.settings.Subscriptions { 282 | if len(p.settings.Regions) == 0 { 283 | regions[subscriptionId] = []string{} 284 | } else { 285 | regions[subscriptionId] = p.settings.Regions 286 | } 287 | } 288 | 289 | if len(p.settings.Regions) != 0 { 290 | return regions, nil 291 | } 292 | 293 | query := fmt.Sprintf(`Resources | where type == "%s" | summarize count() by subscriptionId, location`, strings.ToLower(p.settings.ResourceType)) 294 | 295 | opts := armclient.ResourceGraphOptions{ 296 | Subscriptions: p.settings.Subscriptions, 297 | } 298 | results, err := p.AzureClient.ExecuteResourceGraphQuery(p.ctx, query, opts) 299 | if err != nil { 300 | return nil, err 301 | } 302 | 303 | for _, row := range results { 304 | subscriptionId := row["subscriptionId"].(string) 305 | location := row["location"].(string) 306 | 307 | regions[subscriptionId] = append(regions[subscriptionId], location) 308 | } 309 | 310 | return regions, nil 311 | } 312 | 313 | func (p *MetricProber) collectMetricsFromTargets() { 314 | metricsChannel := make(chan PrometheusMetricResult) 315 | 316 | wgSubscription := sizedwaitgroup.New(p.Conf.Prober.ConcurrencySubscription) 317 | 318 | go func() { 319 | for subscriptionId, resourceList := range p.targets { 320 | wgSubscription.Add() 321 | go func(subscriptionId string, targetList []MetricProbeTarget) { 322 | defer wgSubscription.Done() 323 | 324 | wgSubscriptionResource := sizedwaitgroup.New(p.Conf.Prober.ConcurrencySubscriptionResource) 325 | client, err := p.MetricsClient(subscriptionId) 326 | if err != nil { 327 | // FIXME: find a better way to report errors 328 | p.logger.Error(err) 329 | return 330 | } 331 | 332 | for _, target := range targetList { 333 | wgSubscriptionResource.Add() 334 | go func(target MetricProbeTarget) { 335 | defer wgSubscriptionResource.Done() 336 | 337 | // request metrics in 20 metrics chunks (azure metric api limitation) 338 | for i := 0; i < len(target.Metrics); i += AzureMetricApiMaxMetricNumber { 339 | end := i + AzureMetricApiMaxMetricNumber 340 | if end > len(target.Metrics) { 341 | end = len(target.Metrics) 342 | } 343 | metricList := target.Metrics[i:end] 344 | 345 | if result, err := p.FetchMetricsFromTarget(client, target, metricList, target.Aggregations); err == nil { 346 | result.SendMetricToChannel(metricsChannel) 347 | } else { 348 | p.logger.With(zap.String("resourceID", target.ResourceId)).Warn(err) 349 | } 350 | } 351 | }(target) 352 | } 353 | wgSubscriptionResource.Wait() 354 | 355 | if p.callbackSubscriptionFishish != nil { 356 | p.callbackSubscriptionFishish(subscriptionId) 357 | } 358 | }(subscriptionId, resourceList) 359 | } 360 | wgSubscription.Wait() 361 | close(metricsChannel) 362 | }() 363 | 364 | for result := range metricsChannel { 365 | metric := MetricRow{ 366 | Labels: result.Labels, 367 | Value: result.Value, 368 | } 369 | p.metricList.Add(result.Name, metric) 370 | p.metricList.SetMetricHelp(result.Name, result.Help) 371 | } 372 | } 373 | 374 | func (p *MetricProber) publishMetricList() { 375 | if p.metricList == nil { 376 | return 377 | } 378 | 379 | // create prometheus metrics and set rows 380 | for _, metricName := range p.metricList.GetMetricNames() { 381 | gauge := prometheus.NewGaugeVec( 382 | prometheus.GaugeOpts{ 383 | Name: metricName, 384 | Help: p.metricList.GetMetricHelp(metricName), 385 | }, 386 | p.metricList.GetMetricLabelNames(metricName), 387 | ) 388 | p.prometheus.registry.MustRegister(gauge) 389 | 390 | for _, row := range p.metricList.GetMetricList(metricName) { 391 | gauge.With(row.Labels).Set(row.Value) 392 | } 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /metrics/servicediscovery.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" // #nosec G505 6 | "encoding/json" 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph" 11 | "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" 12 | "github.com/webdevops/go-common/utils/to" 13 | "go.uber.org/zap" 14 | ) 15 | 16 | const ( 17 | ResourceGraphQueryTop = 1000 18 | ) 19 | 20 | type ( 21 | AzureServiceDiscovery struct { 22 | prober *MetricProber 23 | } 24 | 25 | AzureResource struct { 26 | ID string 27 | Location string 28 | Tags map[string]string 29 | } 30 | ) 31 | 32 | func (sd *AzureServiceDiscovery) ResourcesClient(subscriptionId string) (*armresources.Client, error) { 33 | return armresources.NewClient(subscriptionId, sd.prober.AzureClient.GetCred(), sd.prober.AzureClient.NewArmClientOptions()) 34 | } 35 | 36 | func (sd *AzureServiceDiscovery) publishTargetList(targetList []MetricProbeTarget) { 37 | sd.prober.AddTarget(targetList...) 38 | } 39 | 40 | func (sd *AzureServiceDiscovery) fetchResourceList(subscriptionId, filter string) (resourceList []AzureResource, err error) { 41 | // nolint:gosec 42 | cacheKey := fmt.Sprintf( 43 | "%x", 44 | sha1.Sum([]byte(fmt.Sprintf("%v:%v", subscriptionId, filter))), 45 | ) 46 | 47 | // try to fetch info from cache 48 | if cachedResourceList, ok := sd.fetchFromCache(cacheKey); !ok { 49 | client, err := sd.ResourcesClient(subscriptionId) 50 | if err != nil { 51 | err = fmt.Errorf("servicediscovery failed: %w", err) 52 | return resourceList, err 53 | } 54 | 55 | opts := armresources.ClientListOptions{ 56 | Filter: to.StringPtr(filter), 57 | } 58 | pager := client.NewListPager(&opts) 59 | 60 | for pager.More() { 61 | result, err := pager.NextPage(sd.prober.ctx) 62 | if err != nil { 63 | err = fmt.Errorf("servicediscovery failed: %w", err) 64 | return resourceList, err 65 | } 66 | 67 | if result.Value == nil { 68 | continue 69 | } 70 | 71 | for _, row := range result.Value { 72 | resource := row 73 | 74 | resourceList = append( 75 | resourceList, 76 | AzureResource{ 77 | ID: to.String(resource.ID), 78 | Tags: to.StringMap(resource.Tags), 79 | }, 80 | ) 81 | } 82 | } 83 | 84 | // store to cache (if enabled) 85 | sd.saveToCache(cacheKey, resourceList) 86 | } else { 87 | sd.prober.logger.Debugf("using servicediscovery from cache") 88 | resourceList = cachedResourceList 89 | } 90 | 91 | return 92 | } 93 | 94 | func (sd *AzureServiceDiscovery) fetchFromCache(cacheKey string) (resourceList []AzureResource, status bool) { 95 | contextLogger := sd.prober.logger 96 | cache := sd.prober.serviceDiscoveryCache.cache 97 | 98 | if cache != nil { 99 | if v, ok := cache.Get(cacheKey); ok { 100 | if cacheData, ok := v.([]byte); ok { 101 | if err := json.Unmarshal(cacheData, &resourceList); err == nil { 102 | status = true 103 | } else { 104 | contextLogger.Debug("unable to parse cached servicediscovery") 105 | } 106 | } 107 | } 108 | } 109 | 110 | return 111 | } 112 | 113 | func (sd *AzureServiceDiscovery) saveToCache(cacheKey string, resourceList []AzureResource) { 114 | contextLogger := sd.prober.logger 115 | cache := sd.prober.serviceDiscoveryCache.cache 116 | cacheDuration := sd.prober.serviceDiscoveryCache.cacheDuration 117 | 118 | // store to cache (if enabled) 119 | if cache != nil { 120 | contextLogger.Debug("saving servicedisccovery to cache") 121 | if cacheData, err := json.Marshal(resourceList); err == nil { 122 | cache.Set(cacheKey, cacheData, *cacheDuration) 123 | contextLogger.Debugf("saved servicediscovery to cache for %s", cacheDuration.String()) 124 | } 125 | } 126 | } 127 | 128 | func (sd *AzureServiceDiscovery) FindSubscriptionResources(subscriptionId, filter string) { 129 | var targetList []MetricProbeTarget 130 | 131 | if resourceList, err := sd.fetchResourceList(subscriptionId, filter); err == nil { 132 | for _, resource := range resourceList { 133 | targetList = append( 134 | targetList, 135 | MetricProbeTarget{ 136 | ResourceId: resource.ID, 137 | Metrics: sd.prober.settings.Metrics, 138 | Aggregations: sd.prober.settings.Aggregations, 139 | Tags: resource.Tags, 140 | }, 141 | ) 142 | } 143 | } else { 144 | sd.prober.logger.Error(err) 145 | return 146 | } 147 | 148 | sd.publishTargetList(targetList) 149 | } 150 | 151 | func (sd *AzureServiceDiscovery) FindSubscriptionResourcesWithScrapeTags(ctx context.Context, subscriptionId, filter, metricTagName, aggregationTagName string) { 152 | var targetList []MetricProbeTarget 153 | 154 | if resourceList, err := sd.fetchResourceList(subscriptionId, filter); err == nil { 155 | for _, resource := range resourceList { 156 | if metrics, ok := resource.Tags[metricTagName]; ok && metrics != "" { 157 | if aggregations, ok := resource.Tags[aggregationTagName]; ok && aggregations != "" { 158 | targetList = append( 159 | targetList, 160 | MetricProbeTarget{ 161 | ResourceId: resource.ID, 162 | Metrics: stringToStringList(metrics, ","), 163 | Aggregations: stringToStringList(aggregations, ","), 164 | }, 165 | ) 166 | 167 | } 168 | } 169 | } 170 | } else { 171 | sd.prober.logger.Error(err) 172 | return 173 | } 174 | 175 | sd.publishTargetList(targetList) 176 | } 177 | 178 | func (sd *AzureServiceDiscovery) FindResourceGraph(ctx context.Context, subscriptions []string, resourceType, filter string) error { 179 | var targetList []MetricProbeTarget 180 | 181 | client, err := armresourcegraph.NewClient(sd.prober.AzureClient.GetCred(), sd.prober.AzureClient.NewArmClientOptions()) 182 | if err != nil { 183 | return err 184 | } 185 | 186 | if filter != "" { 187 | filter = "| " + filter 188 | } 189 | 190 | queryTemplate := `Resources | where type =~ "%s" %s | project id, tags` 191 | 192 | query := strings.TrimSpace(fmt.Sprintf( 193 | queryTemplate, 194 | strings.ReplaceAll(resourceType, "'", "\\'"), 195 | filter, 196 | )) 197 | 198 | sd.prober.logger.With(zap.String("query", query)).Debugf("using Kusto query") 199 | 200 | queryFormat := armresourcegraph.ResultFormatObjectArray 201 | queryTop := int32(ResourceGraphQueryTop) 202 | queryRequest := armresourcegraph.QueryRequest{ 203 | Query: to.StringPtr(query), 204 | Options: &armresourcegraph.QueryRequestOptions{ 205 | ResultFormat: &queryFormat, 206 | Top: &queryTop, 207 | }, 208 | Subscriptions: to.SlicePtr(subscriptions), 209 | } 210 | 211 | result, err := client.Resources(ctx, queryRequest, nil) 212 | if err != nil { 213 | return err 214 | } 215 | 216 | for { 217 | if resultList, ok := result.Data.([]interface{}); ok { 218 | // check if we got data, otherwise break the for loop 219 | if len(resultList) == 0 { 220 | break 221 | } 222 | 223 | for _, v := range resultList { 224 | if resultRow, ok := v.(map[string]interface{}); ok { 225 | // check if we got data, otherwise break the for loop 226 | if len(resultList) == 0 { 227 | break 228 | } 229 | 230 | if val, ok := resultRow["id"]; ok && val != "" { 231 | if resourceId, ok := val.(string); ok { 232 | targetList = append( 233 | targetList, 234 | MetricProbeTarget{ 235 | ResourceId: resourceId, 236 | Metrics: sd.prober.settings.Metrics, 237 | Aggregations: sd.prober.settings.Aggregations, 238 | Tags: sd.resourceTagsToStringMap(resultRow["tags"]), 239 | }, 240 | ) 241 | } 242 | } 243 | } 244 | } 245 | } 246 | 247 | if result.SkipToken != nil { 248 | queryRequest.Options.SkipToken = result.SkipToken 249 | result, err = client.Resources(ctx, queryRequest, nil) 250 | if err != nil { 251 | return err 252 | } 253 | } else { 254 | break 255 | } 256 | } 257 | 258 | sd.publishTargetList(targetList) 259 | return nil 260 | } 261 | 262 | func (sd *AzureServiceDiscovery) resourceTagsToStringMap(tags interface{}) (ret map[string]string) { 263 | ret = map[string]string{} 264 | 265 | switch tagMap := tags.(type) { 266 | case map[string]interface{}: 267 | for tag, value := range tagMap { 268 | switch v := value.(type) { 269 | case string: 270 | ret[tag] = v 271 | case *string: 272 | ret[tag] = to.String(v) 273 | } 274 | } 275 | case map[string]string: 276 | ret = tagMap 277 | case map[string]*string: 278 | ret = to.StringMap(tagMap) 279 | case map[*string]*string: 280 | for tag, value := range tagMap { 281 | ret[to.String(tag)] = to.String(value) 282 | } 283 | } 284 | 285 | return ret 286 | } 287 | -------------------------------------------------------------------------------- /metrics/settings.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | iso8601 "github.com/channelmeter/iso8601duration" 11 | 12 | "github.com/webdevops/azure-metrics-exporter/config" 13 | ) 14 | 15 | const ( 16 | PrometheusMetricNameDefault = "azurerm_resource_metric" 17 | ) 18 | 19 | type ( 20 | RequestMetricSettings struct { 21 | Name string 22 | Subscriptions []string 23 | ResourceType string 24 | Filter string 25 | Timespan string 26 | Interval *string 27 | Metrics []string 28 | MetricNamespace string 29 | Aggregations []string 30 | Regions []string 31 | 32 | // needed for dimension support 33 | MetricTop *int32 34 | MetricFilter string 35 | MetricOrderBy string 36 | 37 | ValidateDimensions bool 38 | 39 | MetricTemplate string 40 | HelpTemplate string 41 | 42 | DimensionLowercase bool 43 | 44 | // cache 45 | Cache *time.Duration 46 | } 47 | ) 48 | 49 | func NewRequestMetricSettingsForAzureResourceApi(r *http.Request, opts config.Opts) (RequestMetricSettings, error) { 50 | settings, err := NewRequestMetricSettings(r, opts) 51 | if err != nil { 52 | return settings, err 53 | } 54 | 55 | if r.URL.Path == config.ProbeMetricsResourceUrl { 56 | return settings, nil 57 | } else if settings.ResourceType != "" && settings.Filter != "" { 58 | return settings, fmt.Errorf("parameter \"resourceType\" and \"filter\" are mutually exclusive") 59 | } else if settings.ResourceType != "" { 60 | settings.Filter = fmt.Sprintf( 61 | "resourceType eq '%s'", 62 | strings.ReplaceAll(settings.ResourceType, "'", "\\'"), 63 | ) 64 | } else if settings.Filter == "" { 65 | return settings, fmt.Errorf("parameter \"resourceType\" or \"filter\" is missing") 66 | } 67 | 68 | return settings, nil 69 | } 70 | 71 | func NewRequestMetricSettings(r *http.Request, opts config.Opts) (RequestMetricSettings, error) { 72 | ret := RequestMetricSettings{ 73 | // force lowercasing of dimensions 74 | DimensionLowercase: opts.Metrics.Dimensions.Lowercase, 75 | } 76 | 77 | params := r.URL.Query() 78 | 79 | // param name 80 | ret.Name = paramsGetWithDefault(params, "name", PrometheusMetricNameDefault) 81 | 82 | // param subscription 83 | if subscriptionList, err := paramsGetListRequired(params, "subscription"); err == nil { 84 | for _, subscription := range subscriptionList { 85 | subscription = strings.TrimSpace(subscription) 86 | ret.Subscriptions = append(ret.Subscriptions, subscription) 87 | } 88 | } else { 89 | return ret, err 90 | } 91 | 92 | // param region 93 | if val, err := paramsGetList(params, "region"); err == nil { 94 | ret.Regions = val 95 | } else { 96 | return ret, err 97 | } 98 | 99 | // param filter 100 | ret.ResourceType = paramsGetWithDefault(params, "resourceType", "") 101 | ret.Filter = paramsGetWithDefault(params, "filter", "") 102 | if val, err := strconv.ParseBool(paramsGetWithDefault(params, "validateDimensions", "true")); err == nil { 103 | ret.ValidateDimensions = val 104 | } else { 105 | return ret, err 106 | } 107 | 108 | // param timespan 109 | ret.Timespan = paramsGetWithDefault(params, "timespan", "PT1M") 110 | 111 | // param interval 112 | if val := params.Get("interval"); val != "" { 113 | ret.Interval = &val 114 | } 115 | 116 | // param metric 117 | if val, err := paramsGetList(params, "metric"); err == nil { 118 | ret.Metrics = val 119 | } else { 120 | return ret, err 121 | } 122 | 123 | // param metricNamespace 124 | ret.MetricNamespace = paramsGetWithDefault(params, "metricNamespace", "") 125 | 126 | // param aggregation 127 | if val, err := paramsGetList(params, "aggregation"); err == nil { 128 | ret.Aggregations = val 129 | } else { 130 | return ret, err 131 | } 132 | 133 | // param metricTop 134 | if val := params.Get("metricTop"); val != "" { 135 | valInt64, err := strconv.ParseInt(val, 10, 32) 136 | if err != nil { 137 | return ret, err 138 | } 139 | valInt32 := int32(valInt64) 140 | ret.MetricTop = &valInt32 141 | } 142 | 143 | // param metricFilter 144 | ret.MetricFilter = paramsGetWithDefault(params, "metricFilter", "") 145 | 146 | // param metricOrderBy 147 | ret.MetricOrderBy = paramsGetWithDefault(params, "metricOrderBy", "") 148 | 149 | // param template 150 | ret.MetricTemplate = paramsGetWithDefault(params, "template", opts.Metrics.Template) 151 | 152 | // param help 153 | ret.HelpTemplate = paramsGetWithDefault(params, "help", opts.Metrics.Help) 154 | 155 | // param cache (timespan as default) 156 | if opts.Prober.Cache { 157 | cacheDefaultDuration, err := iso8601.FromString(ret.Timespan) 158 | cacheDefaultDurationString := "" 159 | if err == nil { 160 | cacheDefaultDurationString = cacheDefaultDuration.ToDuration().String() 161 | } 162 | 163 | // get value from query (with default from timespan) 164 | cacheDurationString := paramsGetWithDefault(params, "cache", cacheDefaultDurationString) 165 | // only enable caching if value is set 166 | if cacheDurationString != "" { 167 | if val, err := time.ParseDuration(cacheDurationString); err == nil { 168 | ret.Cache = &val 169 | } else { 170 | return ret, err 171 | } 172 | } 173 | } 174 | 175 | return ret, nil 176 | } 177 | 178 | func (s *RequestMetricSettings) CacheDuration(requestTime time.Time) (ret *time.Duration) { 179 | if s.Cache != nil { 180 | bufferDuration := 2 * time.Second 181 | cachedUntilTime := requestTime.Add(*s.Cache).Add(-bufferDuration) 182 | cacheDuration := time.Until(cachedUntilTime) 183 | if cacheDuration.Seconds() > 0 { 184 | ret = &cacheDuration 185 | } 186 | } 187 | return 188 | } 189 | 190 | func (s *RequestMetricSettings) SetMetrics(val string) { 191 | s.Metrics = stringToStringList(val, ",") 192 | } 193 | 194 | func (s *RequestMetricSettings) SetAggregations(val string) { 195 | s.Aggregations = stringToStringList(val, ",") 196 | } 197 | -------------------------------------------------------------------------------- /misc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/url" 7 | "strconv" 8 | "strings" 9 | 10 | stringsCommon "github.com/webdevops/go-common/strings" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | func buildContextLoggerFromRequest(r *http.Request) *zap.SugaredLogger { 15 | contextLogger := logger.With(zap.String("requestPath", r.URL.Path)) 16 | 17 | for name, value := range r.URL.Query() { 18 | fieldName := fmt.Sprintf("param%s", stringsCommon.UppercaseFirst(name)) 19 | fieldValue := value 20 | 21 | contextLogger = contextLogger.With(zap.Any(fieldName, fieldValue)) 22 | } 23 | 24 | return contextLogger 25 | } 26 | 27 | func getPrometheusTimeout(r *http.Request, defaultTimeout float64) (timeout float64, err error) { 28 | // If a timeout is configured via the Prometheus header, add it to the request. 29 | if v := r.Header.Get("X-Prometheus-Scrape-Timeout-Seconds"); v != "" { 30 | timeout, err = strconv.ParseFloat(v, 64) 31 | if err != nil { 32 | return 33 | } 34 | } 35 | if timeout == 0 { 36 | timeout = defaultTimeout 37 | } 38 | 39 | return 40 | } 41 | 42 | func paramsGetRequired(params url.Values, name string) (value string, err error) { 43 | value = params.Get(name) 44 | if value == "" { 45 | err = fmt.Errorf("parameter \"%v\" is missing", name) 46 | } 47 | 48 | return 49 | } 50 | 51 | func paramsGetList(params url.Values, name string) (list []string, err error) { 52 | for _, v := range params[name] { 53 | list = append(list, strings.Split(v, ",")...) 54 | } 55 | return 56 | } 57 | 58 | func paramsGetListRequired(params url.Values, name string) (list []string, err error) { 59 | list, err = paramsGetList(params, name) 60 | 61 | if len(list) == 0 { 62 | err = fmt.Errorf("parameter \"%v\" is missing", name) 63 | return 64 | } 65 | 66 | return 67 | } 68 | -------------------------------------------------------------------------------- /probe_metrics_list.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" // #nosec G505 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | 13 | "github.com/webdevops/azure-metrics-exporter/config" 14 | "github.com/webdevops/azure-metrics-exporter/metrics" 15 | 16 | "go.uber.org/zap" 17 | ) 18 | 19 | func probeMetricsListHandler(w http.ResponseWriter, r *http.Request) { 20 | var err error 21 | var timeoutSeconds float64 22 | 23 | startTime := time.Now() 24 | contextLogger := buildContextLoggerFromRequest(r) 25 | registry := prometheus.NewRegistry() 26 | 27 | // If a timeout is configured via the Prometheus header, add it to the request. 28 | timeoutSeconds, err = getPrometheusTimeout(r, config.ProbeMetricsListTimeoutDefault) 29 | if err != nil { 30 | contextLogger.Warnln(err) 31 | http.Error(w, fmt.Sprintf("failed to parse timeout from Prometheus header: %s", err), http.StatusBadRequest) 32 | return 33 | } 34 | 35 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds*float64(time.Second))) 36 | defer cancel() 37 | r = r.WithContext(ctx) 38 | 39 | var settings metrics.RequestMetricSettings 40 | if settings, err = metrics.NewRequestMetricSettingsForAzureResourceApi(r, Opts); err != nil { 41 | contextLogger.Warnln(err) 42 | http.Error(w, err.Error(), http.StatusBadRequest) 43 | return 44 | } 45 | 46 | if _, err = paramsGetListRequired(r.URL.Query(), "subscription"); err != nil { 47 | contextLogger.Warnln(err) 48 | http.Error(w, err.Error(), http.StatusBadRequest) 49 | return 50 | } 51 | 52 | prober := metrics.NewMetricProber(ctx, contextLogger, w, &settings, Opts) 53 | prober.SetUserAgent(UserAgent + gitTag) 54 | prober.SetAzureClient(AzureClient) 55 | prober.SetAzureResourceTagManager(AzureResourceTagManager) 56 | prober.SetPrometheusRegistry(registry) 57 | if settings.Cache != nil { 58 | cacheKey := fmt.Sprintf("list:%x", sha1.Sum([]byte(r.URL.String()))) // #nosec G401 59 | prober.EnableMetricsCache(metricsCache, cacheKey, settings.CacheDuration(startTime)) 60 | } 61 | 62 | if Opts.Azure.ServiceDiscovery.CacheDuration.Seconds() > 0 { 63 | prober.EnableServiceDiscoveryCache(azureCache, Opts.Azure.ServiceDiscovery.CacheDuration) 64 | } 65 | 66 | if !prober.FetchFromCache() { 67 | for _, subscription := range settings.Subscriptions { 68 | prober.ServiceDiscovery.FindSubscriptionResources(subscription, settings.Filter) 69 | } 70 | 71 | prober.RegisterSubscriptionCollectFinishCallback(func(subscriptionId string) { 72 | // global stats counter 73 | prometheusCollectTime.With(prometheus.Labels{ 74 | "subscriptionID": subscriptionId, 75 | "handler": config.ProbeMetricsListUrl, 76 | "filter": settings.Filter, 77 | }).Observe(time.Since(startTime).Seconds()) 78 | }) 79 | 80 | prober.Run() 81 | } else { 82 | w.Header().Add("X-metrics-cached", "true") 83 | for _, subscriptionId := range settings.Subscriptions { 84 | prometheusMetricRequests.With(prometheus.Labels{ 85 | "subscriptionID": subscriptionId, 86 | "handler": config.ProbeMetricsListUrl, 87 | "filter": settings.Filter, 88 | "result": "cached", 89 | }).Inc() 90 | } 91 | } 92 | 93 | h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) 94 | h.ServeHTTP(w, r) 95 | 96 | latency := time.Since(startTime) 97 | contextLogger.With( 98 | zap.String("method", r.Method), 99 | zap.Int("status", http.StatusOK), 100 | zap.String("latency", latency.String()), 101 | ).Info("Request handled for /probe/metrics/list") 102 | } 103 | -------------------------------------------------------------------------------- /probe_metrics_resource.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" // #nosec G505 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | 13 | "github.com/webdevops/azure-metrics-exporter/config" 14 | "github.com/webdevops/azure-metrics-exporter/metrics" 15 | 16 | "go.uber.org/zap" 17 | ) 18 | 19 | func probeMetricsResourceHandler(w http.ResponseWriter, r *http.Request) { 20 | var err error 21 | var timeoutSeconds float64 22 | 23 | startTime := time.Now() 24 | contextLogger := buildContextLoggerFromRequest(r) 25 | registry := prometheus.NewRegistry() 26 | 27 | // If a timeout is configured via the Prometheus header, add it to the request. 28 | timeoutSeconds, err = getPrometheusTimeout(r, config.ProbeMetricsResourceTimeoutDefault) 29 | if err != nil { 30 | contextLogger.Warnln(err) 31 | http.Error(w, fmt.Sprintf("failed to parse timeout from Prometheus header: %s", err), http.StatusBadRequest) 32 | return 33 | } 34 | 35 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds*float64(time.Second))) 36 | defer cancel() 37 | r = r.WithContext(ctx) 38 | 39 | var settings metrics.RequestMetricSettings 40 | if settings, err = metrics.NewRequestMetricSettingsForAzureResourceApi(r, Opts); err != nil { 41 | contextLogger.Warnln(err) 42 | http.Error(w, err.Error(), http.StatusBadRequest) 43 | return 44 | } 45 | 46 | if _, err = paramsGetListRequired(r.URL.Query(), "subscription"); err != nil { 47 | contextLogger.Warnln(err) 48 | http.Error(w, err.Error(), http.StatusBadRequest) 49 | return 50 | } 51 | 52 | prober := metrics.NewMetricProber(ctx, contextLogger, w, &settings, Opts) 53 | prober.SetUserAgent(UserAgent + gitTag) 54 | prober.SetAzureClient(AzureClient) 55 | prober.SetAzureResourceTagManager(AzureResourceTagManager) 56 | prober.SetPrometheusRegistry(registry) 57 | if settings.Cache != nil { 58 | cacheKey := fmt.Sprintf("resource:%x", sha1.Sum([]byte(r.URL.String()))) // #nosec G401 59 | prober.EnableMetricsCache(metricsCache, cacheKey, settings.CacheDuration(startTime)) 60 | } 61 | 62 | if resourceList, err := paramsGetListRequired(r.URL.Query(), "target"); err == nil { 63 | targetList := []metrics.MetricProbeTarget{} 64 | for _, resourceId := range resourceList { 65 | targetList = append( 66 | targetList, 67 | metrics.MetricProbeTarget{ 68 | ResourceId: resourceId, 69 | Metrics: settings.Metrics, 70 | Aggregations: settings.Aggregations, 71 | }, 72 | ) 73 | } 74 | prober.AddTarget(targetList...) 75 | } else { 76 | contextLogger.Errorln(err) 77 | http.Error(w, err.Error(), http.StatusBadRequest) 78 | return 79 | } 80 | 81 | if !prober.FetchFromCache() { 82 | prober.RegisterSubscriptionCollectFinishCallback(func(subscriptionId string) { 83 | // global stats counter 84 | prometheusCollectTime.With(prometheus.Labels{ 85 | "subscriptionID": subscriptionId, 86 | "handler": config.ProbeMetricsListUrl, 87 | "filter": settings.Filter, 88 | }).Observe(time.Since(startTime).Seconds()) 89 | }) 90 | 91 | prober.Run() 92 | } else { 93 | w.Header().Add("X-metrics-cached", "true") 94 | for _, subscriptionId := range settings.Subscriptions { 95 | prometheusMetricRequests.With(prometheus.Labels{ 96 | "subscriptionID": subscriptionId, 97 | "handler": config.ProbeMetricsListUrl, 98 | "filter": settings.Filter, 99 | "result": "cached", 100 | }).Inc() 101 | } 102 | } 103 | 104 | h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) 105 | h.ServeHTTP(w, r) 106 | 107 | latency := time.Since(startTime) 108 | contextLogger.With( 109 | zap.String("method", r.Method), 110 | zap.Int("status", http.StatusOK), 111 | zap.String("latency", latency.String()), 112 | ).Info("Request handled for /probe/metrics/resource") 113 | } 114 | -------------------------------------------------------------------------------- /probe_metrics_resourcegraph.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" // #nosec G505 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | 13 | "github.com/webdevops/azure-metrics-exporter/config" 14 | "github.com/webdevops/azure-metrics-exporter/metrics" 15 | 16 | "go.uber.org/zap" 17 | ) 18 | 19 | func probeMetricsResourceGraphHandler(w http.ResponseWriter, r *http.Request) { 20 | var err error 21 | var timeoutSeconds float64 22 | 23 | startTime := time.Now() 24 | contextLogger := buildContextLoggerFromRequest(r) 25 | registry := prometheus.NewRegistry() 26 | 27 | // If a timeout is configured via the Prometheus header, add it to the request. 28 | timeoutSeconds, err = getPrometheusTimeout(r, config.ProbeMetricsResourceGraphTimeoutDefault) 29 | if err != nil { 30 | contextLogger.Warnln(err) 31 | http.Error(w, fmt.Sprintf("failed to parse timeout from Prometheus header: %s", err), http.StatusBadRequest) 32 | return 33 | } 34 | 35 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds*float64(time.Second))) 36 | defer cancel() 37 | r = r.WithContext(ctx) 38 | 39 | var settings metrics.RequestMetricSettings 40 | if settings, err = metrics.NewRequestMetricSettings(r, Opts); err != nil { 41 | contextLogger.Warnln(err) 42 | http.Error(w, err.Error(), http.StatusBadRequest) 43 | return 44 | } 45 | 46 | if _, err = paramsGetListRequired(r.URL.Query(), "subscription"); err != nil { 47 | contextLogger.Warnln(err) 48 | http.Error(w, err.Error(), http.StatusBadRequest) 49 | return 50 | } 51 | 52 | resourceType, err := paramsGetRequired(r.URL.Query(), "resourceType") 53 | if err != nil { 54 | contextLogger.Warnln(err) 55 | http.Error(w, err.Error(), http.StatusBadRequest) 56 | return 57 | } 58 | 59 | prober := metrics.NewMetricProber(ctx, contextLogger, w, &settings, Opts) 60 | prober.SetUserAgent(UserAgent + gitTag) 61 | prober.SetAzureClient(AzureClient) 62 | prober.SetAzureResourceTagManager(AzureResourceTagManager) 63 | prober.SetPrometheusRegistry(registry) 64 | if settings.Cache != nil { 65 | cacheKey := fmt.Sprintf("scrape:%x", sha1.Sum([]byte(r.URL.String()))) // #nosec G401 66 | prober.EnableMetricsCache(metricsCache, cacheKey, settings.CacheDuration(startTime)) 67 | } 68 | 69 | if Opts.Azure.ServiceDiscovery.CacheDuration.Seconds() > 0 { 70 | prober.EnableServiceDiscoveryCache(azureCache, Opts.Azure.ServiceDiscovery.CacheDuration) 71 | } 72 | 73 | if !prober.FetchFromCache() { 74 | err := prober.ServiceDiscovery.FindResourceGraph(ctx, settings.Subscriptions, resourceType, settings.Filter) 75 | if err != nil { 76 | contextLogger.Errorln(err) 77 | http.Error(w, err.Error(), http.StatusBadRequest) 78 | return 79 | } 80 | 81 | prober.RegisterSubscriptionCollectFinishCallback(func(subscriptionId string) { 82 | // global stats counter 83 | prometheusCollectTime.With(prometheus.Labels{ 84 | "subscriptionID": subscriptionId, 85 | "handler": config.ProbeMetricsListUrl, 86 | "filter": settings.Filter, 87 | }).Observe(time.Since(startTime).Seconds()) 88 | }) 89 | 90 | prober.Run() 91 | } else { 92 | w.Header().Add("X-metrics-cached", "true") 93 | for _, subscriptionId := range settings.Subscriptions { 94 | prometheusMetricRequests.With(prometheus.Labels{ 95 | "subscriptionID": subscriptionId, 96 | "handler": config.ProbeMetricsListUrl, 97 | "filter": settings.Filter, 98 | "result": "cached", 99 | }).Inc() 100 | } 101 | } 102 | 103 | h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) 104 | h.ServeHTTP(w, r) 105 | 106 | latency := time.Since(startTime) 107 | contextLogger.With( 108 | zap.String("method", r.Method), 109 | zap.Int("status", http.StatusOK), 110 | zap.String("latency", latency.String()), 111 | ).Info("Request handled for /probe/metrics/resourcegraph") 112 | } 113 | -------------------------------------------------------------------------------- /probe_metrics_scrape.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" // #nosec G505 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | 13 | "github.com/webdevops/azure-metrics-exporter/config" 14 | "github.com/webdevops/azure-metrics-exporter/metrics" 15 | 16 | "go.uber.org/zap" 17 | ) 18 | 19 | func probeMetricsScrapeHandler(w http.ResponseWriter, r *http.Request) { 20 | var err error 21 | var timeoutSeconds float64 22 | var metricTagName, aggregationTagName string 23 | 24 | startTime := time.Now() 25 | contextLogger := buildContextLoggerFromRequest(r) 26 | registry := prometheus.NewRegistry() 27 | 28 | // If a timeout is configured via the Prometheus header, add it to the request. 29 | timeoutSeconds, err = getPrometheusTimeout(r, config.ProbeMetricsScrapeTimeoutDefault) 30 | if err != nil { 31 | contextLogger.Warnln(err) 32 | http.Error(w, fmt.Sprintf("failed to parse timeout from Prometheus header: %s", err), http.StatusBadRequest) 33 | return 34 | } 35 | 36 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds*float64(time.Second))) 37 | defer cancel() 38 | r = r.WithContext(ctx) 39 | 40 | var settings metrics.RequestMetricSettings 41 | if settings, err = metrics.NewRequestMetricSettingsForAzureResourceApi(r, Opts); err != nil { 42 | contextLogger.Warnln(err) 43 | http.Error(w, err.Error(), http.StatusBadRequest) 44 | return 45 | } 46 | 47 | if _, err = paramsGetListRequired(r.URL.Query(), "subscription"); err != nil { 48 | contextLogger.Warnln(err) 49 | http.Error(w, err.Error(), http.StatusBadRequest) 50 | return 51 | } 52 | 53 | if metricTagName, err = paramsGetRequired(r.URL.Query(), "metricTagName"); err != nil { 54 | contextLogger.Warnln(err) 55 | http.Error(w, err.Error(), http.StatusBadRequest) 56 | return 57 | } 58 | if aggregationTagName, err = paramsGetRequired(r.URL.Query(), "aggregationTagName"); err != nil { 59 | contextLogger.Warnln(err) 60 | http.Error(w, err.Error(), http.StatusBadRequest) 61 | return 62 | } 63 | 64 | prober := metrics.NewMetricProber(ctx, contextLogger, w, &settings, Opts) 65 | prober.SetUserAgent(UserAgent + gitTag) 66 | prober.SetAzureClient(AzureClient) 67 | prober.SetAzureResourceTagManager(AzureResourceTagManager) 68 | prober.SetPrometheusRegistry(registry) 69 | if settings.Cache != nil { 70 | cacheKey := fmt.Sprintf("scrape:%x", sha1.Sum([]byte(r.URL.String()))) // #nosec G401 71 | prober.EnableMetricsCache(metricsCache, cacheKey, settings.CacheDuration(startTime)) 72 | } 73 | 74 | if Opts.Azure.ServiceDiscovery.CacheDuration.Seconds() > 0 { 75 | prober.EnableServiceDiscoveryCache(azureCache, Opts.Azure.ServiceDiscovery.CacheDuration) 76 | } 77 | 78 | if !prober.FetchFromCache() { 79 | for _, subscription := range settings.Subscriptions { 80 | prober.ServiceDiscovery.FindSubscriptionResourcesWithScrapeTags(ctx, subscription, settings.Filter, metricTagName, aggregationTagName) 81 | } 82 | 83 | prober.RegisterSubscriptionCollectFinishCallback(func(subscriptionId string) { 84 | // global stats counter 85 | prometheusCollectTime.With(prometheus.Labels{ 86 | "subscriptionID": subscriptionId, 87 | "handler": config.ProbeMetricsListUrl, 88 | "filter": settings.Filter, 89 | }).Observe(time.Since(startTime).Seconds()) 90 | }) 91 | 92 | prober.Run() 93 | } else { 94 | w.Header().Add("X-metrics-cached", "true") 95 | for _, subscriptionId := range settings.Subscriptions { 96 | prometheusMetricRequests.With(prometheus.Labels{ 97 | "subscriptionID": subscriptionId, 98 | "handler": config.ProbeMetricsListUrl, 99 | "filter": settings.Filter, 100 | "result": "cached", 101 | }).Inc() 102 | } 103 | } 104 | 105 | h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) 106 | h.ServeHTTP(w, r) 107 | 108 | latency := time.Since(startTime) 109 | contextLogger.With( 110 | zap.String("method", r.Method), 111 | zap.Int("status", http.StatusOK), 112 | zap.String("latency", latency.String()), 113 | ).Info("Request handled for /probe/metrics/scrape") 114 | } 115 | -------------------------------------------------------------------------------- /probe_metrics_subscription.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "crypto/sha1" // #nosec G505 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promhttp" 12 | 13 | "github.com/webdevops/azure-metrics-exporter/config" 14 | "github.com/webdevops/azure-metrics-exporter/metrics" 15 | 16 | "go.uber.org/zap" 17 | ) 18 | 19 | func probeMetricsSubscriptionHandler(w http.ResponseWriter, r *http.Request) { 20 | var err error 21 | var timeoutSeconds float64 22 | 23 | startTime := time.Now() 24 | contextLogger := buildContextLoggerFromRequest(r) 25 | registry := prometheus.NewRegistry() 26 | 27 | // If a timeout is configured via the Prometheus header, add it to the request. 28 | timeoutSeconds, err = getPrometheusTimeout(r, config.ProbeMetricsSubscriptionTimeoutDefault) 29 | if err != nil { 30 | contextLogger.Warn(err) 31 | http.Error(w, fmt.Sprintf("failed to parse timeout from Prometheus header: %s", err), http.StatusBadRequest) 32 | return 33 | } 34 | 35 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSeconds*float64(time.Second))) 36 | defer cancel() 37 | r = r.WithContext(ctx) 38 | 39 | var settings metrics.RequestMetricSettings 40 | if settings, err = metrics.NewRequestMetricSettingsForAzureResourceApi(r, Opts); err != nil { 41 | contextLogger.Warnln(err) 42 | http.Error(w, err.Error(), http.StatusBadRequest) 43 | return 44 | } 45 | 46 | if _, err = paramsGetListRequired(r.URL.Query(), "subscription"); err != nil { 47 | contextLogger.Warnln(err) 48 | http.Error(w, err.Error(), http.StatusBadRequest) 49 | return 50 | } 51 | 52 | prober := metrics.NewMetricProber(ctx, contextLogger, w, &settings, Opts) 53 | prober.SetUserAgent(UserAgent + gitTag) 54 | prober.SetAzureClient(AzureClient) 55 | prober.SetAzureResourceTagManager(AzureResourceTagManager) 56 | prober.SetPrometheusRegistry(registry) 57 | if settings.Cache != nil { 58 | cacheKey := fmt.Sprintf("list:%x", sha1.Sum([]byte(r.URL.String()))) // #nosec G401 59 | prober.EnableMetricsCache(metricsCache, cacheKey, settings.CacheDuration(startTime)) 60 | } 61 | 62 | if !prober.FetchFromCache() { 63 | prober.RegisterSubscriptionCollectFinishCallback(func(subscriptionId string) { 64 | // global stats counter 65 | prometheusCollectTime.With(prometheus.Labels{ 66 | "subscriptionID": subscriptionId, 67 | "handler": config.ProbeMetricsListUrl, 68 | "filter": settings.Filter, 69 | }).Observe(time.Since(startTime).Seconds()) 70 | }) 71 | 72 | prober.RunOnSubscriptionScope() 73 | } else { 74 | w.Header().Add("X-metrics-cached", "true") 75 | for _, subscriptionId := range settings.Subscriptions { 76 | prometheusMetricRequests.With(prometheus.Labels{ 77 | "subscriptionID": subscriptionId, 78 | "handler": config.ProbeMetricsListUrl, 79 | "filter": settings.Filter, 80 | "result": "cached", 81 | }).Inc() 82 | } 83 | } 84 | 85 | h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) 86 | h.ServeHTTP(w, r) 87 | 88 | latency := time.Since(startTime) 89 | contextLogger.With( 90 | zap.String("method", r.Method), 91 | zap.Int("status", http.StatusOK), 92 | zap.String("latency", latency.String()), 93 | ).Info("Request handled for /probe/metrics/subscription") 94 | } 95 | -------------------------------------------------------------------------------- /templates/query.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 114 | azure-metrics-exporter 115 | 116 | 117 | 118 | 137 | 138 |
139 |
140 |

141 | Query settings 142 |

143 | 144 |
145 | 146 |
147 |

General

148 |
149 | 150 |
151 | 152 |
153 | 161 |
azure-metrics-exporter query endpoint
162 |
163 |
164 | 165 |
166 | 167 |
168 | 169 |
Name of metric
170 |
171 |
172 | 173 |
174 | 175 |
176 | 177 |
Metric template support
178 |
179 |
180 | 181 |
182 | 183 |
184 | 185 |
Help text (with template support)
186 |
187 |
188 | 189 |
190 | 191 |
192 | 193 |
Specifies how long metric result should be cached (if caching is enabled)
194 |
195 |
196 | 197 |
198 |

Service Discovery

199 |
200 | 201 |
202 | 203 |
204 | 205 |
List of Azure subscriptions
206 |
207 |
208 | 209 |
210 | 211 |
212 | 213 |
Static target (for /probe/metrics/resource)
214 |
215 |
216 | 217 |
218 | 219 |
220 | 221 |
Azure Resource Type query eg Microsoft.KeyVault/vaults (for service discovery)
222 |
223 |
224 | 225 |
226 | 227 |
228 | 229 |
Regions (eg. global, westeurope, northeurope)
230 |
231 |
232 | 233 |
234 | 235 |
236 | 237 |
Additional filter statement (Kusto statement for /probe/metrics/resourcegraph; Azure API Resource List $filter for rest)
238 |
239 |
240 | 241 |
242 | 243 |
244 | 245 |
Additional metric namespace for namespaced metrics (eg. Azure StorageAccount sub metrics)
246 |
247 |
248 | 249 | 250 |
251 |

Metric settings

252 |
253 | 254 |
255 | 256 |
257 | 258 |
Specifies which Azure metrics should be fetched
259 |
260 |
261 | 262 | 263 |
264 | 265 |
266 | 267 |
Metric interval
268 |
269 |
270 | 271 |
272 | 273 |
274 | 275 |
Metric timeframe
276 |
277 |
278 | 279 |
280 | 281 |
282 | 285 |
Metric aggregation
286 |
287 |
288 | 289 | 290 |
291 |

Dimension support

292 |
293 | 294 |
295 | 296 |
297 | 298 |
Dimension support: filter for metric splitting (eg ConnectionName eq '*')
299 |
300 |
301 | 302 |
303 | 304 |
305 | 306 |
Dimension support: number of fetched dimension rows
307 |
308 |
309 | 310 |
311 | 312 |
313 | 317 |
Dimension support: disable server side validation for dimensions
318 |
319 |
320 | 321 |
322 |
323 | 324 |
325 |
326 |
327 |
328 | 329 |
330 | 331 |

Result

332 | 333 |
334 | 335 |
336 | 337 |
338 |
339 | 340 |
341 | 342 |
343 | 344 |
345 |
346 | 347 |
348 | 349 |
350 | 351 |
352 |
353 |
354 | 355 |
356 |

Prometheus scrape_config

357 | 358 |
359 | 360 |
361 | 362 |
363 |
364 |
365 | 366 | 367 |
368 | 369 | 370 | 371 | 372 | 373 | 374 | 515 | 516 | 517 | --------------------------------------------------------------------------------