├── .circleci └── config.yml ├── .github ├── dependabot.yml └── workflows │ ├── codeql-analysis.yml │ ├── container_description.yml │ └── golangci-lint.yml ├── .gitignore ├── .golangci.yml ├── .promu.yml ├── .yamllint ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── MAINTAINERS.md ├── Makefile ├── Makefile.common ├── NOTICE ├── README.md ├── SECURITY.md ├── VERSION ├── cmd ├── avalanche │ └── avalanche.go └── mtypes │ ├── README.md │ ├── exampleprometheustarget.txt │ ├── main.go │ └── main_test.go ├── example └── kubernetes-deployment.yaml ├── go.mod ├── go.sum ├── metrics ├── serve.go ├── serve_test.go ├── write.go ├── write_test.go └── writev2.go ├── pkg └── errors │ └── errors.go └── scripts └── errcheck_excludes.txt /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2.1 3 | 4 | orbs: 5 | prometheus: prometheus/prometheus@0.17.1 6 | 7 | executors: 8 | # Whenever the Go version is updated here, .promu.yml should also be updated. 9 | golang: 10 | docker: 11 | - image: quay.io/prometheus/golang-builder:1.22-base 12 | 13 | jobs: 14 | test: 15 | executor: golang 16 | 17 | steps: 18 | - prometheus/setup_environment 19 | - run: make 20 | - prometheus/store_artifact: 21 | file: avalanche 22 | - run: git diff --exit-code 23 | 24 | workflows: 25 | version: 2 26 | avalanche: 27 | jobs: 28 | - test: 29 | filters: 30 | tags: 31 | only: /.*/ 32 | - prometheus/build: 33 | name: build 34 | filters: 35 | tags: 36 | only: /.*/ 37 | - prometheus/publish_main: 38 | context: org-context 39 | docker_hub_organization: prometheuscommunity 40 | quay_io_organization: prometheuscommunity 41 | requires: 42 | - test 43 | - build 44 | filters: 45 | branches: 46 | only: main 47 | - prometheus/publish_release: 48 | context: org-context 49 | docker_hub_organization: prometheuscommunity 50 | quay_io_organization: prometheuscommunity 51 | requires: 52 | - test 53 | - build 54 | filters: 55 | tags: 56 | only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/ 57 | branches: 58 | ignore: /.*/ 59 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [main, release-*] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [main] 20 | schedule: 21 | - cron: "26 12 * * 1" 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | 28 | strategy: 29 | fail-fast: false 30 | matrix: 31 | language: ["go"] 32 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 33 | # Learn more: 34 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 35 | 36 | steps: 37 | - name: Checkout repository 38 | uses: actions/checkout@v2 39 | 40 | # Initializes the CodeQL tools for scanning. 41 | - name: Initialize CodeQL 42 | uses: github/codeql-action/init@v1 43 | with: 44 | languages: ${{ matrix.language }} 45 | # If you wish to specify custom queries, you can do so here or in a config file. 46 | # By default, queries listed here will override any specified in a config file. 47 | # Prefix the list here with "+" to use these queries and those in the config file. 48 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 49 | 50 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 51 | # If this step fails, then you should remove it and run the build manually (see below) 52 | - name: Autobuild 53 | uses: github/codeql-action/autobuild@v1 54 | 55 | # ℹ️ Command-line programs to run using the OS shell. 56 | # 📚 https://git.io/JvXDl 57 | 58 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 59 | # and modify them (or add more) to build your code if your project 60 | # uses a compiled language 61 | 62 | #- run: | 63 | # make bootstrap 64 | # make release 65 | 66 | - name: Perform CodeQL Analysis 67 | uses: github/codeql-action/analyze@v1 68 | -------------------------------------------------------------------------------- /.github/workflows/container_description.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Push README to Docker Hub 3 | on: 4 | push: 5 | paths: 6 | - "README.md" 7 | - "README-containers.md" 8 | - ".github/workflows/container_description.yml" 9 | branches: [ main, master ] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | PushDockerHubReadme: 16 | runs-on: ubuntu-latest 17 | name: Push README to Docker Hub 18 | if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. 19 | steps: 20 | - name: git checkout 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | - name: Set docker hub repo name 23 | run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV 24 | - name: Push README to Dockerhub 25 | uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1 26 | env: 27 | DOCKER_USER: ${{ secrets.DOCKER_HUB_LOGIN }} 28 | DOCKER_PASS: ${{ secrets.DOCKER_HUB_PASSWORD }} 29 | with: 30 | destination_container_repo: ${{ env.DOCKER_REPO_NAME }} 31 | provider: dockerhub 32 | short_description: ${{ env.DOCKER_REPO_NAME }} 33 | # Empty string results in README-containers.md being pushed if it 34 | # exists. Otherwise, README.md is pushed. 35 | readme_file: '' 36 | 37 | PushQuayIoReadme: 38 | runs-on: ubuntu-latest 39 | name: Push README to quay.io 40 | if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. 41 | steps: 42 | - name: git checkout 43 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 44 | - name: Set quay.io org name 45 | run: echo "DOCKER_REPO=$(echo quay.io/${GITHUB_REPOSITORY_OWNER} | tr -d '-')" >> $GITHUB_ENV 46 | - name: Set quay.io repo name 47 | run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV 48 | - name: Push README to quay.io 49 | uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1 50 | env: 51 | DOCKER_APIKEY: ${{ secrets.QUAY_IO_API_TOKEN }} 52 | with: 53 | destination_container_repo: ${{ env.DOCKER_REPO_NAME }} 54 | provider: quay 55 | # Empty string results in README-containers.md being pushed if it 56 | # exists. Otherwise, README.md is pushed. 57 | readme_file: '' 58 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This action is synced from https://github.com/prometheus/prometheus 3 | name: golangci-lint 4 | on: 5 | push: 6 | paths: 7 | - "go.sum" 8 | - "go.mod" 9 | - "**.go" 10 | - "scripts/errcheck_excludes.txt" 11 | - ".github/workflows/golangci-lint.yml" 12 | - ".golangci.yml" 13 | pull_request: 14 | 15 | permissions: # added using https://github.com/step-security/secure-repo 16 | contents: read 17 | 18 | jobs: 19 | golangci: 20 | permissions: 21 | contents: read # for actions/checkout to fetch code 22 | pull-requests: read # for golangci/golangci-lint-action to fetch pull requests 23 | name: lint 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 28 | - name: Install Go 29 | uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 30 | with: 31 | go-version: 1.23.x 32 | - name: Install snmp_exporter/generator dependencies 33 | run: sudo apt-get update && sudo apt-get -y install libsnmp-dev 34 | if: github.repository == 'prometheus/snmp_exporter' 35 | - name: Lint 36 | uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 37 | with: 38 | args: --verbose 39 | version: v1.63.4 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ./avalanche 2 | .build/ 3 | .idea/ 4 | mtypes 5 | avalanche -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | deadline: 5m 3 | skip-files: 4 | # Skip autogenerated files. 5 | - ^.*\.(pb|y)\.go$ 6 | 7 | output: 8 | sort-results: true 9 | 10 | linters: 11 | enable: 12 | - gofumpt 13 | - goimports 14 | - revive 15 | 16 | issues: 17 | exclude-rules: 18 | - path: _test.go 19 | linters: 20 | - errcheck 21 | 22 | linters-settings: 23 | errcheck: 24 | exclude: scripts/errcheck_excludes.txt 25 | goimports: 26 | local-prefixes: github.com/prometheus-community/avalanche 27 | gofumpt: 28 | extra-rules: true 29 | -------------------------------------------------------------------------------- /.promu.yml: -------------------------------------------------------------------------------- 1 | go: 2 | # Whenever the Go version is updated here, 3 | # .circle/config.yml should also be updated. 4 | version: 1.22 5 | repository: 6 | path: github.com/prometheus-community/avalanche 7 | build: 8 | binaries: 9 | - name: avalanche 10 | path: ./cmd/avalanche 11 | - name: mtypes 12 | path: ./cmd/mtypes 13 | ldflags: | 14 | -X github.com/prometheus/common/version.Version={{.Version}} 15 | -X github.com/prometheus/common/version.Revision={{.Revision}} 16 | -X github.com/prometheus/common/version.Branch={{.Branch}} 17 | -X github.com/prometheus/common/version.BuildUser={{user}}@{{host}} 18 | -X github.com/prometheus/common/version.BuildDate={{date "20060102-15:04:05"}} 19 | tarball: 20 | files: 21 | - LICENSE 22 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | ignore: | 4 | **/node_modules 5 | 6 | rules: 7 | braces: 8 | max-spaces-inside: 1 9 | level: error 10 | brackets: 11 | max-spaces-inside: 1 12 | level: error 13 | commas: disable 14 | comments: disable 15 | comments-indentation: disable 16 | document-start: disable 17 | indentation: 18 | spaces: consistent 19 | indent-sequences: consistent 20 | key-duplicates: 21 | ignore: | 22 | config/testdata/section_key_dup.bad.yml 23 | line-length: disable 24 | truthy: 25 | check-keys: false 26 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## unreleased 2 | 3 | ## 0.7.0 / 2025-01-14 4 | 5 | * [CHANGE] (breaking) Removed the deprecated `--metric-count` flag (use `--gauge-metric-count` instead). #119 6 | * [CHANGE] (breaking) Removed `remote-pprof-urls` feature. #104 7 | * [BUGFIX] Fixed `--const-label` feature. #113 8 | * [BUGFIX] Fixed remote write run forever mode. #104 9 | * [CHANGE] Remote write errors are logged on an ERROR level as they go (instead of the long interval). #102 10 | * [FEATURE] Added `--remote-concurrency-limit` flag (20 default) on how many concurrent remote writes can happen. #46 11 | 12 | ## 0.6.0 / 2024-10-14 13 | 14 | * [CHANGE] (breaking) `--metric-interval` default value is now zero (not 120). #99 15 | * [CHANGE] (breaking) Change `out-of-order` to `--remote-out-of-order` for consistency. #101 16 | * [CHANGE] Install command for `avalanche` moved to `cmd/avalanche/` from `cmd/` #97 17 | * [FEATURE] Add `mtypes` binary for metric type calculation against targets #97 18 | * [FEATURE] `--remote-requests-count` value -1 now makes remote-write send requests indefinitely #90 19 | * [FEATURE] Add support for all metric types; deprecated --metric-count flag; --*-interval flags set to 0 means no change; added OpenMetrics support #80 #101 20 | 21 | ## 0.5.0 / 2024-09-15 22 | 23 | * [FEATURE] add two new operation modes for metrics generation and cycling; gradual-change between min and max and double-halve #64 24 | * (other changes, not captured due to lost 0.4.0 tag) 25 | 26 | ## 0.4.0 / 2022-03-08 27 | 28 | Initial release under the Prometheus-community org. 29 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Prometheus Community Code of Conduct 2 | 3 | Prometheus follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md). 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for taking the time to contribute! When contributing to this repository, please first 4 | discuss the change you wish to make via issue, email, or any other method with the owners of this 5 | repository before making a change. 6 | 7 | Before you submit any PRs, we encourage you to read the instructions for the [developer certificate of origin (DCO)](DCO-SIGNOFF.md), 8 | as well as the guidelines below: 9 | 10 | Please note we have a code of conduct, please follow it in all your interactions with the project. 11 | 12 | ## Pull Request Process 13 | 14 | 1. Ensure any install or build dependencies are removed before the end of the layer when doing a 15 | build. 16 | 2. Update the README.md with details of changes to the interface, this includes new environment 17 | variables, exposed ports, useful file locations and container parameters. 18 | 3. Increase the version numbers in any examples files and the README.md to the new version that this 19 | Pull Request would represent. The versioning scheme we use is [SemVer](http://semver.org/). 20 | 4. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 21 | do not have permission to do that, you may request the second reviewer to merge it for you. 22 | 23 | ## Code of Conduct 24 | 25 | ### Our Pledge 26 | 27 | In the interest of fostering an open and welcoming environment, we as 28 | contributors and maintainers pledge to making participation in our project and 29 | our community a harassment-free experience for everyone, regardless of age, body 30 | size, disability, ethnicity, gender identity and expression, level of experience, 31 | nationality, personal appearance, race, religion, or sexual identity and 32 | orientation. 33 | 34 | ### Our Standards 35 | 36 | Examples of behavior that contributes to creating a positive environment 37 | include: 38 | 39 | * Using welcoming and inclusive language 40 | * Being respectful of differing viewpoints and experiences 41 | * Gracefully accepting constructive criticism 42 | * Focusing on what is best for the community 43 | * Showing empathy towards other community members 44 | 45 | Examples of unacceptable behavior by participants include: 46 | 47 | * The use of sexualized language or imagery and unwelcome sexual attention or 48 | advances 49 | * Trolling, insulting/derogatory comments, and personal or political attacks 50 | * Public or private harassment 51 | * Publishing others' private information, such as a physical or electronic 52 | address, without explicit permission 53 | * Other conduct which could reasonably be considered inappropriate in a 54 | professional setting 55 | 56 | ### Our Responsibilities 57 | 58 | Project maintainers are responsible for clarifying the standards of acceptable 59 | behavior and are expected to take appropriate and fair corrective action in 60 | response to any instances of unacceptable behavior. 61 | 62 | Project maintainers have the right and responsibility to remove, edit, or 63 | reject comments, commits, code, wiki edits, issues, and other contributions 64 | that are not aligned to this Code of Conduct, or to ban temporarily or 65 | permanently any contributor for other behaviors that they deem inappropriate, 66 | threatening, offensive, or harmful. 67 | 68 | ### Scope 69 | 70 | This Code of Conduct applies both within project spaces and in public spaces 71 | when an individual is representing the project or its community. Examples of 72 | representing a project or community include using an official project e-mail 73 | address, posting via an official social media account, or acting as an appointed 74 | representative at an online or offline event. Representation of a project may be 75 | further defined and clarified by project maintainers. 76 | 77 | ### Enforcement 78 | 79 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 80 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 81 | complaints will be reviewed and investigated and will result in a response that 82 | is deemed necessary and appropriate to the circumstances. The project team is 83 | obligated to maintain confidentiality with regard to the reporter of an incident. 84 | Further details of specific enforcement policies may be posted separately. 85 | 86 | Project maintainers who do not follow or enforce the Code of Conduct in good 87 | faith may face temporary or permanent repercussions as determined by other 88 | members of the project's leadership. 89 | 90 | ### Attribution 91 | 92 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 93 | available at [http://contributor-covenant.org/version/1/4][version] 94 | 95 | [homepage]: http://contributor-covenant.org 96 | [version]: http://contributor-covenant.org/version/1/4/ 97 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG ARCH="amd64" 2 | ARG OS="linux" 3 | FROM quay.io/prometheus/busybox-${OS}-${ARCH}:latest 4 | LABEL maintainer="The Prometheus Authors " 5 | 6 | ARG ARCH="amd64" 7 | ARG OS="linux" 8 | COPY .build/${OS}-${ARCH}/avalanche /bin/avalanche 9 | COPY .build/${OS}-${ARCH}/mtypes /bin/mtypes 10 | 11 | EXPOSE 9101 12 | USER nobody 13 | ENTRYPOINT [ "/bin/avalanche" ] 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | * Bartek Płotka @bwplotka 2 | * Saswata Mukherjee @saswatamcode 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Ensure that 'all' is the default target otherwise it will be the first target from Makefile.common. 2 | all:: 3 | 4 | # Needs to be defined before including Makefile.common to auto-generate targets 5 | DOCKER_ARCHS ?= amd64 armv7 arm64 ppc64le s390x 6 | DOCKER_REPO ?= prometheuscommunity 7 | 8 | include Makefile.common 9 | 10 | DOCKER_IMAGE_NAME ?= avalanche 11 | 12 | GOIMPORTS = goimports 13 | $(GOIMPORTS): 14 | @go install golang.org/x/tools/cmd/goimports@latest 15 | 16 | GOFUMPT = gofumpt 17 | $(GOFUMPT): 18 | @go install mvdan.cc/gofumpt@latest 19 | 20 | GO_FILES = $(shell find . -path ./vendor -prune -o -name '*.go' -print) 21 | 22 | .PHONY: format 23 | format: $(GOFUMPT) $(GOIMPORTS) 24 | @echo ">> formating imports)" 25 | @$(GOIMPORTS) -local github.com/prometheus-community/avalanche -w $(GO_FILES) 26 | @echo ">> gofumpt-ing the code; golangci-lint requires this" 27 | @$(GOFUMPT) -extra -w $(GO_FILES) 28 | -------------------------------------------------------------------------------- /Makefile.common: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Prometheus Authors 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | # A common Makefile that includes rules to be reused in different prometheus projects. 16 | # !!! Open PRs only against the prometheus/prometheus/Makefile.common repository! 17 | 18 | # Example usage : 19 | # Create the main Makefile in the root project directory. 20 | # include Makefile.common 21 | # customTarget: 22 | # @echo ">> Running customTarget" 23 | # 24 | 25 | # Ensure GOBIN is not set during build so that promu is installed to the correct path 26 | unexport GOBIN 27 | 28 | GO ?= go 29 | GOFMT ?= $(GO)fmt 30 | FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH))) 31 | GOOPTS ?= 32 | GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) 33 | GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) 34 | 35 | GO_VERSION ?= $(shell $(GO) version) 36 | GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) 37 | PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') 38 | 39 | PROMU := $(FIRST_GOPATH)/bin/promu 40 | pkgs = ./... 41 | 42 | ifeq (arm, $(GOHOSTARCH)) 43 | GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM) 44 | GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM) 45 | else 46 | GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) 47 | endif 48 | 49 | GOTEST := $(GO) test 50 | GOTEST_DIR := 51 | ifneq ($(CIRCLE_JOB),) 52 | ifneq ($(shell command -v gotestsum 2> /dev/null),) 53 | GOTEST_DIR := test-results 54 | GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml -- 55 | endif 56 | endif 57 | 58 | PROMU_VERSION ?= 0.17.0 59 | PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz 60 | 61 | SKIP_GOLANGCI_LINT := 62 | GOLANGCI_LINT := 63 | GOLANGCI_LINT_OPTS ?= 64 | GOLANGCI_LINT_VERSION ?= v1.63.4 65 | # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. 66 | # windows isn't included here because of the path separator being different. 67 | ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) 68 | ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64)) 69 | # If we're in CI and there is an Actions file, that means the linter 70 | # is being run in Actions, so we don't need to run it here. 71 | ifneq (,$(SKIP_GOLANGCI_LINT)) 72 | GOLANGCI_LINT := 73 | else ifeq (,$(CIRCLE_JOB)) 74 | GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint 75 | else ifeq (,$(wildcard .github/workflows/golangci-lint.yml)) 76 | GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint 77 | endif 78 | endif 79 | endif 80 | 81 | PREFIX ?= $(shell pwd) 82 | BIN_DIR ?= $(shell pwd) 83 | DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) 84 | DOCKERFILE_PATH ?= ./Dockerfile 85 | DOCKERBUILD_CONTEXT ?= ./ 86 | DOCKER_REPO ?= prom 87 | 88 | DOCKER_ARCHS ?= amd64 89 | 90 | BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS)) 91 | PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS)) 92 | TAG_DOCKER_ARCHS = $(addprefix common-docker-tag-latest-,$(DOCKER_ARCHS)) 93 | 94 | SANITIZED_DOCKER_IMAGE_TAG := $(subst +,-,$(DOCKER_IMAGE_TAG)) 95 | 96 | ifeq ($(GOHOSTARCH),amd64) 97 | ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows)) 98 | # Only supported on amd64 99 | test-flags := -race 100 | endif 101 | endif 102 | 103 | # This rule is used to forward a target like "build" to "common-build". This 104 | # allows a new "build" target to be defined in a Makefile which includes this 105 | # one and override "common-build" without override warnings. 106 | %: common-% ; 107 | 108 | .PHONY: common-all 109 | common-all: precheck style check_license lint yamllint unused build test 110 | 111 | .PHONY: common-style 112 | common-style: 113 | @echo ">> checking code style" 114 | @fmtRes=$$($(GOFMT) -d $$(find . -path ./vendor -prune -o -name '*.go' -print)); \ 115 | if [ -n "$${fmtRes}" ]; then \ 116 | echo "gofmt checking failed!"; echo "$${fmtRes}"; echo; \ 117 | echo "Please ensure you are using $$($(GO) version) for formatting code."; \ 118 | exit 1; \ 119 | fi 120 | 121 | .PHONY: common-check_license 122 | common-check_license: 123 | @echo ">> checking license header" 124 | @licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \ 125 | awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \ 126 | done); \ 127 | if [ -n "$${licRes}" ]; then \ 128 | echo "license header checking failed:"; echo "$${licRes}"; \ 129 | exit 1; \ 130 | fi 131 | 132 | .PHONY: common-deps 133 | common-deps: 134 | @echo ">> getting dependencies" 135 | $(GO) mod download 136 | 137 | .PHONY: update-go-deps 138 | update-go-deps: 139 | @echo ">> updating Go dependencies" 140 | @for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ 141 | $(GO) get -d $$m; \ 142 | done 143 | $(GO) mod tidy 144 | 145 | .PHONY: common-test-short 146 | common-test-short: $(GOTEST_DIR) 147 | @echo ">> running short tests" 148 | $(GOTEST) -short $(GOOPTS) $(pkgs) 149 | 150 | .PHONY: common-test 151 | common-test: $(GOTEST_DIR) 152 | @echo ">> running all tests" 153 | $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs) 154 | 155 | $(GOTEST_DIR): 156 | @mkdir -p $@ 157 | 158 | .PHONY: common-format 159 | common-format: 160 | @echo ">> formatting code" 161 | $(GO) fmt $(pkgs) 162 | 163 | .PHONY: common-vet 164 | common-vet: 165 | @echo ">> vetting code" 166 | $(GO) vet $(GOOPTS) $(pkgs) 167 | 168 | .PHONY: common-lint 169 | common-lint: $(GOLANGCI_LINT) 170 | ifdef GOLANGCI_LINT 171 | @echo ">> running golangci-lint" 172 | $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs) 173 | endif 174 | 175 | .PHONY: common-lint-fix 176 | common-lint-fix: $(GOLANGCI_LINT) 177 | ifdef GOLANGCI_LINT 178 | @echo ">> running golangci-lint fix" 179 | $(GOLANGCI_LINT) run --fix $(GOLANGCI_LINT_OPTS) $(pkgs) 180 | endif 181 | 182 | .PHONY: common-yamllint 183 | common-yamllint: 184 | @echo ">> running yamllint on all YAML files in the repository" 185 | ifeq (, $(shell command -v yamllint 2> /dev/null)) 186 | @echo "yamllint not installed so skipping" 187 | else 188 | yamllint . 189 | endif 190 | 191 | # For backward-compatibility. 192 | .PHONY: common-staticcheck 193 | common-staticcheck: lint 194 | 195 | .PHONY: common-unused 196 | common-unused: 197 | @echo ">> running check for unused/missing packages in go.mod" 198 | $(GO) mod tidy 199 | @git diff --exit-code -- go.sum go.mod 200 | 201 | .PHONY: common-build 202 | common-build: promu 203 | @echo ">> building binaries" 204 | $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) 205 | 206 | .PHONY: common-tarball 207 | common-tarball: promu 208 | @echo ">> building release tarball" 209 | $(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR) 210 | 211 | .PHONY: common-docker-repo-name 212 | common-docker-repo-name: 213 | @echo "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)" 214 | 215 | .PHONY: common-docker $(BUILD_DOCKER_ARCHS) 216 | common-docker: $(BUILD_DOCKER_ARCHS) 217 | $(BUILD_DOCKER_ARCHS): common-docker-%: 218 | docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ 219 | -f $(DOCKERFILE_PATH) \ 220 | --build-arg ARCH="$*" \ 221 | --build-arg OS="linux" \ 222 | $(DOCKERBUILD_CONTEXT) 223 | 224 | .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) 225 | common-docker-publish: $(PUBLISH_DOCKER_ARCHS) 226 | $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: 227 | docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" 228 | 229 | DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION))) 230 | .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) 231 | common-docker-tag-latest: $(TAG_DOCKER_ARCHS) 232 | $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: 233 | docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" 234 | docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)" 235 | 236 | .PHONY: common-docker-manifest 237 | common-docker-manifest: 238 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)) 239 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" 240 | 241 | .PHONY: promu 242 | promu: $(PROMU) 243 | 244 | $(PROMU): 245 | $(eval PROMU_TMP := $(shell mktemp -d)) 246 | curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP) 247 | mkdir -p $(FIRST_GOPATH)/bin 248 | cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu 249 | rm -r $(PROMU_TMP) 250 | 251 | .PHONY: proto 252 | proto: 253 | @echo ">> generating code from proto files" 254 | @./scripts/genproto.sh 255 | 256 | ifdef GOLANGCI_LINT 257 | $(GOLANGCI_LINT): 258 | mkdir -p $(FIRST_GOPATH)/bin 259 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_LINT_VERSION)/install.sh \ 260 | | sed -e '/install -d/d' \ 261 | | sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION) 262 | endif 263 | 264 | .PHONY: precheck 265 | precheck:: 266 | 267 | define PRECHECK_COMMAND_template = 268 | precheck:: $(1)_precheck 269 | 270 | PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1))) 271 | .PHONY: $(1)_precheck 272 | $(1)_precheck: 273 | @if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \ 274 | echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \ 275 | exit 1; \ 276 | fi 277 | endef 278 | 279 | govulncheck: install-govulncheck 280 | govulncheck ./... 281 | 282 | install-govulncheck: 283 | command -v govulncheck > /dev/null || go install golang.org/x/vuln/cmd/govulncheck@latest 284 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Avalanche 2 | Copyright 2018-2021 FreshTracks 3 | Copyright 2022 The Prometheus Authors 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # avalanche 2 | 3 | Avalanche is a load-testing binary capable of generating metrics that can be either: 4 | 5 | * scraped via [Prometheus scrape formats](https://prometheus.io/docs/instrumenting/exposition_formats/) (including [OpenMetrics](https://github.com/OpenObservability/OpenMetrics)) endpoint. 6 | * written via Prometheus Remote Write (v1 only for now) to a target endpoint. 7 | 8 | This allows load testing services that can scrape (e.g. Prometheus, OpenTelemetry Collector and so), as well as, services accepting data via Prometheus remote_write API such as [Thanos](https://github.com/thanos-io/thanos), [Cortex](https://github.com/cortexproject/cortex), [M3DB](https://m3db.github.io/m3/integrations/prometheus/), [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics/) and other services [listed here](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage). 9 | 10 | Metric names and unique series change over time to simulate series churn. 11 | 12 | Checkout the (old-ish) [blog post](https://blog.freshtracks.io/load-testing-prometheus-metric-ingestion-5b878711711c). 13 | 14 | ## Installing 15 | 16 | ### Locally 17 | 18 | ```bash 19 | go install github.com/prometheus-community/avalanche/cmd/avalanche@latest 20 | ${GOPATH}/bin/avalanche --help 21 | ``` 22 | 23 | ### Docker 24 | 25 | ```bash 26 | docker run quay.io/prometheuscommunity/avalanche:latest --help 27 | ``` 28 | 29 | NOTE: We recommend using pinned image to a certain version (see all tags [here](https://quay.io/repository/prometheuscommunity/avalanche?tab=tags&tag=latest)) 30 | 31 | ## Using 32 | 33 | See [example](example/kubernetes-deployment.yaml) k8s manifest for deploying avalanche as an always running scrape target. 34 | 35 | ### Configuration 36 | 37 | See `--help` for all flags and their documentation. 38 | 39 | Notably, from 0.6.0 version, `avalanche` allows specifying various counts per various metric types. 40 | 41 | You can choose you own distribution, but usually it makes more sense to mimic realistic distribution used by your example targets. Feel free to use a [handy `mtypes` Go CLI](./cmd/mtypes) to gather type distributions from a target and generate avalanche flags from it. 42 | 43 | On top of scrape target functionality, avalanche is capable of Remote Write client load simulation, following the same, configured metric distribution via `--remote*` flags. 44 | 45 | ### Endpoints 46 | 47 | Two endpoints are available : 48 | * `/metrics` - metrics endpoint 49 | * `/health` - healthcheck endpoint 50 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting a security issue 2 | 3 | The Prometheus security policy, including how to report vulnerabilities, can be 4 | found here: 5 | 6 | 7 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.7.0 2 | -------------------------------------------------------------------------------- /cmd/avalanche/avalanche.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | "log" 20 | "log/slog" 21 | "net/http" 22 | "os" 23 | "syscall" 24 | 25 | "github.com/nelkinda/health-go" 26 | "github.com/oklog/run" 27 | "github.com/prometheus/client_golang/prometheus" 28 | "github.com/prometheus/client_golang/prometheus/promhttp" 29 | "github.com/prometheus/common/version" 30 | "gopkg.in/alecthomas/kingpin.v2" 31 | 32 | "github.com/prometheus-community/avalanche/metrics" 33 | ) 34 | 35 | func main() { 36 | kingpin.Version(version.Print("avalanche")) 37 | log.SetFlags(log.Ltime | log.Lshortfile) // Show file name and line in logs. 38 | kingpin.CommandLine.Help = "avalanche - metrics test server\n" + 39 | "\n" + 40 | "Capable of generating metrics to server on \\metrics or send via Remote Write.\n" + 41 | "\n" + 42 | "\nOptionally, on top of the --value-interval, --series-interval, --metric-interval logic, you can specify advanced --series-operation-mode:\n" + 43 | " double-halve:\n" + 44 | " Alternately doubles and halves the series count at regular intervals.\n" + 45 | " Usage: ./avalanche --series-operation-mode=double-halve --series-change-interval=30 --series-count=500\n" + 46 | " Description: This mode alternately doubles and halves the series count at regular intervals.\n" + 47 | " The series count is doubled on one tick and halved on the next, ensuring it never drops below 1.\n" + 48 | "\n" + 49 | " gradual-change:\n" + 50 | " Gradually changes the series count by a fixed rate at regular intervals.\n" + 51 | " Usage: ./avalanche --series-operation-mode=gradual-change --series-change-interval=30 --series-change-rate=100 --max-series-count=2000 --min-series-count=200\n" + 52 | " Description: This mode gradually increases the series count by seriesChangeRate on each tick up to maxSeriesCount,\n" + 53 | " then decreases it back to the minSeriesCount, and repeats this cycle indefinitely.\n" + 54 | " The series count is incremented by seriesChangeRate on each tick, ensuring it never drops below 1." + 55 | "\n" + 56 | " spike:\n" + 57 | " Periodically spikes the series count by a given multiplier.\n" + 58 | " Usage: ./avalanche --series-operation-mode=spike --series-change-interval=180 --series-count=100 --spike-multiplier=1.5\n" + 59 | " Description: This mode periodically increases the series count by a spike multiplier on one tick and\n" + 60 | " then returns it to the original count on the next tick. This pattern repeats indefinitely,\n" + 61 | " creating a spiking effect in the series count.\n" 62 | 63 | cfg := metrics.NewConfigFromFlags(kingpin.Flag) 64 | port := kingpin.Flag("port", "Port to serve at").Default("9001").Int() 65 | writeCfg := metrics.NewWriteConfigFromFlags(kingpin.Flag) 66 | 67 | kingpin.Parse() 68 | if err := cfg.Validate(); err != nil { 69 | kingpin.FatalUsage("configuration error: %v", err) 70 | } 71 | if err := writeCfg.Validate(); err != nil { 72 | kingpin.FatalUsage("remote write config validation failed: %v", err) 73 | } 74 | 75 | collector := metrics.NewCollector(*cfg) 76 | reg := prometheus.NewRegistry() 77 | reg.MustRegister(collector) 78 | writeCfg.UpdateNotify = collector.UpdateNotifyCh() 79 | 80 | log.Println("initializing avalanche...") 81 | 82 | var g run.Group 83 | g.Add(run.SignalHandler(context.Background(), os.Interrupt, syscall.SIGTERM)) 84 | g.Add(collector.Run, collector.Stop) 85 | 86 | // One-off remote write send mode. 87 | if writeCfg.URL != nil { 88 | ctx, cancel := context.WithCancel(context.Background()) 89 | g.Add(func() error { 90 | if err := metrics.RunRemoteWriting(ctx, slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})), writeCfg, reg); err != nil { 91 | return err 92 | } 93 | return nil // One-off. 94 | }, func(error) { cancel() }) 95 | } 96 | 97 | httpSrv := &http.Server{Addr: fmt.Sprintf(":%v", *port)} 98 | g.Add(func() error { 99 | fmt.Printf("Serving your metrics at :%v/metrics\n", *port) 100 | http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{ 101 | EnableOpenMetrics: true, 102 | })) 103 | http.HandleFunc("/health", health.New(health.Health{}).Handler) 104 | return httpSrv.ListenAndServe() 105 | }, func(_ error) { 106 | _ = httpSrv.Shutdown(context.Background()) 107 | }) 108 | 109 | log.Println("starting avalanche...") 110 | if err := g.Run(); err != nil { 111 | //nolint:errcheck 112 | log.Fatalf("running avalanche failed %v", err) 113 | } 114 | log.Println("avalanche finished") 115 | } 116 | -------------------------------------------------------------------------------- /cmd/mtypes/README.md: -------------------------------------------------------------------------------- 1 | # mtypes 2 | 3 | Go CLI gathering statistics around the distribution of types, average number of buckets (and more) across your Prometheus metrics/series. 4 | 5 | ## Usage 6 | 7 | The main usage allows to take resource (from stdin, file or HTTP /metrics endpoint) and calculate type statistics e.g.: 8 | 9 | ```bash 10 | go install github.com/prometheus-community/avalanche/cmd/mtypes@latest # or locally: alias mtypes="go run ./cmd/mtypes" 11 | $ mtypes -resource=http://localhost:9090/metrics 12 | $ mtypes -resource=./cmd/mtypes/exampleprometheustarget.txt 13 | $ cat ./cmd/mtypes/exampleprometheustarget.txt | mtypes 14 | ``` 15 | 16 | ```bash 17 | Metric Type Metric Families Series (adjusted) Series (adjusted) % Average Buckets/Objectives 18 | GAUGE 76 93 (93) 31.958763 (16.909091) - 19 | COUNTER 96 157 (157) 53.951890 (28.545455) - 20 | HISTOGRAM 8 14 (186) 4.810997 (33.818182) 11.285714 21 | SUMMARY 15 27 (114) 9.278351 (20.727273) 2.222222 22 | --- --- --- --- --- 23 | * 195 291 (550) 100.000000 (100.000000) - 24 | ``` 25 | 26 | > NOTE: "Adjusted" series, means actual number of individual series stored in Prometheus. Classic histograms and summaries are stored as a set of counters. This is relevant as the cost of indexing new series is higher than storing complex values (this is why we slowly move to native histograms). 27 | 28 | Additionally, you can pass `--avalanche-flags-for-adjusted-series=10000` to print Avalanche v0.6.0+ flags to configure, for avalanche to generate metric target with the given amount of adjusted series, while maintaining a similar distribution e.g. 29 | 30 | ```bash 31 | cat ./cmd/mtypes/exampleprometheustarget.txt | mtypes --avalanche-flags-for-adjusted-series=1000 32 | Metric Type Metric Families Series (adjusted) Series (adjusted) % Average Buckets/Objectives 33 | GAUGE 76 93 (93) 31.958763 (16.909091) - 34 | COUNTER 96 157 (157) 53.951890 (28.545455) - 35 | HISTOGRAM 8 14 (186) 4.810997 (33.818182) 11.285714 36 | SUMMARY 15 27 (114) 9.278351 (20.727273) 2.222222 37 | --- --- --- --- --- 38 | * 195 291 (550) 100.000000 (100.000000) - 39 | 40 | Avalanche flags for the similar distribution to get to the adjusted series goal of: 1000 41 | --gauge-metric-count=16 42 | --counter-metric-count=28 43 | --histogram-metric-count=2 44 | --histogram-metric-bucket-count=10 45 | --native-histogram-metric-count=0 46 | --summary-metric-count=5 47 | --summary-metric-objective-count=2 48 | --series-count=10 49 | --value-interval=300 # Changes values every 5m. 50 | --series-interval=3600 # 1h series churn. 51 | --metric-interval=0 52 | This should give the total adjusted series to: 900 53 | ``` 54 | -------------------------------------------------------------------------------- /cmd/mtypes/exampleprometheustarget.txt: -------------------------------------------------------------------------------- 1 | # HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles. 2 | # TYPE go_gc_duration_seconds summary 3 | go_gc_duration_seconds{quantile="0"} 5.8641e-05 4 | go_gc_duration_seconds{quantile="0.25"} 8.4045e-05 5 | go_gc_duration_seconds{quantile="0.5"} 0.000119609 6 | go_gc_duration_seconds{quantile="0.75"} 0.000149195 7 | go_gc_duration_seconds{quantile="1"} 0.000312434 8 | go_gc_duration_seconds_sum 11.324308382 9 | go_gc_duration_seconds_count 92364 10 | # HELP go_goroutines Number of goroutines that currently exist. 11 | # TYPE go_goroutines gauge 12 | go_goroutines 112 13 | # HELP go_info Information about the Go environment. 14 | # TYPE go_info gauge 15 | go_info{version="go1.20.14"} 1 16 | # HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. 17 | # TYPE go_memstats_alloc_bytes gauge 18 | go_memstats_alloc_bytes 1.09818568e+08 19 | # HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. 20 | # TYPE go_memstats_alloc_bytes_total counter 21 | go_memstats_alloc_bytes_total 7.420978933248e+12 22 | # HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. 23 | # TYPE go_memstats_buck_hash_sys_bytes gauge 24 | go_memstats_buck_hash_sys_bytes 3.653156e+06 25 | # HELP go_memstats_frees_total Total number of frees. 26 | # TYPE go_memstats_frees_total counter 27 | go_memstats_frees_total 1.19996693238e+11 28 | # HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. 29 | # TYPE go_memstats_gc_sys_bytes gauge 30 | go_memstats_gc_sys_bytes 1.6556264e+07 31 | # HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. 32 | # TYPE go_memstats_heap_alloc_bytes gauge 33 | go_memstats_heap_alloc_bytes 1.09818568e+08 34 | # HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. 35 | # TYPE go_memstats_heap_idle_bytes gauge 36 | go_memstats_heap_idle_bytes 1.8628608e+08 37 | # HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. 38 | # TYPE go_memstats_heap_inuse_bytes gauge 39 | go_memstats_heap_inuse_bytes 1.3860864e+08 40 | # HELP go_memstats_heap_objects Number of allocated objects. 41 | # TYPE go_memstats_heap_objects gauge 42 | go_memstats_heap_objects 738856 43 | # HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. 44 | # TYPE go_memstats_heap_released_bytes gauge 45 | go_memstats_heap_released_bytes 1.42557184e+08 46 | # HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. 47 | # TYPE go_memstats_heap_sys_bytes gauge 48 | go_memstats_heap_sys_bytes 3.2489472e+08 49 | # HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. 50 | # TYPE go_memstats_last_gc_time_seconds gauge 51 | go_memstats_last_gc_time_seconds 1.7278073317025118e+09 52 | # HELP go_memstats_lookups_total Total number of pointer lookups. 53 | # TYPE go_memstats_lookups_total counter 54 | go_memstats_lookups_total 0 55 | # HELP go_memstats_mallocs_total Total number of mallocs. 56 | # TYPE go_memstats_mallocs_total counter 57 | go_memstats_mallocs_total 1.19997432094e+11 58 | # HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. 59 | # TYPE go_memstats_mcache_inuse_bytes gauge 60 | go_memstats_mcache_inuse_bytes 4800 61 | # HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. 62 | # TYPE go_memstats_mcache_sys_bytes gauge 63 | go_memstats_mcache_sys_bytes 15600 64 | # HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. 65 | # TYPE go_memstats_mspan_inuse_bytes gauge 66 | go_memstats_mspan_inuse_bytes 1.8024e+06 67 | # HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. 68 | # TYPE go_memstats_mspan_sys_bytes gauge 69 | go_memstats_mspan_sys_bytes 3.24768e+06 70 | # HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. 71 | # TYPE go_memstats_next_gc_bytes gauge 72 | go_memstats_next_gc_bytes 1.636618e+08 73 | # HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. 74 | # TYPE go_memstats_other_sys_bytes gauge 75 | go_memstats_other_sys_bytes 1.202956e+06 76 | # HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. 77 | # TYPE go_memstats_stack_inuse_bytes gauge 78 | go_memstats_stack_inuse_bytes 2.260992e+06 79 | # HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. 80 | # TYPE go_memstats_stack_sys_bytes gauge 81 | go_memstats_stack_sys_bytes 2.260992e+06 82 | # HELP go_memstats_sys_bytes Number of bytes obtained from system. 83 | # TYPE go_memstats_sys_bytes gauge 84 | go_memstats_sys_bytes 3.51831368e+08 85 | # HELP go_threads Number of OS threads created. 86 | # TYPE go_threads gauge 87 | go_threads 12 88 | # HELP grpc_client_handled_total Total number of RPCs completed by the client, regardless of success or failure. 89 | # TYPE grpc_client_handled_total counter 90 | grpc_client_handled_total{grpc_code="Canceled",grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 9 91 | grpc_client_handled_total{grpc_code="DeadlineExceeded",grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 82 92 | grpc_client_handled_total{grpc_code="Internal",grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 4 93 | grpc_client_handled_total{grpc_code="OK",grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 1.0831867e+07 94 | grpc_client_handled_total{grpc_code="Unauthenticated",grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 1 95 | grpc_client_handled_total{grpc_code="Unavailable",grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 494 96 | # HELP grpc_client_handling_seconds Histogram of response latency (seconds) of the gRPC until it is finished by the application. 97 | # TYPE grpc_client_handling_seconds histogram 98 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.005"} 0 99 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.01"} 0 100 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.025"} 34059 101 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.05"} 1.127825e+06 102 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.1"} 9.058302e+06 103 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.25"} 1.0721886e+07 104 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="0.5"} 1.0759498e+07 105 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="1"} 1.0774023e+07 106 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="2.5"} 1.079026e+07 107 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="5"} 1.0800098e+07 108 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="10"} 1.0832159e+07 109 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="15"} 1.0832261e+07 110 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="20"} 1.0832299e+07 111 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="30"} 1.0832376e+07 112 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="40"} 1.0832457e+07 113 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="50"} 1.0832457e+07 114 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="60"} 1.0832457e+07 115 | grpc_client_handling_seconds_bucket{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary",le="+Inf"} 1.0832457e+07 116 | grpc_client_handling_seconds_sum{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 1.2123103039707085e+06 117 | grpc_client_handling_seconds_count{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 1.0832457e+07 118 | # HELP grpc_client_msg_received_total Total number of RPC stream messages received by the client. 119 | # TYPE grpc_client_msg_received_total counter 120 | grpc_client_msg_received_total{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 590 121 | # HELP grpc_client_msg_sent_total Total number of gRPC stream messages sent by the client. 122 | # TYPE grpc_client_msg_sent_total counter 123 | grpc_client_msg_sent_total{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 1.0832458e+07 124 | # HELP grpc_client_started_total Total number of RPCs started on the client. 125 | # TYPE grpc_client_started_total counter 126 | grpc_client_started_total{grpc_method="CreateTimeSeries",grpc_service="google.monitoring.v3.MetricService",grpc_type="unary"} 1.0832458e+07 127 | # HELP net_conntrack_dialer_conn_attempted_total Total number of connections attempted by the given dialer a given name. 128 | # TYPE net_conntrack_dialer_conn_attempted_total counter 129 | net_conntrack_dialer_conn_attempted_total{dialer_name="cadvisor"} 94 130 | net_conntrack_dialer_conn_attempted_total{dialer_name="default"} 0 131 | net_conntrack_dialer_conn_attempted_total{dialer_name="kube-state-metrics"} 2 132 | net_conntrack_dialer_conn_attempted_total{dialer_name="pods"} 179445 133 | # HELP net_conntrack_dialer_conn_closed_total Total number of connections closed which originated from the dialer of a given name. 134 | # TYPE net_conntrack_dialer_conn_closed_total counter 135 | net_conntrack_dialer_conn_closed_total{dialer_name="cadvisor"} 3 136 | net_conntrack_dialer_conn_closed_total{dialer_name="default"} 0 137 | net_conntrack_dialer_conn_closed_total{dialer_name="kube-state-metrics"} 0 138 | net_conntrack_dialer_conn_closed_total{dialer_name="pods"} 179394 139 | # HELP net_conntrack_dialer_conn_established_total Total number of connections successfully established by the given dialer a given name. 140 | # TYPE net_conntrack_dialer_conn_established_total counter 141 | net_conntrack_dialer_conn_established_total{dialer_name="cadvisor"} 4 142 | net_conntrack_dialer_conn_established_total{dialer_name="default"} 0 143 | net_conntrack_dialer_conn_established_total{dialer_name="kube-state-metrics"} 2 144 | net_conntrack_dialer_conn_established_total{dialer_name="pods"} 179399 145 | # HELP net_conntrack_dialer_conn_failed_total Total number of connections failed to dial by the dialer a given name. 146 | # TYPE net_conntrack_dialer_conn_failed_total counter 147 | net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="refused"} 7 148 | net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="resolution"} 0 149 | net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="timeout"} 83 150 | net_conntrack_dialer_conn_failed_total{dialer_name="cadvisor",reason="unknown"} 90 151 | net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="refused"} 0 152 | net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="resolution"} 0 153 | net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="timeout"} 0 154 | net_conntrack_dialer_conn_failed_total{dialer_name="default",reason="unknown"} 0 155 | net_conntrack_dialer_conn_failed_total{dialer_name="kube-state-metrics",reason="refused"} 0 156 | net_conntrack_dialer_conn_failed_total{dialer_name="kube-state-metrics",reason="resolution"} 0 157 | net_conntrack_dialer_conn_failed_total{dialer_name="kube-state-metrics",reason="timeout"} 0 158 | net_conntrack_dialer_conn_failed_total{dialer_name="kube-state-metrics",reason="unknown"} 0 159 | net_conntrack_dialer_conn_failed_total{dialer_name="pods",reason="refused"} 4 160 | net_conntrack_dialer_conn_failed_total{dialer_name="pods",reason="resolution"} 0 161 | net_conntrack_dialer_conn_failed_total{dialer_name="pods",reason="timeout"} 42 162 | net_conntrack_dialer_conn_failed_total{dialer_name="pods",reason="unknown"} 46 163 | # HELP net_conntrack_listener_conn_accepted_total Total number of connections opened to the listener of a given name. 164 | # TYPE net_conntrack_listener_conn_accepted_total counter 165 | net_conntrack_listener_conn_accepted_total{listener_name="http"} 8 166 | # HELP net_conntrack_listener_conn_closed_total Total number of connections closed that were made to the listener of a given name. 167 | # TYPE net_conntrack_listener_conn_closed_total counter 168 | net_conntrack_listener_conn_closed_total{listener_name="http"} 3 169 | # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. 170 | # TYPE process_cpu_seconds_total counter 171 | process_cpu_seconds_total 64026.65 172 | # HELP process_max_fds Maximum number of open file descriptors. 173 | # TYPE process_max_fds gauge 174 | process_max_fds 1.048576e+06 175 | # HELP process_open_fds Number of open file descriptors. 176 | # TYPE process_open_fds gauge 177 | process_open_fds 105 178 | # HELP process_resident_memory_bytes Resident memory size in bytes. 179 | # TYPE process_resident_memory_bytes gauge 180 | process_resident_memory_bytes 2.81624576e+08 181 | # HELP process_start_time_seconds Start time of the process since unix epoch in seconds. 182 | # TYPE process_start_time_seconds gauge 183 | process_start_time_seconds 1.72511698039e+09 184 | # HELP process_virtual_memory_bytes Virtual memory size in bytes. 185 | # TYPE process_virtual_memory_bytes gauge 186 | process_virtual_memory_bytes 2.8450332672e+10 187 | # HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. 188 | # TYPE process_virtual_memory_max_bytes gauge 189 | process_virtual_memory_max_bytes 1.8446744073709552e+19 190 | # HELP prometheus_api_remote_read_queries The current number of remote read queries being executed or waiting. 191 | # TYPE prometheus_api_remote_read_queries gauge 192 | prometheus_api_remote_read_queries 0 193 | # HELP prometheus_build_info A metric with a constant '1' value labeled by version, revision, branch, goversion from which prometheus was built, and the goos and goarch for the build. 194 | # TYPE prometheus_build_info gauge 195 | prometheus_build_info{branch="",goarch="amd64",goos="linux",goversion="go1.20.14",revision="d7b199739aa7e0d00e7ebd0792339dd4b167a269-modified",tags="builtinassets",version="2.45.3"} 1 196 | # HELP prometheus_config_last_reload_success_timestamp_seconds Timestamp of the last successful configuration reload. 197 | # TYPE prometheus_config_last_reload_success_timestamp_seconds gauge 198 | prometheus_config_last_reload_success_timestamp_seconds 1.725116982549508e+09 199 | # HELP prometheus_config_last_reload_successful Whether the last configuration reload attempt was successful. 200 | # TYPE prometheus_config_last_reload_successful gauge 201 | prometheus_config_last_reload_successful 1 202 | # HELP prometheus_engine_queries The current number of queries being executed or waiting. 203 | # TYPE prometheus_engine_queries gauge 204 | prometheus_engine_queries 0 205 | # HELP prometheus_engine_queries_concurrent_max The max number of concurrent queries. 206 | # TYPE prometheus_engine_queries_concurrent_max gauge 207 | prometheus_engine_queries_concurrent_max 20 208 | # HELP prometheus_engine_query_duration_seconds Query timings 209 | # TYPE prometheus_engine_query_duration_seconds summary 210 | prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.5"} NaN 211 | prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.9"} NaN 212 | prometheus_engine_query_duration_seconds{slice="inner_eval",quantile="0.99"} NaN 213 | prometheus_engine_query_duration_seconds_sum{slice="inner_eval"} 0 214 | prometheus_engine_query_duration_seconds_count{slice="inner_eval"} 0 215 | prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.5"} NaN 216 | prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.9"} NaN 217 | prometheus_engine_query_duration_seconds{slice="prepare_time",quantile="0.99"} NaN 218 | prometheus_engine_query_duration_seconds_sum{slice="prepare_time"} 0 219 | prometheus_engine_query_duration_seconds_count{slice="prepare_time"} 0 220 | prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.5"} NaN 221 | prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.9"} NaN 222 | prometheus_engine_query_duration_seconds{slice="queue_time",quantile="0.99"} NaN 223 | prometheus_engine_query_duration_seconds_sum{slice="queue_time"} 0 224 | prometheus_engine_query_duration_seconds_count{slice="queue_time"} 0 225 | prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.5"} NaN 226 | prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.9"} NaN 227 | prometheus_engine_query_duration_seconds{slice="result_sort",quantile="0.99"} NaN 228 | prometheus_engine_query_duration_seconds_sum{slice="result_sort"} 0 229 | prometheus_engine_query_duration_seconds_count{slice="result_sort"} 0 230 | # HELP prometheus_engine_query_log_enabled State of the query log. 231 | # TYPE prometheus_engine_query_log_enabled gauge 232 | prometheus_engine_query_log_enabled 0 233 | # HELP prometheus_engine_query_log_failures_total The number of query log failures. 234 | # TYPE prometheus_engine_query_log_failures_total counter 235 | prometheus_engine_query_log_failures_total 0 236 | # HELP prometheus_engine_query_samples_total The total number of samples loaded by all queries. 237 | # TYPE prometheus_engine_query_samples_total counter 238 | prometheus_engine_query_samples_total 0 239 | # HELP prometheus_http_request_duration_seconds Histogram of latencies for HTTP requests. 240 | # TYPE prometheus_http_request_duration_seconds histogram 241 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.1"} 2 242 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.2"} 2 243 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="0.4"} 2 244 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="1"} 2 245 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="3"} 2 246 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="8"} 2 247 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="20"} 2 248 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="60"} 2 249 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="120"} 2 250 | prometheus_http_request_duration_seconds_bucket{handler="/-/ready",le="+Inf"} 2 251 | prometheus_http_request_duration_seconds_sum{handler="/-/ready"} 4.7443999999999995e-05 252 | prometheus_http_request_duration_seconds_count{handler="/-/ready"} 2 253 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="0.1"} 1 254 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="0.2"} 1 255 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="0.4"} 1 256 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="1"} 1 257 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="3"} 1 258 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="8"} 1 259 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="20"} 1 260 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="60"} 1 261 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="120"} 1 262 | prometheus_http_request_duration_seconds_bucket{handler="/-/reload",le="+Inf"} 1 263 | prometheus_http_request_duration_seconds_sum{handler="/-/reload"} 0.002356799 264 | prometheus_http_request_duration_seconds_count{handler="/-/reload"} 1 265 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.1"} 358716 266 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.2"} 358716 267 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="0.4"} 358716 268 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="1"} 358716 269 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="3"} 358716 270 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="8"} 358716 271 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="20"} 358716 272 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="60"} 448346 273 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="120"} 448346 274 | prometheus_http_request_duration_seconds_bucket{handler="/debug/*subpath",le="+Inf"} 448346 275 | prometheus_http_request_duration_seconds_sum{handler="/debug/*subpath"} 2.692262582005182e+06 276 | prometheus_http_request_duration_seconds_count{handler="/debug/*subpath"} 448346 277 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.1"} 179357 278 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.2"} 179357 279 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="0.4"} 179357 280 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="1"} 179357 281 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="3"} 179357 282 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="8"} 179357 283 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="20"} 179357 284 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="60"} 179357 285 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="120"} 179357 286 | prometheus_http_request_duration_seconds_bucket{handler="/metrics",le="+Inf"} 179357 287 | prometheus_http_request_duration_seconds_sum{handler="/metrics"} 552.2173053479947 288 | prometheus_http_request_duration_seconds_count{handler="/metrics"} 179357 289 | # HELP prometheus_http_requests_total Counter of HTTP requests. 290 | # TYPE prometheus_http_requests_total counter 291 | prometheus_http_requests_total{code="200",handler="/-/ready"} 1 292 | prometheus_http_requests_total{code="200",handler="/-/reload"} 1 293 | prometheus_http_requests_total{code="200",handler="/debug/*subpath"} 448346 294 | prometheus_http_requests_total{code="200",handler="/metrics"} 179357 295 | prometheus_http_requests_total{code="503",handler="/-/ready"} 1 296 | # HELP prometheus_http_response_size_bytes Histogram of response size for HTTP requests. 297 | # TYPE prometheus_http_response_size_bytes histogram 298 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="100"} 2 299 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1000"} 2 300 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="10000"} 2 301 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="100000"} 2 302 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+06"} 2 303 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+07"} 2 304 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+08"} 2 305 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="1e+09"} 2 306 | prometheus_http_response_size_bytes_bucket{handler="/-/ready",le="+Inf"} 2 307 | prometheus_http_response_size_bytes_sum{handler="/-/ready"} 47 308 | prometheus_http_response_size_bytes_count{handler="/-/ready"} 2 309 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="100"} 1 310 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="1000"} 1 311 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="10000"} 1 312 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="100000"} 1 313 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="1e+06"} 1 314 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="1e+07"} 1 315 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="1e+08"} 1 316 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="1e+09"} 1 317 | prometheus_http_response_size_bytes_bucket{handler="/-/reload",le="+Inf"} 1 318 | prometheus_http_response_size_bytes_sum{handler="/-/reload"} 0 319 | prometheus_http_response_size_bytes_count{handler="/-/reload"} 1 320 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="100"} 0 321 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1000"} 179358 322 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="10000"} 269558 323 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="100000"} 359969 324 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+06"} 448346 325 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+07"} 448346 326 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+08"} 448346 327 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="1e+09"} 448346 328 | prometheus_http_response_size_bytes_bucket{handler="/debug/*subpath",le="+Inf"} 448346 329 | prometheus_http_response_size_bytes_sum{handler="/debug/*subpath"} 1.7640059511e+10 330 | prometheus_http_response_size_bytes_count{handler="/debug/*subpath"} 448346 331 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="100"} 0 332 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1000"} 0 333 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="10000"} 191 334 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="100000"} 179357 335 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+06"} 179357 336 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+07"} 179357 337 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+08"} 179357 338 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="1e+09"} 179357 339 | prometheus_http_response_size_bytes_bucket{handler="/metrics",le="+Inf"} 179357 340 | prometheus_http_response_size_bytes_sum{handler="/metrics"} 1.895799365e+09 341 | prometheus_http_response_size_bytes_count{handler="/metrics"} 179357 342 | # HELP prometheus_notifications_alertmanagers_discovered The number of alertmanagers discovered and active. 343 | # TYPE prometheus_notifications_alertmanagers_discovered gauge 344 | prometheus_notifications_alertmanagers_discovered 0 345 | # HELP prometheus_notifications_dropped_total Total number of alerts dropped due to errors when sending to Alertmanager. 346 | # TYPE prometheus_notifications_dropped_total counter 347 | prometheus_notifications_dropped_total 0 348 | # HELP prometheus_notifications_queue_capacity The capacity of the alert notifications queue. 349 | # TYPE prometheus_notifications_queue_capacity gauge 350 | prometheus_notifications_queue_capacity 10000 351 | # HELP prometheus_notifications_queue_length The number of alert notifications in the queue. 352 | # TYPE prometheus_notifications_queue_length gauge 353 | prometheus_notifications_queue_length 0 354 | # HELP prometheus_ready Whether Prometheus startup was fully completed and the server is ready for normal operation. 355 | # TYPE prometheus_ready gauge 356 | prometheus_ready 1 357 | # HELP prometheus_remote_storage_exemplars_in_total Exemplars in to remote storage, compare to exemplars out for queue managers. 358 | # TYPE prometheus_remote_storage_exemplars_in_total counter 359 | prometheus_remote_storage_exemplars_in_total 0 360 | # HELP prometheus_remote_storage_highest_timestamp_in_seconds Highest timestamp that has come into the remote storage via the Appender interface, in seconds since epoch. 361 | # TYPE prometheus_remote_storage_highest_timestamp_in_seconds gauge 362 | prometheus_remote_storage_highest_timestamp_in_seconds 1.727807345e+09 363 | # HELP prometheus_remote_storage_histograms_in_total HistogramSamples in to remote storage, compare to histograms out for queue managers. 364 | # TYPE prometheus_remote_storage_histograms_in_total counter 365 | prometheus_remote_storage_histograms_in_total 0 366 | # HELP prometheus_remote_storage_samples_in_total Samples in to remote storage, compare to samples out for queue managers. 367 | # TYPE prometheus_remote_storage_samples_in_total counter 368 | prometheus_remote_storage_samples_in_total 1.966333233e+09 369 | # HELP prometheus_remote_storage_string_interner_zero_reference_releases_total The number of times release has been called for strings that are not interned. 370 | # TYPE prometheus_remote_storage_string_interner_zero_reference_releases_total counter 371 | prometheus_remote_storage_string_interner_zero_reference_releases_total 0 372 | # HELP prometheus_rule_evaluation_duration_seconds The duration for a rule to execute. 373 | # TYPE prometheus_rule_evaluation_duration_seconds summary 374 | prometheus_rule_evaluation_duration_seconds{quantile="0.5"} NaN 375 | prometheus_rule_evaluation_duration_seconds{quantile="0.9"} NaN 376 | prometheus_rule_evaluation_duration_seconds{quantile="0.99"} NaN 377 | prometheus_rule_evaluation_duration_seconds_sum 0 378 | prometheus_rule_evaluation_duration_seconds_count 0 379 | # HELP prometheus_rule_group_duration_seconds The duration of rule group evaluations. 380 | # TYPE prometheus_rule_group_duration_seconds summary 381 | prometheus_rule_group_duration_seconds{quantile="0.01"} NaN 382 | prometheus_rule_group_duration_seconds{quantile="0.05"} NaN 383 | prometheus_rule_group_duration_seconds{quantile="0.5"} NaN 384 | prometheus_rule_group_duration_seconds{quantile="0.9"} NaN 385 | prometheus_rule_group_duration_seconds{quantile="0.99"} NaN 386 | prometheus_rule_group_duration_seconds_sum 0 387 | prometheus_rule_group_duration_seconds_count 0 388 | # HELP prometheus_sd_azure_failures_total Number of Azure service discovery refresh failures. 389 | # TYPE prometheus_sd_azure_failures_total counter 390 | prometheus_sd_azure_failures_total 0 391 | # HELP prometheus_sd_consul_rpc_duration_seconds The duration of a Consul RPC call in seconds. 392 | # TYPE prometheus_sd_consul_rpc_duration_seconds summary 393 | prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.5"} NaN 394 | prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.9"} NaN 395 | prometheus_sd_consul_rpc_duration_seconds{call="service",endpoint="catalog",quantile="0.99"} NaN 396 | prometheus_sd_consul_rpc_duration_seconds_sum{call="service",endpoint="catalog"} 0 397 | prometheus_sd_consul_rpc_duration_seconds_count{call="service",endpoint="catalog"} 0 398 | prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.5"} NaN 399 | prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.9"} NaN 400 | prometheus_sd_consul_rpc_duration_seconds{call="services",endpoint="catalog",quantile="0.99"} NaN 401 | prometheus_sd_consul_rpc_duration_seconds_sum{call="services",endpoint="catalog"} 0 402 | prometheus_sd_consul_rpc_duration_seconds_count{call="services",endpoint="catalog"} 0 403 | # HELP prometheus_sd_consul_rpc_failures_total The number of Consul RPC call failures. 404 | # TYPE prometheus_sd_consul_rpc_failures_total counter 405 | prometheus_sd_consul_rpc_failures_total 0 406 | # HELP prometheus_sd_discovered_targets Current number of discovered targets. 407 | # TYPE prometheus_sd_discovered_targets gauge 408 | prometheus_sd_discovered_targets{config="cadvisor",name="scrape"} 2 409 | prometheus_sd_discovered_targets{config="kube-state-metrics",name="scrape"} 9 410 | prometheus_sd_discovered_targets{config="pods",name="scrape"} 82 411 | # HELP prometheus_sd_dns_lookup_failures_total The number of DNS-SD lookup failures. 412 | # TYPE prometheus_sd_dns_lookup_failures_total counter 413 | prometheus_sd_dns_lookup_failures_total 0 414 | # HELP prometheus_sd_dns_lookups_total The number of DNS-SD lookups. 415 | # TYPE prometheus_sd_dns_lookups_total counter 416 | prometheus_sd_dns_lookups_total 0 417 | # HELP prometheus_sd_failed_configs Current number of service discovery configurations that failed to load. 418 | # TYPE prometheus_sd_failed_configs gauge 419 | prometheus_sd_failed_configs{name="notify"} 0 420 | prometheus_sd_failed_configs{name="scrape"} 0 421 | # HELP prometheus_sd_file_read_errors_total The number of File-SD read errors. 422 | # TYPE prometheus_sd_file_read_errors_total counter 423 | prometheus_sd_file_read_errors_total 0 424 | # HELP prometheus_sd_file_scan_duration_seconds The duration of the File-SD scan in seconds. 425 | # TYPE prometheus_sd_file_scan_duration_seconds summary 426 | prometheus_sd_file_scan_duration_seconds{quantile="0.5"} NaN 427 | prometheus_sd_file_scan_duration_seconds{quantile="0.9"} NaN 428 | prometheus_sd_file_scan_duration_seconds{quantile="0.99"} NaN 429 | prometheus_sd_file_scan_duration_seconds_sum 0 430 | prometheus_sd_file_scan_duration_seconds_count 0 431 | # HELP prometheus_sd_file_watcher_errors_total The number of File-SD errors caused by filesystem watch failures. 432 | # TYPE prometheus_sd_file_watcher_errors_total counter 433 | prometheus_sd_file_watcher_errors_total 0 434 | # HELP prometheus_sd_http_failures_total Number of HTTP service discovery refresh failures. 435 | # TYPE prometheus_sd_http_failures_total counter 436 | prometheus_sd_http_failures_total 0 437 | # HELP prometheus_sd_kubernetes_events_total The number of Kubernetes events handled. 438 | # TYPE prometheus_sd_kubernetes_events_total counter 439 | prometheus_sd_kubernetes_events_total{event="add",role="endpoints"} 0 440 | prometheus_sd_kubernetes_events_total{event="add",role="endpointslice"} 0 441 | prometheus_sd_kubernetes_events_total{event="add",role="ingress"} 0 442 | prometheus_sd_kubernetes_events_total{event="add",role="node"} 5 443 | prometheus_sd_kubernetes_events_total{event="add",role="pod"} 169 444 | prometheus_sd_kubernetes_events_total{event="add",role="service"} 9 445 | prometheus_sd_kubernetes_events_total{event="delete",role="endpoints"} 0 446 | prometheus_sd_kubernetes_events_total{event="delete",role="endpointslice"} 0 447 | prometheus_sd_kubernetes_events_total{event="delete",role="ingress"} 0 448 | prometheus_sd_kubernetes_events_total{event="delete",role="node"} 3 449 | prometheus_sd_kubernetes_events_total{event="delete",role="pod"} 128 450 | prometheus_sd_kubernetes_events_total{event="delete",role="service"} 2 451 | prometheus_sd_kubernetes_events_total{event="update",role="endpoints"} 0 452 | prometheus_sd_kubernetes_events_total{event="update",role="endpointslice"} 0 453 | prometheus_sd_kubernetes_events_total{event="update",role="ingress"} 0 454 | prometheus_sd_kubernetes_events_total{event="update",role="node"} 35525 455 | prometheus_sd_kubernetes_events_total{event="update",role="pod"} 1034 456 | prometheus_sd_kubernetes_events_total{event="update",role="service"} 29 457 | # HELP prometheus_sd_kubernetes_http_request_duration_seconds Summary of latencies for HTTP requests to the Kubernetes API by endpoint. 458 | # TYPE prometheus_sd_kubernetes_http_request_duration_seconds summary 459 | prometheus_sd_kubernetes_http_request_duration_seconds_sum{endpoint="/api/v1/nodes"} 0.017348603 460 | prometheus_sd_kubernetes_http_request_duration_seconds_count{endpoint="/api/v1/nodes"} 4 461 | prometheus_sd_kubernetes_http_request_duration_seconds_sum{endpoint="/api/v1/pods"} 0.038949225999999997 462 | prometheus_sd_kubernetes_http_request_duration_seconds_count{endpoint="/api/v1/pods"} 4 463 | prometheus_sd_kubernetes_http_request_duration_seconds_sum{endpoint="/api/v1/services"} 0.014277334000000001 464 | prometheus_sd_kubernetes_http_request_duration_seconds_count{endpoint="/api/v1/services"} 4 465 | # HELP prometheus_sd_kubernetes_http_request_total Total number of HTTP requests to the Kubernetes API by status code. 466 | # TYPE prometheus_sd_kubernetes_http_request_total counter 467 | prometheus_sd_kubernetes_http_request_total{status_code="200"} 17957 468 | prometheus_sd_kubernetes_http_request_total{status_code=""} 83 469 | # HELP prometheus_sd_kubernetes_workqueue_depth Current depth of the work queue. 470 | # TYPE prometheus_sd_kubernetes_workqueue_depth gauge 471 | prometheus_sd_kubernetes_workqueue_depth{queue_name="node"} 0 472 | prometheus_sd_kubernetes_workqueue_depth{queue_name="pod"} 0 473 | prometheus_sd_kubernetes_workqueue_depth{queue_name="service"} 0 474 | # HELP prometheus_sd_kubernetes_workqueue_items_total Total number of items added to the work queue. 475 | # TYPE prometheus_sd_kubernetes_workqueue_items_total counter 476 | prometheus_sd_kubernetes_workqueue_items_total{queue_name="node"} 35533 477 | prometheus_sd_kubernetes_workqueue_items_total{queue_name="pod"} 1329 478 | prometheus_sd_kubernetes_workqueue_items_total{queue_name="service"} 40 479 | # HELP prometheus_sd_kubernetes_workqueue_latency_seconds How long an item stays in the work queue. 480 | # TYPE prometheus_sd_kubernetes_workqueue_latency_seconds summary 481 | prometheus_sd_kubernetes_workqueue_latency_seconds_sum{queue_name="node"} 0.49772388200000356 482 | prometheus_sd_kubernetes_workqueue_latency_seconds_count{queue_name="node"} 35533 483 | prometheus_sd_kubernetes_workqueue_latency_seconds_sum{queue_name="pod"} 4.155762530999996 484 | prometheus_sd_kubernetes_workqueue_latency_seconds_count{queue_name="pod"} 1329 485 | prometheus_sd_kubernetes_workqueue_latency_seconds_sum{queue_name="service"} 0.8281205150000001 486 | prometheus_sd_kubernetes_workqueue_latency_seconds_count{queue_name="service"} 40 487 | # HELP prometheus_sd_kubernetes_workqueue_longest_running_processor_seconds Duration of the longest running processor in the work queue. 488 | # TYPE prometheus_sd_kubernetes_workqueue_longest_running_processor_seconds gauge 489 | prometheus_sd_kubernetes_workqueue_longest_running_processor_seconds{queue_name="node"} 0 490 | prometheus_sd_kubernetes_workqueue_longest_running_processor_seconds{queue_name="pod"} 0 491 | prometheus_sd_kubernetes_workqueue_longest_running_processor_seconds{queue_name="service"} 0 492 | # HELP prometheus_sd_kubernetes_workqueue_unfinished_work_seconds How long an item has remained unfinished in the work queue. 493 | # TYPE prometheus_sd_kubernetes_workqueue_unfinished_work_seconds gauge 494 | prometheus_sd_kubernetes_workqueue_unfinished_work_seconds{queue_name="node"} 0 495 | prometheus_sd_kubernetes_workqueue_unfinished_work_seconds{queue_name="pod"} 0 496 | prometheus_sd_kubernetes_workqueue_unfinished_work_seconds{queue_name="service"} 0 497 | # HELP prometheus_sd_kubernetes_workqueue_work_duration_seconds How long processing an item from the work queue takes. 498 | # TYPE prometheus_sd_kubernetes_workqueue_work_duration_seconds summary 499 | prometheus_sd_kubernetes_workqueue_work_duration_seconds_sum{queue_name="node"} 5.840500786999983 500 | prometheus_sd_kubernetes_workqueue_work_duration_seconds_count{queue_name="node"} 35533 501 | prometheus_sd_kubernetes_workqueue_work_duration_seconds_sum{queue_name="pod"} 0.034607483000000085 502 | prometheus_sd_kubernetes_workqueue_work_duration_seconds_count{queue_name="pod"} 1329 503 | prometheus_sd_kubernetes_workqueue_work_duration_seconds_sum{queue_name="service"} 0.0010254919999999998 504 | prometheus_sd_kubernetes_workqueue_work_duration_seconds_count{queue_name="service"} 40 505 | # HELP prometheus_sd_kuma_fetch_duration_seconds The duration of a Kuma MADS fetch call. 506 | # TYPE prometheus_sd_kuma_fetch_duration_seconds summary 507 | prometheus_sd_kuma_fetch_duration_seconds{quantile="0.5"} NaN 508 | prometheus_sd_kuma_fetch_duration_seconds{quantile="0.9"} NaN 509 | prometheus_sd_kuma_fetch_duration_seconds{quantile="0.99"} NaN 510 | prometheus_sd_kuma_fetch_duration_seconds_sum 0 511 | prometheus_sd_kuma_fetch_duration_seconds_count 0 512 | # HELP prometheus_sd_kuma_fetch_failures_total The number of Kuma MADS fetch call failures. 513 | # TYPE prometheus_sd_kuma_fetch_failures_total counter 514 | prometheus_sd_kuma_fetch_failures_total 0 515 | # HELP prometheus_sd_kuma_fetch_skipped_updates_total The number of Kuma MADS fetch calls that result in no updates to the targets. 516 | # TYPE prometheus_sd_kuma_fetch_skipped_updates_total counter 517 | prometheus_sd_kuma_fetch_skipped_updates_total 0 518 | # HELP prometheus_sd_linode_failures_total Number of Linode service discovery refresh failures. 519 | # TYPE prometheus_sd_linode_failures_total counter 520 | prometheus_sd_linode_failures_total 0 521 | # HELP prometheus_sd_nomad_failures_total Number of nomad service discovery refresh failures. 522 | # TYPE prometheus_sd_nomad_failures_total counter 523 | prometheus_sd_nomad_failures_total 0 524 | # HELP prometheus_sd_received_updates_total Total number of update events received from the SD providers. 525 | # TYPE prometheus_sd_received_updates_total counter 526 | prometheus_sd_received_updates_total{name="scrape"} 36897 527 | # HELP prometheus_sd_updates_total Total number of update events sent to the SD consumers. 528 | # TYPE prometheus_sd_updates_total counter 529 | prometheus_sd_updates_total{name="scrape"} 34137 530 | # HELP prometheus_target_interval_length_seconds Actual intervals between scrapes. 531 | # TYPE prometheus_target_interval_length_seconds summary 532 | prometheus_target_interval_length_seconds{interval="15s",quantile="0.01"} 14.99914058 533 | prometheus_target_interval_length_seconds{interval="15s",quantile="0.05"} 14.999310634 534 | prometheus_target_interval_length_seconds{interval="15s",quantile="0.5"} 15.000008779 535 | prometheus_target_interval_length_seconds{interval="15s",quantile="0.9"} 15.000545764 536 | prometheus_target_interval_length_seconds{interval="15s",quantile="0.99"} 15.000857257 537 | prometheus_target_interval_length_seconds_sum{interval="15s"} 2.4210266343189236e+07 538 | prometheus_target_interval_length_seconds_count{interval="15s"} 1.614017e+06 539 | # HELP prometheus_target_metadata_cache_bytes The number of bytes that are currently used for storing metric metadata in the cache 540 | # TYPE prometheus_target_metadata_cache_bytes gauge 541 | prometheus_target_metadata_cache_bytes{scrape_job="cadvisor"} 6898 542 | prometheus_target_metadata_cache_bytes{scrape_job="kube-state-metrics"} 1933 543 | prometheus_target_metadata_cache_bytes{scrape_job="pods"} 34437 544 | # HELP prometheus_target_metadata_cache_entries Total number of metric metadata entries in the cache 545 | # TYPE prometheus_target_metadata_cache_entries gauge 546 | prometheus_target_metadata_cache_entries{scrape_job="cadvisor"} 138 547 | prometheus_target_metadata_cache_entries{scrape_job="kube-state-metrics"} 39 548 | prometheus_target_metadata_cache_entries{scrape_job="pods"} 583 549 | # HELP prometheus_target_scrape_pool_exceeded_label_limits_total Total number of times scrape pools hit the label limits, during sync or config reload. 550 | # TYPE prometheus_target_scrape_pool_exceeded_label_limits_total counter 551 | prometheus_target_scrape_pool_exceeded_label_limits_total 0 552 | # HELP prometheus_target_scrape_pool_exceeded_target_limit_total Total number of times scrape pools hit the target limit, during sync or config reload. 553 | # TYPE prometheus_target_scrape_pool_exceeded_target_limit_total counter 554 | prometheus_target_scrape_pool_exceeded_target_limit_total 0 555 | # HELP prometheus_target_scrape_pool_reloads_failed_total Total number of failed scrape pool reloads. 556 | # TYPE prometheus_target_scrape_pool_reloads_failed_total counter 557 | prometheus_target_scrape_pool_reloads_failed_total 0 558 | # HELP prometheus_target_scrape_pool_reloads_total Total number of scrape pool reloads. 559 | # TYPE prometheus_target_scrape_pool_reloads_total counter 560 | prometheus_target_scrape_pool_reloads_total 0 561 | # HELP prometheus_target_scrape_pool_sync_total Total number of syncs that were executed on a scrape pool. 562 | # TYPE prometheus_target_scrape_pool_sync_total counter 563 | prometheus_target_scrape_pool_sync_total{scrape_job="cadvisor"} 34137 564 | prometheus_target_scrape_pool_sync_total{scrape_job="kube-state-metrics"} 34137 565 | prometheus_target_scrape_pool_sync_total{scrape_job="pods"} 34137 566 | # HELP prometheus_target_scrape_pool_target_limit Maximum number of targets allowed in this scrape pool. 567 | # TYPE prometheus_target_scrape_pool_target_limit gauge 568 | prometheus_target_scrape_pool_target_limit{scrape_job="cadvisor"} 0 569 | prometheus_target_scrape_pool_target_limit{scrape_job="kube-state-metrics"} 0 570 | prometheus_target_scrape_pool_target_limit{scrape_job="pods"} 0 571 | # HELP prometheus_target_scrape_pool_targets Current number of targets in this scrape pool. 572 | # TYPE prometheus_target_scrape_pool_targets gauge 573 | prometheus_target_scrape_pool_targets{scrape_job="cadvisor"} 2 574 | prometheus_target_scrape_pool_targets{scrape_job="kube-state-metrics"} 2 575 | prometheus_target_scrape_pool_targets{scrape_job="pods"} 5 576 | # HELP prometheus_target_scrape_pools_failed_total Total number of scrape pool creations that failed. 577 | # TYPE prometheus_target_scrape_pools_failed_total counter 578 | prometheus_target_scrape_pools_failed_total 0 579 | # HELP prometheus_target_scrape_pools_total Total number of scrape pool creation attempts. 580 | # TYPE prometheus_target_scrape_pools_total counter 581 | prometheus_target_scrape_pools_total 3 582 | # HELP prometheus_target_scrapes_cache_flush_forced_total How many times a scrape cache was flushed due to getting big while scrapes are failing. 583 | # TYPE prometheus_target_scrapes_cache_flush_forced_total counter 584 | prometheus_target_scrapes_cache_flush_forced_total 0 585 | # HELP prometheus_target_scrapes_exceeded_body_size_limit_total Total number of scrapes that hit the body size limit 586 | # TYPE prometheus_target_scrapes_exceeded_body_size_limit_total counter 587 | prometheus_target_scrapes_exceeded_body_size_limit_total 0 588 | # HELP prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total Total number of scrapes that hit the native histogram bucket limit and were rejected. 589 | # TYPE prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total counter 590 | prometheus_target_scrapes_exceeded_native_histogram_bucket_limit_total 0 591 | # HELP prometheus_target_scrapes_exceeded_sample_limit_total Total number of scrapes that hit the sample limit and were rejected. 592 | # TYPE prometheus_target_scrapes_exceeded_sample_limit_total counter 593 | prometheus_target_scrapes_exceeded_sample_limit_total 0 594 | # HELP prometheus_target_scrapes_exemplar_out_of_order_total Total number of exemplar rejected due to not being out of the expected order. 595 | # TYPE prometheus_target_scrapes_exemplar_out_of_order_total counter 596 | prometheus_target_scrapes_exemplar_out_of_order_total 0 597 | # HELP prometheus_target_scrapes_sample_duplicate_timestamp_total Total number of samples rejected due to duplicate timestamps but different values. 598 | # TYPE prometheus_target_scrapes_sample_duplicate_timestamp_total counter 599 | prometheus_target_scrapes_sample_duplicate_timestamp_total 0 600 | # HELP prometheus_target_scrapes_sample_out_of_bounds_total Total number of samples rejected due to timestamp falling outside of the time bounds. 601 | # TYPE prometheus_target_scrapes_sample_out_of_bounds_total counter 602 | prometheus_target_scrapes_sample_out_of_bounds_total 0 603 | # HELP prometheus_target_scrapes_sample_out_of_order_total Total number of samples rejected due to not being out of the expected order. 604 | # TYPE prometheus_target_scrapes_sample_out_of_order_total counter 605 | prometheus_target_scrapes_sample_out_of_order_total 0 606 | # HELP prometheus_target_sync_failed_total Total number of target sync failures. 607 | # TYPE prometheus_target_sync_failed_total counter 608 | prometheus_target_sync_failed_total{scrape_job="cadvisor"} 0 609 | prometheus_target_sync_failed_total{scrape_job="kube-state-metrics"} 0 610 | prometheus_target_sync_failed_total{scrape_job="pods"} 0 611 | # HELP prometheus_target_sync_length_seconds Actual interval to sync the scrape pool. 612 | # TYPE prometheus_target_sync_length_seconds summary 613 | prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.01"} 0.00016778 614 | prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.05"} 0.00016778 615 | prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.5"} 0.000201532 616 | prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.9"} 0.000217346 617 | prometheus_target_sync_length_seconds{scrape_job="cadvisor",quantile="0.99"} 0.000217346 618 | prometheus_target_sync_length_seconds_sum{scrape_job="cadvisor"} 9.36278804700008 619 | prometheus_target_sync_length_seconds_count{scrape_job="cadvisor"} 34137 620 | prometheus_target_sync_length_seconds{scrape_job="kube-state-metrics",quantile="0.01"} 0.000148145 621 | prometheus_target_sync_length_seconds{scrape_job="kube-state-metrics",quantile="0.05"} 0.000148145 622 | prometheus_target_sync_length_seconds{scrape_job="kube-state-metrics",quantile="0.5"} 0.000175667 623 | prometheus_target_sync_length_seconds{scrape_job="kube-state-metrics",quantile="0.9"} 0.000188701 624 | prometheus_target_sync_length_seconds{scrape_job="kube-state-metrics",quantile="0.99"} 0.000188701 625 | prometheus_target_sync_length_seconds_sum{scrape_job="kube-state-metrics"} 6.007913164999995 626 | prometheus_target_sync_length_seconds_count{scrape_job="kube-state-metrics"} 34137 627 | prometheus_target_sync_length_seconds{scrape_job="pods",quantile="0.01"} 0.000867282 628 | prometheus_target_sync_length_seconds{scrape_job="pods",quantile="0.05"} 0.000867282 629 | prometheus_target_sync_length_seconds{scrape_job="pods",quantile="0.5"} 0.000913952 630 | prometheus_target_sync_length_seconds{scrape_job="pods",quantile="0.9"} 0.001163668 631 | prometheus_target_sync_length_seconds{scrape_job="pods",quantile="0.99"} 0.001163668 632 | prometheus_target_sync_length_seconds_sum{scrape_job="pods"} 44.38431514700025 633 | prometheus_target_sync_length_seconds_count{scrape_job="pods"} 34137 634 | # HELP prometheus_template_text_expansion_failures_total The total number of template text expansion failures. 635 | # TYPE prometheus_template_text_expansion_failures_total counter 636 | prometheus_template_text_expansion_failures_total 0 637 | # HELP prometheus_template_text_expansions_total The total number of template text expansions. 638 | # TYPE prometheus_template_text_expansions_total counter 639 | prometheus_template_text_expansions_total 0 640 | # HELP prometheus_treecache_watcher_goroutines The current number of watcher goroutines. 641 | # TYPE prometheus_treecache_watcher_goroutines gauge 642 | prometheus_treecache_watcher_goroutines 0 643 | # HELP prometheus_treecache_zookeeper_failures_total The total number of ZooKeeper failures. 644 | # TYPE prometheus_treecache_zookeeper_failures_total counter 645 | prometheus_treecache_zookeeper_failures_total 0 646 | # HELP prometheus_tsdb_blocks_loaded Number of currently loaded data blocks 647 | # TYPE prometheus_tsdb_blocks_loaded gauge 648 | prometheus_tsdb_blocks_loaded 16 649 | # HELP prometheus_tsdb_checkpoint_creations_failed_total Total number of checkpoint creations that failed. 650 | # TYPE prometheus_tsdb_checkpoint_creations_failed_total counter 651 | prometheus_tsdb_checkpoint_creations_failed_total 0 652 | # HELP prometheus_tsdb_checkpoint_creations_total Total number of checkpoint creations attempted. 653 | # TYPE prometheus_tsdb_checkpoint_creations_total counter 654 | prometheus_tsdb_checkpoint_creations_total 187 655 | # HELP prometheus_tsdb_checkpoint_deletions_failed_total Total number of checkpoint deletions that failed. 656 | # TYPE prometheus_tsdb_checkpoint_deletions_failed_total counter 657 | prometheus_tsdb_checkpoint_deletions_failed_total 0 658 | # HELP prometheus_tsdb_checkpoint_deletions_total Total number of checkpoint deletions attempted. 659 | # TYPE prometheus_tsdb_checkpoint_deletions_total counter 660 | prometheus_tsdb_checkpoint_deletions_total 187 661 | # HELP prometheus_tsdb_clean_start -1: lockfile is disabled. 0: a lockfile from a previous execution was replaced. 1: lockfile creation was clean 662 | # TYPE prometheus_tsdb_clean_start gauge 663 | prometheus_tsdb_clean_start -1 664 | # HELP prometheus_tsdb_compaction_chunk_range_seconds Final time range of chunks on their first compaction 665 | # TYPE prometheus_tsdb_compaction_chunk_range_seconds histogram 666 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="100"} 673 667 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="400"} 673 668 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="1600"} 673 669 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="6400"} 673 670 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="25600"} 952 671 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="102400"} 2954 672 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="409600"} 11240 673 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="1.6384e+06"} 34940 674 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="6.5536e+06"} 1.3837075e+07 675 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="2.62144e+07"} 1.3837077e+07 676 | prometheus_tsdb_compaction_chunk_range_seconds_bucket{le="+Inf"} 1.3837077e+07 677 | prometheus_tsdb_compaction_chunk_range_seconds_sum 2.9219718662064e+13 678 | prometheus_tsdb_compaction_chunk_range_seconds_count 1.3837077e+07 679 | # HELP prometheus_tsdb_compaction_chunk_samples Final number of samples on their first compaction 680 | # TYPE prometheus_tsdb_compaction_chunk_samples histogram 681 | prometheus_tsdb_compaction_chunk_samples_bucket{le="4"} 1813 682 | prometheus_tsdb_compaction_chunk_samples_bucket{le="6"} 2625 683 | prometheus_tsdb_compaction_chunk_samples_bucket{le="9"} 5359 684 | prometheus_tsdb_compaction_chunk_samples_bucket{le="13.5"} 7578 685 | prometheus_tsdb_compaction_chunk_samples_bucket{le="20.25"} 10695 686 | prometheus_tsdb_compaction_chunk_samples_bucket{le="30.375"} 14153 687 | prometheus_tsdb_compaction_chunk_samples_bucket{le="45.5625"} 20641 688 | prometheus_tsdb_compaction_chunk_samples_bucket{le="68.34375"} 26828 689 | prometheus_tsdb_compaction_chunk_samples_bucket{le="102.515625"} 37088 690 | prometheus_tsdb_compaction_chunk_samples_bucket{le="153.7734375"} 1.3192758e+07 691 | prometheus_tsdb_compaction_chunk_samples_bucket{le="230.66015625"} 1.3830353e+07 692 | prometheus_tsdb_compaction_chunk_samples_bucket{le="345.990234375"} 1.3837077e+07 693 | prometheus_tsdb_compaction_chunk_samples_bucket{le="+Inf"} 1.3837077e+07 694 | prometheus_tsdb_compaction_chunk_samples_sum 1.852852608e+09 695 | prometheus_tsdb_compaction_chunk_samples_count 1.3837077e+07 696 | # HELP prometheus_tsdb_compaction_chunk_size_bytes Final size of chunks on their first compaction 697 | # TYPE prometheus_tsdb_compaction_chunk_size_bytes histogram 698 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="32"} 5907 699 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="48"} 3.717611e+06 700 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="72"} 3.972949e+06 701 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="108"} 4.043949e+06 702 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="162"} 4.106797e+06 703 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="243"} 4.42655e+06 704 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="364.5"} 1.075848e+07 705 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="546.75"} 1.2225892e+07 706 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="820.125"} 1.3311939e+07 707 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="1230.1875"} 1.3795122e+07 708 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="1845.28125"} 1.3836776e+07 709 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="2767.921875"} 1.3837077e+07 710 | prometheus_tsdb_compaction_chunk_size_bytes_bucket{le="+Inf"} 1.3837077e+07 711 | prometheus_tsdb_compaction_chunk_size_bytes_sum 4.281044268e+09 712 | prometheus_tsdb_compaction_chunk_size_bytes_count 1.3837077e+07 713 | # HELP prometheus_tsdb_compaction_duration_seconds Duration of compaction runs 714 | # TYPE prometheus_tsdb_compaction_duration_seconds histogram 715 | prometheus_tsdb_compaction_duration_seconds_bucket{le="1"} 540 716 | prometheus_tsdb_compaction_duration_seconds_bucket{le="2"} 540 717 | prometheus_tsdb_compaction_duration_seconds_bucket{le="4"} 554 718 | prometheus_tsdb_compaction_duration_seconds_bucket{le="8"} 559 719 | prometheus_tsdb_compaction_duration_seconds_bucket{le="16"} 559 720 | prometheus_tsdb_compaction_duration_seconds_bucket{le="32"} 561 721 | prometheus_tsdb_compaction_duration_seconds_bucket{le="64"} 561 722 | prometheus_tsdb_compaction_duration_seconds_bucket{le="128"} 561 723 | prometheus_tsdb_compaction_duration_seconds_bucket{le="256"} 561 724 | prometheus_tsdb_compaction_duration_seconds_bucket{le="512"} 561 725 | prometheus_tsdb_compaction_duration_seconds_bucket{le="1024"} 561 726 | prometheus_tsdb_compaction_duration_seconds_bucket{le="2048"} 561 727 | prometheus_tsdb_compaction_duration_seconds_bucket{le="4096"} 561 728 | prometheus_tsdb_compaction_duration_seconds_bucket{le="8192"} 561 729 | prometheus_tsdb_compaction_duration_seconds_bucket{le="+Inf"} 561 730 | prometheus_tsdb_compaction_duration_seconds_sum 272.2973793669999 731 | prometheus_tsdb_compaction_duration_seconds_count 561 732 | # HELP prometheus_tsdb_compaction_populating_block Set to 1 when a block is currently being written to the disk. 733 | # TYPE prometheus_tsdb_compaction_populating_block gauge 734 | prometheus_tsdb_compaction_populating_block 0 735 | # HELP prometheus_tsdb_compactions_failed_total Total number of compactions that failed for the partition. 736 | # TYPE prometheus_tsdb_compactions_failed_total counter 737 | prometheus_tsdb_compactions_failed_total 0 738 | # HELP prometheus_tsdb_compactions_skipped_total Total number of skipped compactions due to disabled auto compaction. 739 | # TYPE prometheus_tsdb_compactions_skipped_total counter 740 | prometheus_tsdb_compactions_skipped_total 0 741 | # HELP prometheus_tsdb_compactions_total Total number of compactions that were executed for the partition. 742 | # TYPE prometheus_tsdb_compactions_total counter 743 | prometheus_tsdb_compactions_total 561 744 | # HELP prometheus_tsdb_compactions_triggered_total Total number of triggered compactions for the partition. 745 | # TYPE prometheus_tsdb_compactions_triggered_total counter 746 | prometheus_tsdb_compactions_triggered_total 44842 747 | # HELP prometheus_tsdb_data_replay_duration_seconds Time taken to replay the data on disk. 748 | # TYPE prometheus_tsdb_data_replay_duration_seconds gauge 749 | prometheus_tsdb_data_replay_duration_seconds 0.767674068 750 | # HELP prometheus_tsdb_exemplar_exemplars_appended_total Total number of appended exemplars. 751 | # TYPE prometheus_tsdb_exemplar_exemplars_appended_total counter 752 | prometheus_tsdb_exemplar_exemplars_appended_total 0 753 | # HELP prometheus_tsdb_exemplar_exemplars_in_storage Number of exemplars currently in circular storage. 754 | # TYPE prometheus_tsdb_exemplar_exemplars_in_storage gauge 755 | prometheus_tsdb_exemplar_exemplars_in_storage 0 756 | # HELP prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what timerange the current exemplar buffer limit allows. This usually means the last timestampfor all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. 757 | # TYPE prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds gauge 758 | prometheus_tsdb_exemplar_last_exemplars_timestamp_seconds 0 759 | # HELP prometheus_tsdb_exemplar_max_exemplars Total number of exemplars the exemplar storage can store, resizeable. 760 | # TYPE prometheus_tsdb_exemplar_max_exemplars gauge 761 | prometheus_tsdb_exemplar_max_exemplars 0 762 | # HELP prometheus_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. 763 | # TYPE prometheus_tsdb_exemplar_out_of_order_exemplars_total counter 764 | prometheus_tsdb_exemplar_out_of_order_exemplars_total 0 765 | # HELP prometheus_tsdb_exemplar_series_with_exemplars_in_storage Number of series with exemplars currently in circular storage. 766 | # TYPE prometheus_tsdb_exemplar_series_with_exemplars_in_storage gauge 767 | prometheus_tsdb_exemplar_series_with_exemplars_in_storage 0 768 | # HELP prometheus_tsdb_head_active_appenders Number of currently active appender transactions 769 | # TYPE prometheus_tsdb_head_active_appenders gauge 770 | prometheus_tsdb_head_active_appenders 0 771 | # HELP prometheus_tsdb_head_chunks Total number of chunks in the head block. 772 | # TYPE prometheus_tsdb_head_chunks gauge 773 | prometheus_tsdb_head_chunks 47276 774 | # HELP prometheus_tsdb_head_chunks_created_total Total number of chunks created in the head 775 | # TYPE prometheus_tsdb_head_chunks_created_total counter 776 | prometheus_tsdb_head_chunks_created_total 1.3884353e+07 777 | # HELP prometheus_tsdb_head_chunks_removed_total Total number of chunks removed in the head 778 | # TYPE prometheus_tsdb_head_chunks_removed_total counter 779 | prometheus_tsdb_head_chunks_removed_total 1.3837077e+07 780 | # HELP prometheus_tsdb_head_chunks_storage_size_bytes Size of the chunks_head directory. 781 | # TYPE prometheus_tsdb_head_chunks_storage_size_bytes gauge 782 | prometheus_tsdb_head_chunks_storage_size_bytes 2.0828256e+07 783 | # HELP prometheus_tsdb_head_gc_duration_seconds Runtime of garbage collection in the head block. 784 | # TYPE prometheus_tsdb_head_gc_duration_seconds summary 785 | prometheus_tsdb_head_gc_duration_seconds_sum 3.114924039999997 786 | prometheus_tsdb_head_gc_duration_seconds_count 373 787 | # HELP prometheus_tsdb_head_max_time Maximum timestamp of the head block. The unit is decided by the library consumer. 788 | # TYPE prometheus_tsdb_head_max_time gauge 789 | prometheus_tsdb_head_max_time 1.727807345546e+12 790 | # HELP prometheus_tsdb_head_max_time_seconds Maximum timestamp of the head block. 791 | # TYPE prometheus_tsdb_head_max_time_seconds gauge 792 | prometheus_tsdb_head_max_time_seconds 1.727807345e+09 793 | # HELP prometheus_tsdb_head_min_time Minimum time bound of the head block. The unit is decided by the library consumer. 794 | # TYPE prometheus_tsdb_head_min_time gauge 795 | prometheus_tsdb_head_min_time 1.727798400141e+12 796 | # HELP prometheus_tsdb_head_min_time_seconds Minimum time bound of the head block. 797 | # TYPE prometheus_tsdb_head_min_time_seconds gauge 798 | prometheus_tsdb_head_min_time_seconds 1.7277984e+09 799 | # HELP prometheus_tsdb_head_out_of_order_samples_appended_total Total number of appended out of order samples. 800 | # TYPE prometheus_tsdb_head_out_of_order_samples_appended_total counter 801 | prometheus_tsdb_head_out_of_order_samples_appended_total 0 802 | # HELP prometheus_tsdb_head_samples_appended_total Total number of appended samples. 803 | # TYPE prometheus_tsdb_head_samples_appended_total counter 804 | prometheus_tsdb_head_samples_appended_total{type="float"} 1.856200861e+09 805 | prometheus_tsdb_head_samples_appended_total{type="histogram"} 0 806 | # HELP prometheus_tsdb_head_series Total number of series in the head block. 807 | # TYPE prometheus_tsdb_head_series gauge 808 | prometheus_tsdb_head_series 10789 809 | # HELP prometheus_tsdb_head_series_created_total Total number of series created in the head 810 | # TYPE prometheus_tsdb_head_series_created_total counter 811 | prometheus_tsdb_head_series_created_total 42838 812 | # HELP prometheus_tsdb_head_series_not_found_total Total number of requests for series that were not found. 813 | # TYPE prometheus_tsdb_head_series_not_found_total counter 814 | prometheus_tsdb_head_series_not_found_total 0 815 | # HELP prometheus_tsdb_head_series_removed_total Total number of series removed in the head 816 | # TYPE prometheus_tsdb_head_series_removed_total counter 817 | prometheus_tsdb_head_series_removed_total 32049 818 | # HELP prometheus_tsdb_head_truncations_failed_total Total number of head truncations that failed. 819 | # TYPE prometheus_tsdb_head_truncations_failed_total counter 820 | prometheus_tsdb_head_truncations_failed_total 0 821 | # HELP prometheus_tsdb_head_truncations_total Total number of head truncations attempted. 822 | # TYPE prometheus_tsdb_head_truncations_total counter 823 | prometheus_tsdb_head_truncations_total 373 824 | # HELP prometheus_tsdb_isolation_high_watermark The highest TSDB append ID that has been given out. 825 | # TYPE prometheus_tsdb_isolation_high_watermark gauge 826 | prometheus_tsdb_isolation_high_watermark 1.614044e+06 827 | # HELP prometheus_tsdb_isolation_low_watermark The lowest TSDB append ID that is still referenced. 828 | # TYPE prometheus_tsdb_isolation_low_watermark gauge 829 | prometheus_tsdb_isolation_low_watermark 1.614044e+06 830 | # HELP prometheus_tsdb_lowest_timestamp Lowest timestamp value stored in the database. The unit is decided by the library consumer. 831 | # TYPE prometheus_tsdb_lowest_timestamp gauge 832 | prometheus_tsdb_lowest_timestamp 1.711547243455e+12 833 | # HELP prometheus_tsdb_lowest_timestamp_seconds Lowest timestamp value stored in the database. 834 | # TYPE prometheus_tsdb_lowest_timestamp_seconds gauge 835 | prometheus_tsdb_lowest_timestamp_seconds 1.711547243e+09 836 | # HELP prometheus_tsdb_mmap_chunk_corruptions_total Total number of memory-mapped chunk corruptions. 837 | # TYPE prometheus_tsdb_mmap_chunk_corruptions_total counter 838 | prometheus_tsdb_mmap_chunk_corruptions_total 0 839 | # HELP prometheus_tsdb_out_of_bound_samples_total Total number of out of bound samples ingestion failed attempts with out of order support disabled. 840 | # TYPE prometheus_tsdb_out_of_bound_samples_total counter 841 | prometheus_tsdb_out_of_bound_samples_total{type="float"} 0 842 | # HELP prometheus_tsdb_out_of_order_samples_total Total number of out of order samples ingestion failed attempts due to out of order being disabled. 843 | # TYPE prometheus_tsdb_out_of_order_samples_total counter 844 | prometheus_tsdb_out_of_order_samples_total{type="float"} 0 845 | prometheus_tsdb_out_of_order_samples_total{type="histogram"} 0 846 | # HELP prometheus_tsdb_reloads_failures_total Number of times the database failed to reloadBlocks block data from disk. 847 | # TYPE prometheus_tsdb_reloads_failures_total counter 848 | prometheus_tsdb_reloads_failures_total 0 849 | # HELP prometheus_tsdb_reloads_total Number of times the database reloaded block data from disk. 850 | # TYPE prometheus_tsdb_reloads_total counter 851 | prometheus_tsdb_reloads_total 45030 852 | # HELP prometheus_tsdb_retention_limit_bytes Max number of bytes to be retained in the tsdb blocks, configured 0 means disabled 853 | # TYPE prometheus_tsdb_retention_limit_bytes gauge 854 | prometheus_tsdb_retention_limit_bytes 5.36870912e+11 855 | # HELP prometheus_tsdb_size_retentions_total The number of times that blocks were deleted because the maximum number of bytes was exceeded. 856 | # TYPE prometheus_tsdb_size_retentions_total counter 857 | prometheus_tsdb_size_retentions_total 0 858 | # HELP prometheus_tsdb_snapshot_replay_error_total Total number snapshot replays that failed. 859 | # TYPE prometheus_tsdb_snapshot_replay_error_total counter 860 | prometheus_tsdb_snapshot_replay_error_total 0 861 | # HELP prometheus_tsdb_storage_blocks_bytes The number of bytes that are currently used for local storage by all blocks. 862 | # TYPE prometheus_tsdb_storage_blocks_bytes gauge 863 | prometheus_tsdb_storage_blocks_bytes 2.7078242758e+10 864 | # HELP prometheus_tsdb_symbol_table_size_bytes Size of symbol table in memory for loaded blocks 865 | # TYPE prometheus_tsdb_symbol_table_size_bytes gauge 866 | prometheus_tsdb_symbol_table_size_bytes 6624 867 | # HELP prometheus_tsdb_time_retentions_total The number of times that blocks were deleted because the maximum time limit was exceeded. 868 | # TYPE prometheus_tsdb_time_retentions_total counter 869 | prometheus_tsdb_time_retentions_total 0 870 | # HELP prometheus_tsdb_tombstone_cleanup_seconds The time taken to recompact blocks to remove tombstones. 871 | # TYPE prometheus_tsdb_tombstone_cleanup_seconds histogram 872 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.005"} 0 873 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.01"} 0 874 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.025"} 0 875 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.05"} 0 876 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.1"} 0 877 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.25"} 0 878 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="0.5"} 0 879 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="1"} 0 880 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="2.5"} 0 881 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="5"} 0 882 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="10"} 0 883 | prometheus_tsdb_tombstone_cleanup_seconds_bucket{le="+Inf"} 0 884 | prometheus_tsdb_tombstone_cleanup_seconds_sum 0 885 | prometheus_tsdb_tombstone_cleanup_seconds_count 0 886 | # HELP prometheus_tsdb_too_old_samples_total Total number of out of order samples ingestion failed attempts with out of support enabled, but sample outside of time window. 887 | # TYPE prometheus_tsdb_too_old_samples_total counter 888 | prometheus_tsdb_too_old_samples_total{type="float"} 0 889 | # HELP prometheus_tsdb_vertical_compactions_total Total number of compactions done on overlapping blocks. 890 | # TYPE prometheus_tsdb_vertical_compactions_total counter 891 | prometheus_tsdb_vertical_compactions_total 0 892 | # HELP prometheus_tsdb_wal_completed_pages_total Total number of completed pages. 893 | # TYPE prometheus_tsdb_wal_completed_pages_total counter 894 | prometheus_tsdb_wal_completed_pages_total 397233 895 | # HELP prometheus_tsdb_wal_corruptions_total Total number of WAL corruptions. 896 | # TYPE prometheus_tsdb_wal_corruptions_total counter 897 | prometheus_tsdb_wal_corruptions_total 0 898 | # HELP prometheus_tsdb_wal_fsync_duration_seconds Duration of write log fsync. 899 | # TYPE prometheus_tsdb_wal_fsync_duration_seconds summary 900 | prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.5"} NaN 901 | prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.9"} NaN 902 | prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.99"} NaN 903 | prometheus_tsdb_wal_fsync_duration_seconds_sum 0.805116427 904 | prometheus_tsdb_wal_fsync_duration_seconds_count 373 905 | # HELP prometheus_tsdb_wal_page_flushes_total Total number of page flushes. 906 | # TYPE prometheus_tsdb_wal_page_flushes_total counter 907 | prometheus_tsdb_wal_page_flushes_total 2.011145e+06 908 | # HELP prometheus_tsdb_wal_segment_current Write log segment index that TSDB is currently writing to. 909 | # TYPE prometheus_tsdb_wal_segment_current gauge 910 | prometheus_tsdb_wal_segment_current 2277 911 | # HELP prometheus_tsdb_wal_storage_size_bytes Size of the write log directory. 912 | # TYPE prometheus_tsdb_wal_storage_size_bytes gauge 913 | prometheus_tsdb_wal_storage_size_bytes 9.6264943e+07 914 | # HELP prometheus_tsdb_wal_truncate_duration_seconds Duration of WAL truncation. 915 | # TYPE prometheus_tsdb_wal_truncate_duration_seconds summary 916 | prometheus_tsdb_wal_truncate_duration_seconds_sum 69.80804534300002 917 | prometheus_tsdb_wal_truncate_duration_seconds_count 187 918 | # HELP prometheus_tsdb_wal_truncations_failed_total Total number of write log truncations that failed. 919 | # TYPE prometheus_tsdb_wal_truncations_failed_total counter 920 | prometheus_tsdb_wal_truncations_failed_total 0 921 | # HELP prometheus_tsdb_wal_truncations_total Total number of write log truncations attempted. 922 | # TYPE prometheus_tsdb_wal_truncations_total counter 923 | prometheus_tsdb_wal_truncations_total 187 924 | # HELP prometheus_tsdb_wal_writes_failed_total Total number of write log writes that failed. 925 | # TYPE prometheus_tsdb_wal_writes_failed_total counter 926 | prometheus_tsdb_wal_writes_failed_total 0 927 | # HELP prometheus_web_federation_errors_total Total number of errors that occurred while sending federation responses. 928 | # TYPE prometheus_web_federation_errors_total counter 929 | prometheus_web_federation_errors_total 0 930 | # HELP prometheus_web_federation_warnings_total Total number of warnings that occurred while sending federation responses. 931 | # TYPE prometheus_web_federation_warnings_total counter 932 | prometheus_web_federation_warnings_total 0 933 | # HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served. 934 | # TYPE promhttp_metric_handler_requests_in_flight gauge 935 | promhttp_metric_handler_requests_in_flight 1 936 | # HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code. 937 | # TYPE promhttp_metric_handler_requests_total counter 938 | promhttp_metric_handler_requests_total{code="200"} 179357 939 | promhttp_metric_handler_requests_total{code="500"} 0 940 | promhttp_metric_handler_requests_total{code="503"} 0 941 | -------------------------------------------------------------------------------- /cmd/mtypes/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // Package main implements mtypes CLI, see README for details. 15 | // Initially hosted and created by @bwplotka in https://github.com/bwplotka/prombenchy/pull/12. 16 | package main 17 | 18 | import ( 19 | "errors" 20 | "flag" 21 | "fmt" 22 | "io" 23 | "log" 24 | "net/http" 25 | "net/url" 26 | "os" 27 | "strings" 28 | "text/tabwriter" 29 | 30 | dto "github.com/prometheus/client_model/go" 31 | "github.com/prometheus/common/expfmt" 32 | ) 33 | 34 | type stats struct { 35 | families, series, buckets, objectives int 36 | 37 | // adjustedSeries represents series that would result in "series" in Prometheus data model 38 | // (includes _bucket, _count, _sum, _quantile). 39 | adjustedSeries int 40 | } 41 | 42 | var metricType_NATIVE_HISTOGRAM dto.MetricType = 999 //nolint:revive 43 | 44 | func main() { 45 | resource := flag.String("resource", "", "Path or URL to the resource (file, /metrics) containing Prometheus metric format.") 46 | avalancheFlagsForTotal := flag.Int("avalanche-flags-for-adjusted-series", 0, "If more than zero, it additionally prints flags for the avalanche 0.6.0 command line to generate metrics for the similar type distribution; to get the total number of adjusted series to the given value.") 47 | flag.Parse() 48 | 49 | var input io.Reader = os.Stdin 50 | if *resource != "" { 51 | switch { 52 | case strings.HasPrefix(*resource, "https://"), strings.HasPrefix(*resource, "http://"): 53 | if _, err := url.Parse(*resource); err != nil { 54 | log.Fatalf("error parsing HTTP URL to the resource %v; got %v", *resource, err) 55 | } 56 | resp, err := http.Get(*resource) 57 | if err != nil { 58 | log.Fatalf("http get against %v failed", err) 59 | } 60 | defer resp.Body.Close() 61 | input = resp.Body 62 | default: 63 | // Open the input file. 64 | file, err := os.Open(*resource) 65 | if err != nil { 66 | log.Fatalf("Error opening file: %v", err) //nolint:gocritic 67 | } 68 | defer file.Close() 69 | input = file 70 | } 71 | } 72 | statistics, err := calculateTargetStatistics(input) 73 | if err != nil { 74 | log.Fatal(err) 75 | } 76 | var total stats 77 | for _, s := range statistics { 78 | total.families += s.families 79 | total.series += s.series 80 | total.adjustedSeries += s.adjustedSeries 81 | } 82 | 83 | writeStatistics(os.Stdout, total, statistics) 84 | 85 | if *avalancheFlagsForTotal > 0 { 86 | // adjustedGoal is tracking the # of adjusted series we want to generate with avalanche. 87 | adjustedGoal := float64(*avalancheFlagsForTotal) 88 | fmt.Println() 89 | fmt.Println("Avalanche flags for the similar distribution to get to the adjusted series goal of:", adjustedGoal) 90 | 91 | adjustedGoal /= 10.0 // Assuming --series-count=10 92 | // adjustedSum is tracking the total sum of series so far (at the end hopefully adjustedSum ~= adjustedGoal) 93 | adjustedSum := 0 94 | for _, mtype := range allTypes { 95 | s := statistics[mtype] 96 | 97 | // adjustedSeriesRatio is tracking the ratio of this type in the input file. 98 | // We try to get similar ratio, but with different absolute counts, given the total sum of series we are aiming for. 99 | adjustedSeriesRatio := float64(s.adjustedSeries) / float64(total.adjustedSeries) 100 | 101 | // adjustedSeriesForType is tracking (per metric type), how many unique series of that 102 | // metric type avalanche needs to create according to the ratio we got from our input. 103 | adjustedSeriesForType := int(adjustedGoal * adjustedSeriesRatio) 104 | 105 | switch mtype { 106 | case dto.MetricType_GAUGE: 107 | fmt.Printf("--gauge-metric-count=%v\n", adjustedSeriesForType) 108 | adjustedSum += adjustedSeriesForType 109 | case dto.MetricType_COUNTER: 110 | fmt.Printf("--counter-metric-count=%v\n", adjustedSeriesForType) 111 | adjustedSum += adjustedSeriesForType 112 | case dto.MetricType_HISTOGRAM: 113 | avgBkts := s.buckets / s.series 114 | adjustedSeriesForType /= 2 + avgBkts 115 | fmt.Printf("--histogram-metric-count=%v\n", adjustedSeriesForType) 116 | fmt.Printf("--histogram-metric-bucket-count=%v\n", avgBkts-1) // -1 is due to caveat of additional +Inf not added by avalanche. 117 | adjustedSum += adjustedSeriesForType * (2 + avgBkts) 118 | case metricType_NATIVE_HISTOGRAM: 119 | fmt.Printf("--native-histogram-metric-count=%v\n", adjustedSeriesForType) 120 | adjustedSum += adjustedSeriesForType 121 | case dto.MetricType_SUMMARY: 122 | avgObjs := s.objectives / s.series 123 | adjustedSeriesForType /= 2 + avgObjs 124 | fmt.Printf("--summary-metric-count=%v\n", adjustedSeriesForType) 125 | fmt.Printf("--summary-metric-objective-count=%v\n", avgObjs) 126 | adjustedSum += adjustedSeriesForType * (2 + avgObjs) 127 | default: 128 | if s.series > 0 { 129 | log.Fatalf("not supported %v metric in avalanche", mtype) 130 | } 131 | } 132 | } 133 | fmt.Printf("--series-count=10\n") 134 | fmt.Printf("--value-interval=300 # Changes values every 5m.\n") 135 | fmt.Printf("--series-interval=3600 # 1h series churn.\n") 136 | fmt.Printf("--metric-interval=0\n") 137 | 138 | fmt.Println("This should give the total adjusted series to:", adjustedSum*10) 139 | } 140 | } 141 | 142 | var allTypes = []dto.MetricType{dto.MetricType_GAUGE, dto.MetricType_COUNTER, dto.MetricType_HISTOGRAM, metricType_NATIVE_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM, dto.MetricType_SUMMARY, dto.MetricType_UNTYPED} 143 | 144 | func writeStatistics(writer io.Writer, total stats, statistics map[dto.MetricType]stats) { 145 | w := tabwriter.NewWriter(writer, 0, 0, 4, ' ', 0) 146 | fmt.Fprintln(w, "Metric Type\tMetric Families\tSeries (adjusted)\tSeries (adjusted) %\tAverage Buckets/Objectives") 147 | 148 | for _, mtype := range allTypes { 149 | s, ok := statistics[mtype] 150 | if !ok { 151 | continue 152 | } 153 | 154 | mtypeStr := mtype.String() 155 | if mtype == metricType_NATIVE_HISTOGRAM { 156 | mtypeStr = "HISTOGRAM (native)" 157 | } 158 | 159 | seriesRatio := 100 * float64(s.series) / float64(total.series) 160 | adjustedSeriesRatio := 100 * float64(s.adjustedSeries) / float64(total.adjustedSeries) 161 | switch { 162 | case s.buckets > 0: 163 | fmt.Fprintf(w, "%s\t%d\t%d (%d)\t%f (%f)\t%f\n", mtypeStr, s.families, s.series, s.adjustedSeries, seriesRatio, adjustedSeriesRatio, float64(s.buckets)/float64(s.series)) 164 | case s.objectives > 0: 165 | fmt.Fprintf(w, "%s\t%d\t%d (%d)\t%f (%f)\t%f\n", mtypeStr, s.families, s.series, s.adjustedSeries, seriesRatio, adjustedSeriesRatio, float64(s.objectives)/float64(s.series)) 166 | default: 167 | fmt.Fprintf(w, "%s\t%d\t%d (%d)\t%f (%f)\t-\n", mtypeStr, s.families, s.series, s.adjustedSeries, seriesRatio, adjustedSeriesRatio) 168 | } 169 | } 170 | fmt.Fprintf(w, "---\t---\t---\t---\t---\n") 171 | fmt.Fprintf(w, "*\t%d\t%d (%d)\t%f (%f)\t-\n", total.families, total.series, total.adjustedSeries, 100.0, 100.0) 172 | _ = w.Flush() 173 | } 174 | 175 | func calculateTargetStatistics(r io.Reader) (statistics map[dto.MetricType]stats, _ error) { 176 | // Parse the Prometheus Text format. 177 | parser := expfmt.NewDecoder(r, expfmt.NewFormat(expfmt.TypeProtoText)) 178 | 179 | statistics = map[dto.MetricType]stats{} 180 | nativeS := statistics[metricType_NATIVE_HISTOGRAM] 181 | for { 182 | var mf dto.MetricFamily 183 | if err := parser.Decode(&mf); err != nil { 184 | if errors.Is(err, io.EOF) { 185 | break 186 | } 187 | return nil, fmt.Errorf("parsing %w", err) 188 | } 189 | 190 | s := statistics[mf.GetType()] 191 | 192 | var mfAccounted, mfAccountedNative bool 193 | switch mf.GetType() { 194 | case dto.MetricType_GAUGE_HISTOGRAM, dto.MetricType_HISTOGRAM: 195 | for _, m := range mf.GetMetric() { 196 | if m.GetHistogram().GetSchema() == 0 { 197 | // classic one. 198 | s.series++ 199 | s.buckets += len(m.GetHistogram().GetBucket()) 200 | s.adjustedSeries += 2 + len(m.GetHistogram().GetBucket()) 201 | 202 | if !mfAccounted { 203 | s.families++ 204 | mfAccounted = true 205 | } 206 | } else { 207 | // native one. 208 | nativeS.series++ 209 | nativeS.buckets += len(m.GetHistogram().GetNegativeDelta()) 210 | nativeS.buckets += len(m.GetHistogram().GetNegativeCount()) 211 | nativeS.buckets += len(m.GetHistogram().GetPositiveDelta()) 212 | nativeS.buckets += len(m.GetHistogram().GetPositiveCount()) 213 | nativeS.adjustedSeries++ 214 | 215 | if !mfAccountedNative { 216 | nativeS.families++ 217 | mfAccountedNative = true 218 | } 219 | } 220 | } 221 | case dto.MetricType_SUMMARY: 222 | s.series += len(mf.GetMetric()) 223 | s.families++ 224 | for _, m := range mf.GetMetric() { 225 | s.objectives += len(m.GetSummary().GetQuantile()) 226 | s.adjustedSeries += 2 + len(m.GetSummary().GetQuantile()) 227 | } 228 | default: 229 | s.series += len(mf.GetMetric()) 230 | s.families++ 231 | s.adjustedSeries += len(mf.GetMetric()) 232 | } 233 | statistics[mf.GetType()] = s 234 | } 235 | if nativeS.series > 0 { 236 | statistics[metricType_NATIVE_HISTOGRAM] = nativeS 237 | } 238 | return statistics, nil 239 | } 240 | -------------------------------------------------------------------------------- /example/kubernetes-deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: avalanche 5 | spec: 6 | finalizers: 7 | - kubernetes 8 | --- 9 | apiVersion: apps/v1 10 | kind: Deployment 11 | metadata: 12 | name: avalanche 13 | namespace: avalanche 14 | labels: 15 | name: avalanche 16 | spec: 17 | selector: 18 | matchLabels: 19 | app: avalanche 20 | replicas: 1 21 | template: 22 | metadata: 23 | labels: 24 | app: avalanche 25 | spec: 26 | containers: 27 | - name: avalanche 28 | image: quay.io/prometheuscommunity/avalanche:latest 29 | args: 30 | # Example metric distribution. See cmd/mtypes on how to generate different distributions 31 | # based on real scrape targets. 32 | - "--gauge-metric-count=16" 33 | - "--counter-metric-count=28" 34 | - "--histogram-metric-count=2" 35 | - "--histogram-metric-bucket-count=10" # Does not count +Inf 36 | - "--native-histogram-metric-count=0" 37 | - "--summary-metric-count=5" # One metric gives 2 series. 38 | - "--summary-metric-objective-count=2" # One metric gives 2 series. 39 | - "--series-count=10" 40 | - "--value-interval=300" # Changes values every 5m 41 | - "--series-interval=3600" # 1h series churn. 42 | - "--metric-interval=0" 43 | - "--port=9001" 44 | ports: 45 | - containerPort: 9001 46 | --- 47 | #create service avalanche-svc 48 | apiVersion: v1 49 | kind: Service 50 | metadata: 51 | name: avalanche-svc 52 | namespace: avalanche 53 | labels: 54 | app: avalanche 55 | spec: 56 | ports: 57 | # the port that this service should serve on 58 | - port: 9001 59 | targetPort: 9001 60 | name: http-avalanche 61 | type: ClusterIP 62 | clusterIP: None 63 | # label keys and values that must match in order to receive traffic for this service 64 | selector: 65 | app: avalanche 66 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/prometheus-community/avalanche 2 | 3 | go 1.22.0 4 | 5 | toolchain go1.23.1 6 | 7 | require ( 8 | github.com/google/go-cmp v0.7.0 9 | github.com/nelkinda/health-go v0.0.1 10 | github.com/oklog/run v1.1.0 11 | github.com/prometheus/client_golang v1.21.0 12 | github.com/prometheus/client_golang/exp v0.0.0-20250227122456-ad23ad6d5468 13 | github.com/prometheus/client_model v0.6.1 14 | github.com/prometheus/common v0.62.0 15 | github.com/prometheus/prometheus v0.53.1 16 | github.com/stretchr/testify v1.10.0 17 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 18 | ) 19 | 20 | require ( 21 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect 22 | github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect 23 | github.com/beorn7/perks v1.0.1 // indirect 24 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 25 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 26 | github.com/gogo/protobuf v1.3.2 // indirect 27 | github.com/klauspost/compress v1.17.11 // indirect 28 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 29 | github.com/nelkinda/http-go v0.0.1 // indirect 30 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 31 | github.com/prometheus/procfs v0.15.1 // indirect 32 | golang.org/x/sys v0.28.0 // indirect 33 | google.golang.org/protobuf v1.36.5 // indirect 34 | gopkg.in/yaml.v3 v3.0.1 // indirect 35 | ) 36 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 3 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 4 | github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 h1:t3eaIm0rUkzbrIewtiFmMK5RXHej2XnoXNhxVsAYUfg= 5 | github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= 6 | github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= 7 | github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= 8 | github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 12 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 13 | github.com/christianhujer/assert v0.0.2 h1:j+nZAzx9h4su7L8hw0NGdd93J1BtjwnTyp8jd4wiRXs= 14 | github.com/christianhujer/assert v0.0.2/go.mod h1:yszWvVhUvkosrPxaPy9FqnC6XH16zFoFs8+hPXKs4ZQ= 15 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 16 | github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= 17 | github.com/cucumber/godog v0.9.0/go.mod h1:roWCHkpeK6UTOyIRRl7IR+fgfBeZ4vZR7OSq2J/NbM4= 18 | github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg= 19 | github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY= 20 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 23 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 24 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 25 | github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= 26 | github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= 27 | github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= 28 | github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 29 | github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= 30 | github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= 31 | github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= 32 | github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= 33 | github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= 34 | github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= 35 | github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= 36 | github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= 37 | github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= 38 | github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= 39 | github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= 40 | github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= 41 | github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= 42 | github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= 43 | github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= 44 | github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= 45 | github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= 46 | github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= 47 | github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= 48 | github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= 49 | github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 50 | github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= 51 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 52 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 53 | github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 54 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 55 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 56 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 57 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 58 | github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 59 | github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= 60 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 61 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 62 | github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= 63 | github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= 64 | github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= 65 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 66 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 67 | github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 68 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 69 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 70 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 71 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 72 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 73 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 74 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 75 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 76 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 77 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 78 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 79 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 80 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 81 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 82 | github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= 83 | github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= 84 | github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= 85 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 86 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 87 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 88 | github.com/nelkinda/health-go v0.0.1 h1:HQJv+Bt9KWTCDZ9qzuGgZRaQBPsi+LAps8SP+T3DoD0= 89 | github.com/nelkinda/health-go v0.0.1/go.mod h1:oNvFVrveHIH/xPW5DqjFfdtlyhLXHFmNzULgu1Lhs5M= 90 | github.com/nelkinda/http-go v0.0.1 h1:RL3RttZzzs/kzQaVPmtv5dxMaWValqCqGNjCXyjPI1k= 91 | github.com/nelkinda/http-go v0.0.1/go.mod h1:DxPiZGVufTVSeO63nmVR5QO01TmSC0HHtEIZTHL5QEk= 92 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 93 | github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= 94 | github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= 95 | github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= 96 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 97 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 98 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 99 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 100 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 101 | github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= 102 | github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= 103 | github.com/prometheus/client_golang/exp v0.0.0-20250227122456-ad23ad6d5468 h1:Lg8dMj9NGvHGy9ORtzyCkTvHpz1rsE78HT3S9W4ZC1U= 104 | github.com/prometheus/client_golang/exp v0.0.0-20250227122456-ad23ad6d5468/go.mod h1:QdwnzTHLXXx636iZ1pfTiCI1Bn/b/20AgMqkPQr4xfA= 105 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 106 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 107 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 108 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 109 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 110 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 111 | github.com/prometheus/prometheus v0.53.1 h1:B0xu4VuVTKYrIuBMn/4YSUoIPYxs956qsOfcS4rqCuA= 112 | github.com/prometheus/prometheus v0.53.1/go.mod h1:RZDkzs+ShMBDkAPQkLEaLBXpjmDcjhNxU2drUVPgKUU= 113 | github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 114 | github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 115 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 116 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 117 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 118 | github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 119 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 120 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 121 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 122 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 123 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 124 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 125 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 126 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 127 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 128 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 129 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 130 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 131 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 132 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 133 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 134 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 135 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 136 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 137 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 138 | github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 139 | github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 140 | github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 141 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 142 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 143 | go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= 144 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 145 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 146 | golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= 147 | golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 148 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 149 | golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 150 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 151 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 152 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 153 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 154 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 155 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 156 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 157 | golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 158 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 159 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 160 | golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 161 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 162 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 163 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 164 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 165 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 166 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 167 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 168 | golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 169 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 170 | golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 171 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 172 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 173 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 174 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 175 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 176 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 177 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 178 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 179 | golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 180 | golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 181 | golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 182 | golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 183 | golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 184 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 185 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 186 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 187 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 188 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 189 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 190 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 191 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 192 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 193 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 194 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 195 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 196 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 197 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 198 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 199 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 200 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 201 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 202 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 203 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 204 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 205 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 206 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 207 | -------------------------------------------------------------------------------- /metrics/serve.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package metrics 15 | 16 | import ( 17 | "fmt" 18 | "math" 19 | "math/rand" 20 | "strings" 21 | "sync" 22 | "time" 23 | 24 | "github.com/prometheus/client_golang/prometheus" 25 | "gopkg.in/alecthomas/kingpin.v2" 26 | ) 27 | 28 | type Collector struct { 29 | cfg Config 30 | valGen *rand.Rand 31 | 32 | gauges []*prometheus.GaugeVec 33 | counters []*prometheus.CounterVec 34 | histograms []*prometheus.HistogramVec 35 | nativeHistograms []*prometheus.HistogramVec 36 | summaries []*prometheus.SummaryVec 37 | labelKeys []string 38 | 39 | updateNotifyCh chan struct{} 40 | stopCh chan struct{} 41 | valueTick *time.Ticker 42 | seriesTick *time.Ticker 43 | metricTick *time.Ticker 44 | changeSeriesTick *time.Ticker 45 | 46 | mu sync.Mutex 47 | } 48 | 49 | // NewCollector returns Prometheus collector that can be registered in registry 50 | // that handles metric creation and changes, based on the given configuration. 51 | func NewCollector(cfg Config) *Collector { 52 | c := &Collector{ 53 | cfg: cfg, 54 | valGen: rand.New(rand.NewSource(time.Now().UnixNano())), 55 | updateNotifyCh: make(chan struct{}, 1), 56 | stopCh: make(chan struct{}), 57 | } 58 | 59 | if cfg.ValueInterval > 0 { 60 | c.valueTick = time.NewTicker(time.Duration(cfg.ValueInterval) * time.Second) 61 | } 62 | if cfg.SeriesInterval > 0 { 63 | c.seriesTick = time.NewTicker(time.Duration(cfg.SeriesInterval) * time.Second) 64 | } 65 | if cfg.MetricInterval > 0 { 66 | c.metricTick = time.NewTicker(time.Duration(cfg.MetricInterval) * time.Second) 67 | } 68 | if cfg.SeriesChangeInterval > 0 { 69 | c.changeSeriesTick = time.NewTicker(time.Duration(cfg.SeriesChangeInterval) * time.Second) 70 | } 71 | return c 72 | } 73 | 74 | func (c *Collector) UpdateNotifyCh() chan struct{} { 75 | return c.updateNotifyCh 76 | } 77 | 78 | type Config struct { 79 | GaugeMetricCount, CounterMetricCount, HistogramMetricCount, NativeHistogramMetricCount, SummaryMetricCount int 80 | 81 | HistogramBuckets, SummaryObjectives int 82 | 83 | LabelCount, SeriesCount int 84 | MaxSeriesCount, MinSeriesCount int 85 | MetricLength, LabelLength int 86 | 87 | ValueInterval, SeriesInterval, MetricInterval, SeriesChangeInterval, SeriesChangeRate int 88 | 89 | SpikeMultiplier float64 90 | SeriesOperationMode opMode 91 | ConstLabels []string 92 | } 93 | 94 | func NewConfigFromFlags(flagReg func(name, help string) *kingpin.FlagClause) *Config { 95 | cfg := &Config{} 96 | // NOTE: By default avalanche creates 500 gauges, to keep old behaviour. We could break compatibility, 97 | // but it's less surprising to ask users to adjust and add more types themselves. 98 | flagReg("gauge-metric-count", "Number of gauge metrics to serve.").Default("500"). 99 | IntVar(&cfg.GaugeMetricCount) 100 | flagReg("counter-metric-count", "Number of counter metrics to serve.").Default("0"). 101 | IntVar(&cfg.CounterMetricCount) 102 | flagReg("histogram-metric-count", "Number of explicit (classic) histogram metrics to serve. Use -histogram-metric-bucket-count to control number of buckets. Note that the overall number of series for a single classic histogram metric is equal to 2 (count and sum) + + 1 (+Inf bucket).").Default("0"). 103 | IntVar(&cfg.HistogramMetricCount) 104 | flagReg("histogram-metric-bucket-count", "Number of explicit buckets (classic) histogram metrics, excluding +Inf bucket.").Default("7"). 105 | IntVar(&cfg.HistogramBuckets) 106 | flagReg("native-histogram-metric-count", "Number of native (exponential) histogram metrics to serve.").Default("0"). 107 | IntVar(&cfg.NativeHistogramMetricCount) 108 | flagReg("summary-metric-count", "Number of summary metrics to serve. Use -summary-metric-objective-count to control number of quantile objectives. Note that the overall number of series for a single summary metric is equal to 2 (count and sum) + .").Default("0"). 109 | IntVar(&cfg.SummaryMetricCount) 110 | flagReg("summary-metric-objective-count", "Number of objectives in the summary metrics to serve.").Default("2"). 111 | IntVar(&cfg.SummaryObjectives) 112 | 113 | flagReg("label-count", "Number of labels per-metric.").Default("10"). 114 | IntVar(&cfg.LabelCount) 115 | flagReg("series-count", "Number of series per-metric. This excludes the extra series (e.g. _bucket) that will be added for complex types like classic histograms and summaries.").Default("100"). 116 | IntVar(&cfg.SeriesCount) 117 | flagReg("max-series-count", "Maximum number of series to serve. Applies to 'gradual-change' mode.").Default("1000"). 118 | IntVar(&cfg.MaxSeriesCount) 119 | flagReg("min-series-count", "Minimum number of series to serve. Applies to 'gradual-change' mode.").Default("100"). 120 | IntVar(&cfg.MinSeriesCount) 121 | flagReg("spike-multiplier", "Multiplier for the spike mode.").Default("1.5"). 122 | Float64Var(&cfg.SpikeMultiplier) 123 | flagReg("metricname-length", "Modify length of metric names.").Default("5"). 124 | IntVar(&cfg.MetricLength) 125 | flagReg("labelname-length", "Modify length of label names.").Default("5"). 126 | IntVar(&cfg.LabelLength) 127 | flagReg("const-label", "Constant label to add to every metric. Format is labelName=labelValue. Flag can be specified multiple times."). 128 | StringsVar(&cfg.ConstLabels) 129 | 130 | flagReg("value-interval", "Change series values every {interval} seconds. 0 means no change.").Default("30"). 131 | IntVar(&cfg.ValueInterval) 132 | flagReg("series-interval", "Change series_id label values every {interval} seconds. 0 means no change.").Default("60"). 133 | IntVar(&cfg.SeriesInterval) 134 | flagReg("metric-interval", "Change __name__ label values every {interval} seconds. 0 means no change.").Default("0"). 135 | IntVar(&cfg.MetricInterval) 136 | flagReg("series-change-interval", "Change the number of series every {interval} seconds. Applies to 'gradual-change', 'double-halve' and 'spike' modes. 0 means no change.").Default("30"). 137 | IntVar(&cfg.SeriesChangeInterval) 138 | flagReg("series-change-rate", "The rate at which the number of active series changes over time. Applies to 'gradual-change' mode.").Default("100"). 139 | IntVar(&cfg.SeriesChangeRate) 140 | 141 | flagReg("series-operation-mode", "Mode of operation, so optional advanced behaviours on top of --value-interval, --series-interval and --metric-interval.").Default(disabledOpMode). 142 | EnumVar(&cfg.SeriesOperationMode, disabledOpMode, gradualChangeOpMode, doubleHalveOpMode, spikeOpMode) 143 | return cfg 144 | } 145 | 146 | type opMode = string 147 | 148 | const ( 149 | disabledOpMode opMode = "disabled" 150 | gradualChangeOpMode opMode = "gradual-change" 151 | doubleHalveOpMode opMode = "double-halve" 152 | spikeOpMode opMode = "spike" 153 | ) 154 | 155 | func (c Config) Validate() error { 156 | if c.MaxSeriesCount <= c.MinSeriesCount { 157 | return fmt.Errorf("--max-series-count (%d) must be greater than --min-series-count (%d)", c.MaxSeriesCount, c.MinSeriesCount) 158 | } 159 | if c.MinSeriesCount < 0 { 160 | return fmt.Errorf("--min-series-count must be 0 or higher, got %d", c.MinSeriesCount) 161 | } 162 | for _, cLabel := range c.ConstLabels { 163 | split := strings.Split(cLabel, "=") 164 | if len(split) != 2 { 165 | return fmt.Errorf("constant label argument must have format labelName=labelValue but got %s", cLabel) 166 | } 167 | } 168 | 169 | switch c.SeriesOperationMode { 170 | case gradualChangeOpMode: 171 | if c.SeriesChangeRate <= 0 { 172 | return fmt.Errorf("--series-change-rate must be greater than 0, got %d", c.SeriesChangeRate) 173 | } 174 | case spikeOpMode: 175 | if c.SpikeMultiplier < 1 { 176 | return fmt.Errorf("--spike-multiplier must be greater than or equal to 1, got %f", c.SpikeMultiplier) 177 | } 178 | case doubleHalveOpMode, disabledOpMode: 179 | default: 180 | return fmt.Errorf("unknown --series-operation-mode %v", c.SeriesOperationMode) 181 | } 182 | return nil 183 | } 184 | 185 | // Describe is used when registering metrics. It's noop avoiding us to have an easier dynamicity. 186 | // No descriptors allow this collector to be "unchecked", but more efficient with what we try to do here. 187 | func (c *Collector) Describe(chan<- *prometheus.Desc) {} 188 | 189 | func (c *Collector) Collect(metricCh chan<- prometheus.Metric) { 190 | c.mu.Lock() 191 | defer c.mu.Unlock() 192 | 193 | for _, m := range c.gauges { 194 | m.Collect(metricCh) 195 | } 196 | for _, m := range c.counters { 197 | m.Collect(metricCh) 198 | } 199 | for _, m := range c.histograms { 200 | m.Collect(metricCh) 201 | } 202 | for _, m := range c.nativeHistograms { 203 | m.Collect(metricCh) 204 | } 205 | for _, m := range c.summaries { 206 | m.Collect(metricCh) 207 | } 208 | } 209 | 210 | func help(mName string) string { 211 | return fmt.Sprintf("Metric %v is generated by https://github.com/prometheus-community/avalanche project allowing you to load test your Prometheus or Prometheus-compatible systems. It's not too long, not too short to simulate, often chunky descriptions on user metrics. It also contains metric name, so help is slighly different across metrics.", mName) 212 | } 213 | 214 | func (c *Collector) recreateMetrics(unsafeGetState readOnlyStateFn) { 215 | c.mu.Lock() 216 | defer c.mu.Unlock() 217 | s := unsafeGetState() 218 | for id := range c.gauges { 219 | mName := fmt.Sprintf("avalanche_gauge_metric_%s_%v_%v", strings.Repeat("m", c.cfg.MetricLength), s.metricCycle, id) 220 | gauge := prometheus.NewGaugeVec( 221 | prometheus.GaugeOpts{Name: mName, Help: help(mName)}, 222 | append([]string{"series_id", "cycle_id"}, c.labelKeys...), 223 | ) 224 | c.gauges[id] = gauge 225 | } 226 | for id := range c.counters { 227 | mName := fmt.Sprintf("avalanche_counter_metric_%s_%v_%v_total", strings.Repeat("m", c.cfg.MetricLength), s.metricCycle, id) 228 | counter := prometheus.NewCounterVec( 229 | prometheus.CounterOpts{Name: mName, Help: help(mName)}, 230 | append([]string{"series_id", "cycle_id"}, c.labelKeys...), 231 | ) 232 | c.counters[id] = counter 233 | } 234 | 235 | bkts := make([]float64, c.cfg.HistogramBuckets) 236 | for i := range bkts { 237 | bkts[i] = 0.0001 * math.Pow10(i) 238 | } 239 | for id := range c.histograms { 240 | mName := fmt.Sprintf("avalanche_histogram_metric_%s_%v_%v", strings.Repeat("m", c.cfg.MetricLength), s.metricCycle, id) 241 | histogram := prometheus.NewHistogramVec( 242 | prometheus.HistogramOpts{Name: mName, Help: help(mName), Buckets: bkts}, 243 | append([]string{"series_id", "cycle_id"}, c.labelKeys...), 244 | ) 245 | c.histograms[id] = histogram 246 | } 247 | 248 | for id := range c.nativeHistograms { 249 | mName := fmt.Sprintf("avalanche_native_histogram_metric_%s_%v_%v", strings.Repeat("m", c.cfg.MetricLength), s.metricCycle, id) 250 | histogram := prometheus.NewHistogramVec( 251 | prometheus.HistogramOpts{Name: mName, Help: help(mName), NativeHistogramBucketFactor: 1.1}, 252 | append([]string{"series_id", "cycle_id"}, c.labelKeys...), 253 | ) 254 | c.nativeHistograms[id] = histogram 255 | } 256 | 257 | // Mimic some quantile objectives. 258 | objectives := map[float64]float64{} 259 | if c.cfg.SummaryObjectives > 0 { 260 | parts := 100 / c.cfg.SummaryObjectives 261 | for i := 0; i < c.cfg.SummaryObjectives; i++ { 262 | q := parts * (i + 1) 263 | if q == 100 { 264 | q = 99 265 | } 266 | objectives[float64(q)/100.0] = float64(100-q) / 1000.0 267 | } 268 | } 269 | for id := range c.summaries { 270 | mName := fmt.Sprintf("avalanche_summary_metric_%s_%v_%v", strings.Repeat("m", c.cfg.MetricLength), s.metricCycle, id) 271 | summary := prometheus.NewSummaryVec( 272 | prometheus.SummaryOpts{Name: mName, Help: help(mName), Objectives: objectives}, 273 | append([]string{"series_id", "cycle_id"}, c.labelKeys...), 274 | ) 275 | c.summaries[id] = summary 276 | } 277 | } 278 | 279 | func seriesLabels(seriesID, cycleID int, labelKeys, labelValues []string) prometheus.Labels { 280 | labels := prometheus.Labels{ 281 | "series_id": fmt.Sprintf("%v", seriesID), 282 | "cycle_id": fmt.Sprintf("%v", cycleID), 283 | } 284 | for idx, key := range labelKeys { 285 | labels[key] = labelValues[idx] 286 | } 287 | return labels 288 | } 289 | 290 | type seriesDeleter interface { 291 | Delete(labels prometheus.Labels) bool 292 | } 293 | 294 | func deleteValues[T seriesDeleter](metrics []T, labelKeys []string, s metricState) { 295 | for _, metric := range metrics { 296 | for idx := 0; idx < s.seriesCount; idx++ { 297 | labels := seriesLabels(idx, s.seriesCycle, labelKeys, s.labelValues) 298 | metric.Delete(labels) 299 | } 300 | } 301 | } 302 | 303 | func (c *Collector) cycleValues(unsafeGetState readOnlyStateFn) { 304 | c.mu.Lock() 305 | defer c.mu.Unlock() 306 | 307 | s := unsafeGetState() 308 | for idx := 0; idx < s.seriesCount; idx++ { 309 | labels := seriesLabels(idx, s.seriesCycle, c.labelKeys, s.labelValues) 310 | for _, metric := range c.gauges { 311 | metric.With(labels).Set(float64(c.valGen.Intn(100))) 312 | } 313 | for _, metric := range c.counters { 314 | metric.With(labels).Add(float64(c.valGen.Intn(100))) 315 | } 316 | for _, metric := range c.histograms { 317 | metric.With(labels).Observe(float64(c.valGen.Intn(100))) 318 | } 319 | for _, metric := range c.nativeHistograms { 320 | metric.With(labels).Observe(float64(c.valGen.Intn(100))) 321 | } 322 | for _, metric := range c.summaries { 323 | metric.With(labels).Observe(float64(c.valGen.Intn(100))) 324 | } 325 | } 326 | } 327 | 328 | func (c *Collector) handleValueTicks(unsafeGetState readOnlyStateFn) { 329 | if c.valueTick == nil { 330 | return 331 | } 332 | for tick := range c.valueTick.C { 333 | fmt.Printf("%v: refreshing metric values\n", tick) 334 | c.cycleValues(unsafeGetState) 335 | 336 | select { 337 | case c.updateNotifyCh <- struct{}{}: 338 | default: 339 | } 340 | } 341 | } 342 | 343 | func (c *Collector) handleSeriesTicks(seriesCycle *int, unsafeGetState readOnlyStateFn) { 344 | if c.seriesTick == nil { 345 | return 346 | } 347 | for tick := range c.seriesTick.C { 348 | c.mu.Lock() 349 | fmt.Printf("%v: refreshing series cycle\n", tick) 350 | s := unsafeGetState() 351 | deleteValues(c.gauges, c.labelKeys, s) 352 | deleteValues(c.counters, c.labelKeys, s) 353 | deleteValues(c.histograms, c.labelKeys, s) 354 | deleteValues(c.nativeHistograms, c.labelKeys, s) 355 | deleteValues(c.summaries, c.labelKeys, s) 356 | *seriesCycle++ 357 | c.mu.Unlock() 358 | c.cycleValues(unsafeGetState) 359 | 360 | select { 361 | case c.updateNotifyCh <- struct{}{}: 362 | default: 363 | } 364 | } 365 | } 366 | 367 | func (c *Collector) handleMetricTicks(metricCycle *int, unsafeGetState readOnlyStateFn) { 368 | if c.metricTick == nil { 369 | return 370 | } 371 | 372 | for tick := range c.metricTick.C { 373 | fmt.Printf("%v: refreshing metric cycle\n", tick) 374 | *metricCycle++ 375 | c.recreateMetrics(unsafeGetState) 376 | select { 377 | case c.updateNotifyCh <- struct{}{}: 378 | default: 379 | } 380 | } 381 | } 382 | 383 | func changeSeriesGradual(seriesChangeRate, maxSeriesCount, minSeriesCount int, currentSeriesCount *int, seriesIncrease *bool) { 384 | fmt.Printf("Current series count: %d\n", *currentSeriesCount) 385 | if *seriesIncrease { 386 | *currentSeriesCount += seriesChangeRate 387 | if *currentSeriesCount >= maxSeriesCount { 388 | *currentSeriesCount = maxSeriesCount 389 | *seriesIncrease = false 390 | } 391 | } else { 392 | *currentSeriesCount -= seriesChangeRate 393 | if *currentSeriesCount < minSeriesCount { 394 | *currentSeriesCount = minSeriesCount 395 | *seriesIncrease = true 396 | } 397 | } 398 | } 399 | 400 | func changeSeriesDoubleHalve(currentSeriesCount *int, seriesIncrease *bool) { 401 | if *seriesIncrease { 402 | *currentSeriesCount *= 2 403 | } else { 404 | *currentSeriesCount /= 2 405 | if *currentSeriesCount < 1 { 406 | *currentSeriesCount = 1 407 | } 408 | } 409 | *seriesIncrease = !*seriesIncrease 410 | } 411 | 412 | func (c *Collector) handleDoubleHalveMode(seriesCount *int, unsafeGetState readOnlyStateFn) { 413 | if c.changeSeriesTick == nil { 414 | return 415 | } 416 | 417 | seriesIncrease := true 418 | for tick := range c.changeSeriesTick.C { 419 | c.recreateMetrics(unsafeGetState) 420 | c.cycleValues(unsafeGetState) 421 | 422 | c.mu.Lock() 423 | changeSeriesDoubleHalve(seriesCount, &seriesIncrease) 424 | fmt.Printf("%v: Adjusting series count. New count: %d\n", tick, *seriesCount) 425 | c.mu.Unlock() 426 | 427 | select { 428 | case c.updateNotifyCh <- struct{}{}: 429 | default: 430 | } 431 | } 432 | } 433 | 434 | func (c *Collector) handleGradualChangeMode(seriesCount *int, unsafeGetState readOnlyStateFn) { 435 | if c.changeSeriesTick == nil { 436 | return 437 | } 438 | 439 | seriesIncrease := true 440 | for tick := range c.changeSeriesTick.C { 441 | c.recreateMetrics(unsafeGetState) 442 | c.cycleValues(unsafeGetState) 443 | 444 | c.mu.Lock() 445 | changeSeriesGradual(c.cfg.SeriesChangeRate, c.cfg.MaxSeriesCount, c.cfg.MinSeriesCount, seriesCount, &seriesIncrease) 446 | fmt.Printf("%v: Adjusting series count. New count: %d\n", tick, *seriesCount) 447 | c.mu.Unlock() 448 | 449 | select { 450 | case c.updateNotifyCh <- struct{}{}: 451 | default: 452 | } 453 | } 454 | } 455 | 456 | func (c *Collector) handleSpikeMode(seriesCount *int, unsafeGetState readOnlyStateFn, spikeMultiplier float64) { 457 | if c.changeSeriesTick == nil { 458 | return 459 | } 460 | 461 | initialSeriesCount := *seriesCount 462 | for tick := range c.changeSeriesTick.C { 463 | c.recreateMetrics(unsafeGetState) 464 | c.cycleValues(unsafeGetState) 465 | 466 | c.mu.Lock() 467 | if *seriesCount > initialSeriesCount { 468 | *seriesCount = initialSeriesCount 469 | } else { 470 | *seriesCount = int(float64(initialSeriesCount) * spikeMultiplier) 471 | } 472 | fmt.Printf("%v: Adjusting series count. New count: %d\n", tick, *seriesCount) 473 | c.mu.Unlock() 474 | 475 | select { 476 | case c.updateNotifyCh <- struct{}{}: 477 | default: 478 | } 479 | } 480 | } 481 | 482 | // metricState represents current state of ids, cycles and current series. 483 | type metricState struct { 484 | seriesCount, seriesCycle, metricCycle int 485 | 486 | labelValues []string 487 | } 488 | 489 | type readOnlyStateFn func() metricState 490 | 491 | // Run creates a set of Prometheus test series that update over time. 492 | // NOTE: Only one execution of RunMetrics is currently expected. 493 | func (c *Collector) Run() error { 494 | labelKeys := make([]string, c.cfg.LabelCount) 495 | for idx := 0; idx < c.cfg.LabelCount; idx++ { 496 | labelKeys[idx] = fmt.Sprintf("label_key_%s_%v", strings.Repeat("k", c.cfg.LabelLength), idx) 497 | } 498 | labelValues := make([]string, c.cfg.LabelCount) 499 | for idx := 0; idx < c.cfg.LabelCount; idx++ { 500 | labelValues[idx] = fmt.Sprintf("label_val_%s_%v", strings.Repeat("v", c.cfg.LabelLength), idx) 501 | } 502 | for _, cLabel := range c.cfg.ConstLabels { 503 | split := strings.Split(cLabel, "=") 504 | labelKeys = append(labelKeys, split[0]) 505 | labelValues = append(labelValues, split[1]) 506 | } 507 | 508 | mutableState := &metricState{seriesCount: c.cfg.SeriesCount, labelValues: labelValues} 509 | // unsafe means you need to lock c.mu to use it. 510 | unsafeReadOnlyGetState := func() metricState { return *mutableState } 511 | 512 | c.mu.Lock() // Just to make race detector happy, not really needed in practice. 513 | c.labelKeys = labelKeys 514 | c.gauges = make([]*prometheus.GaugeVec, c.cfg.GaugeMetricCount) 515 | c.counters = make([]*prometheus.CounterVec, c.cfg.CounterMetricCount) 516 | c.histograms = make([]*prometheus.HistogramVec, c.cfg.HistogramMetricCount) 517 | c.nativeHistograms = make([]*prometheus.HistogramVec, c.cfg.NativeHistogramMetricCount) 518 | c.summaries = make([]*prometheus.SummaryVec, c.cfg.SummaryMetricCount) 519 | c.mu.Unlock() 520 | 521 | c.recreateMetrics(unsafeReadOnlyGetState) 522 | 523 | switch c.cfg.SeriesOperationMode { 524 | case doubleHalveOpMode: 525 | fmt.Printf("Starting double-halve mode; starting series: %d, change series interval: %d seconds\n", c.cfg.SeriesCount, c.cfg.SeriesChangeInterval) 526 | go c.handleDoubleHalveMode(&mutableState.seriesCount, unsafeReadOnlyGetState) 527 | 528 | case gradualChangeOpMode: 529 | fmt.Printf("Starting gradual-change mode; min series: %d, max series: %d, series change rate: %d, change series interval: %d seconds\n", c.cfg.MinSeriesCount, c.cfg.MaxSeriesCount, c.cfg.SeriesChangeRate, c.cfg.SeriesChangeInterval) 530 | c.mu.Lock() 531 | mutableState.seriesCount = c.cfg.MinSeriesCount 532 | c.mu.Unlock() 533 | go c.handleGradualChangeMode(&mutableState.seriesCount, unsafeReadOnlyGetState) 534 | 535 | case spikeOpMode: 536 | fmt.Printf("Starting spike mode; initial series: %d, spike multiplier: %f, spike interval: %v\n", c.cfg.SeriesCount, c.cfg.SpikeMultiplier, c.cfg.SeriesChangeInterval) 537 | go c.handleSpikeMode(&mutableState.seriesCount, unsafeReadOnlyGetState, c.cfg.SpikeMultiplier) 538 | } 539 | c.cycleValues(unsafeReadOnlyGetState) 540 | 541 | go c.handleValueTicks(unsafeReadOnlyGetState) 542 | go c.handleSeriesTicks(&mutableState.seriesCycle, unsafeReadOnlyGetState) 543 | go c.handleMetricTicks(&mutableState.metricCycle, unsafeReadOnlyGetState) 544 | 545 | // Mark best-effort update, so remote write knows (if enabled). 546 | select { 547 | case c.updateNotifyCh <- struct{}{}: 548 | default: 549 | } 550 | 551 | <-c.stopCh 552 | return nil 553 | } 554 | 555 | func (c *Collector) Stop(_ error) { 556 | if c.valueTick != nil { 557 | c.valueTick.Stop() 558 | } 559 | if c.seriesTick != nil { 560 | c.seriesTick.Stop() 561 | } 562 | if c.metricTick != nil { 563 | c.metricTick.Stop() 564 | } 565 | if c.changeSeriesTick != nil { 566 | c.changeSeriesTick.Stop() 567 | } 568 | close(c.stopCh) 569 | } 570 | -------------------------------------------------------------------------------- /metrics/serve_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package metrics 15 | 16 | import ( 17 | "fmt" 18 | "math" 19 | "strconv" 20 | "testing" 21 | "time" 22 | 23 | "github.com/prometheus/client_golang/prometheus" 24 | io_prometheus_client "github.com/prometheus/client_model/go" 25 | "github.com/stretchr/testify/assert" 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | // Helper function to count the series in the registry 30 | func countSeries(t *testing.T, registry *prometheus.Registry) (seriesCount int) { 31 | t.Helper() 32 | 33 | metricsFamilies, err := registry.Gather() 34 | assert.NoError(t, err) 35 | 36 | for _, mf := range metricsFamilies { 37 | for range mf.Metric { 38 | seriesCount++ 39 | } 40 | } 41 | return seriesCount 42 | } 43 | 44 | // countSeriesTypes gives exact count of all types. For complex types that are represented by counters in Prometheus 45 | // data model (and text exposition formats), we count all individual resulting series. 46 | func countSeriesTypes(t *testing.T, registry *prometheus.Registry) (gauges, counters, histograms, nhistograms, summaries int) { 47 | t.Helper() 48 | 49 | metricsFamilies, err := registry.Gather() 50 | assert.NoError(t, err) 51 | 52 | for _, mf := range metricsFamilies { 53 | for _, m := range mf.Metric { 54 | switch mf.GetType() { 55 | case io_prometheus_client.MetricType_GAUGE: 56 | gauges++ 57 | case io_prometheus_client.MetricType_COUNTER: 58 | counters++ 59 | case io_prometheus_client.MetricType_HISTOGRAM: 60 | if bkts := len(m.GetHistogram().Bucket); bkts == 0 { 61 | nhistograms++ 62 | } else { 63 | histograms += 2 // count and sum. 64 | histograms += len(m.GetHistogram().GetBucket()) 65 | if m.GetHistogram().GetBucket()[bkts-1].GetUpperBound() != math.Inf(+1) { 66 | // In the proto model we don't put explicit +Inf bucket, unless there is an exemplar, 67 | // but it will appear as series in text format and Prometheus model. Account for that. 68 | histograms++ 69 | } 70 | } 71 | case io_prometheus_client.MetricType_SUMMARY: 72 | summaries += 2 // count and sum. 73 | summaries += len(m.GetSummary().GetQuantile()) 74 | default: 75 | t.Fatalf("unknown metric type found %v", mf.GetType()) 76 | } 77 | } 78 | } 79 | return gauges, counters, histograms, nhistograms, summaries 80 | } 81 | 82 | func TestRunMetrics(t *testing.T) { 83 | testCfg := Config{ 84 | GaugeMetricCount: 200, 85 | CounterMetricCount: 200, 86 | HistogramMetricCount: 10, 87 | HistogramBuckets: 7, 88 | NativeHistogramMetricCount: 10, 89 | SummaryMetricCount: 10, 90 | SummaryObjectives: 2, 91 | SeriesOperationMode: disabledOpMode, 92 | 93 | MinSeriesCount: 0, 94 | MaxSeriesCount: 1000, 95 | LabelCount: 1, 96 | SeriesCount: 10, 97 | MetricLength: 1, 98 | LabelLength: 1, 99 | ConstLabels: []string{"constLabel=test"}, 100 | } 101 | assert.NoError(t, testCfg.Validate()) 102 | 103 | reg := prometheus.NewRegistry() 104 | coll := NewCollector(testCfg) 105 | reg.MustRegister(coll) 106 | 107 | go coll.Run() 108 | t.Cleanup(func() { 109 | coll.Stop(nil) 110 | }) 111 | 112 | time.Sleep(2 * time.Second) 113 | 114 | g, c, h, nh, s := countSeriesTypes(t, reg) 115 | assert.Equal(t, testCfg.GaugeMetricCount*testCfg.SeriesCount, g) 116 | assert.Equal(t, testCfg.CounterMetricCount*testCfg.SeriesCount, c) 117 | assert.Equal(t, (2+testCfg.HistogramBuckets+1)*testCfg.HistogramMetricCount*testCfg.SeriesCount, h) 118 | assert.Equal(t, testCfg.NativeHistogramMetricCount*testCfg.SeriesCount, nh) 119 | assert.Equal(t, (2+testCfg.SummaryObjectives)*testCfg.SummaryMetricCount*testCfg.SeriesCount, s) 120 | } 121 | 122 | func TestRunMetrics_ValueChange_SeriesCountSame(t *testing.T) { 123 | testCfg := Config{ 124 | GaugeMetricCount: 200, 125 | CounterMetricCount: 200, 126 | HistogramMetricCount: 10, 127 | HistogramBuckets: 7, 128 | NativeHistogramMetricCount: 10, 129 | SummaryMetricCount: 10, 130 | SummaryObjectives: 2, 131 | SeriesOperationMode: disabledOpMode, 132 | 133 | MinSeriesCount: 0, 134 | MaxSeriesCount: 1000, 135 | LabelCount: 1, 136 | SeriesCount: 10, 137 | MetricLength: 1, 138 | LabelLength: 1, 139 | ConstLabels: []string{"constLabel=test"}, 140 | 141 | ValueInterval: 1, // Change value every second. 142 | } 143 | assert.NoError(t, testCfg.Validate()) 144 | 145 | reg := prometheus.NewRegistry() 146 | coll := NewCollector(testCfg) 147 | reg.MustRegister(coll) 148 | 149 | go coll.Run() 150 | t.Cleanup(func() { 151 | coll.Stop(nil) 152 | }) 153 | 154 | // We can't assert value, or even it's change without mocking random generator, 155 | // but let's at least assert series count does not change. 156 | for i := 0; i < 5; i++ { 157 | time.Sleep(2 * time.Second) 158 | 159 | g, c, h, nh, s := countSeriesTypes(t, reg) 160 | assert.Equal(t, testCfg.GaugeMetricCount*testCfg.SeriesCount, g) 161 | assert.Equal(t, testCfg.CounterMetricCount*testCfg.SeriesCount, c) 162 | assert.Equal(t, (2+testCfg.HistogramBuckets+1)*testCfg.HistogramMetricCount*testCfg.SeriesCount, h) 163 | assert.Equal(t, testCfg.NativeHistogramMetricCount*testCfg.SeriesCount, nh) 164 | assert.Equal(t, (2+testCfg.SummaryObjectives)*testCfg.SummaryMetricCount*testCfg.SeriesCount, s) 165 | } 166 | } 167 | 168 | func currentCycleID(t *testing.T, registry *prometheus.Registry) (cycleID int) { 169 | t.Helper() 170 | 171 | metricsFamilies, err := registry.Gather() 172 | assert.NoError(t, err) 173 | 174 | cycleID = -1 175 | for _, mf := range metricsFamilies { 176 | for _, m := range mf.Metric { 177 | for _, l := range m.GetLabel() { 178 | if l.GetName() == "cycle_id" { 179 | gotCycleID, err := strconv.Atoi(l.GetValue()) 180 | require.NoError(t, err) 181 | 182 | if cycleID == -1 { 183 | cycleID = gotCycleID 184 | continue 185 | } 186 | if cycleID != gotCycleID { 187 | t.Fatalf("expected cycle ID to be the same across all metrics, previous metric had cycle_id=%v; now found %v", cycleID, m.GetLabel()) 188 | } 189 | } 190 | } 191 | } 192 | } 193 | return cycleID 194 | } 195 | 196 | func TestRunMetrics_SeriesChurn(t *testing.T) { 197 | testCfg := Config{ 198 | GaugeMetricCount: 200, 199 | CounterMetricCount: 200, 200 | HistogramMetricCount: 10, 201 | HistogramBuckets: 7, 202 | NativeHistogramMetricCount: 10, 203 | SummaryMetricCount: 10, 204 | SummaryObjectives: 2, 205 | SeriesOperationMode: disabledOpMode, 206 | 207 | MinSeriesCount: 0, 208 | MaxSeriesCount: 1000, 209 | LabelCount: 1, 210 | SeriesCount: 10, 211 | MetricLength: 1, 212 | LabelLength: 1, 213 | ConstLabels: []string{"constLabel=test"}, 214 | 215 | SeriesInterval: 1, // Churn series every second. 216 | // Change value every second too, there was a regression when both value and series cycle. 217 | ValueInterval: 1, 218 | } 219 | assert.NoError(t, testCfg.Validate()) 220 | 221 | reg := prometheus.NewRegistry() 222 | coll := NewCollector(testCfg) 223 | reg.MustRegister(coll) 224 | 225 | go coll.Run() 226 | t.Cleanup(func() { 227 | coll.Stop(nil) 228 | }) 229 | 230 | cycleID := -1 231 | // No matter how much time we wait, we should see always same series count, just 232 | // different cycle_id. 233 | for i := 0; i < 5; i++ { 234 | time.Sleep(2 * time.Second) 235 | 236 | g, c, h, nh, s := countSeriesTypes(t, reg) 237 | assert.Equal(t, testCfg.GaugeMetricCount*testCfg.SeriesCount, g) 238 | assert.Equal(t, testCfg.CounterMetricCount*testCfg.SeriesCount, c) 239 | assert.Equal(t, (2+testCfg.HistogramBuckets+1)*testCfg.HistogramMetricCount*testCfg.SeriesCount, h) 240 | assert.Equal(t, testCfg.NativeHistogramMetricCount*testCfg.SeriesCount, nh) 241 | assert.Equal(t, (2+testCfg.SummaryObjectives)*testCfg.SummaryMetricCount*testCfg.SeriesCount, s) 242 | 243 | gotCycleID := currentCycleID(t, reg) 244 | require.Greater(t, gotCycleID, cycleID) 245 | cycleID = gotCycleID 246 | } 247 | } 248 | 249 | func TestRunMetricsSeriesCountChangeDoubleHalve(t *testing.T) { 250 | testCfg := Config{ 251 | GaugeMetricCount: 1, 252 | LabelCount: 1, 253 | SeriesCount: 5, // Initial. 254 | MaxSeriesCount: 10, 255 | MinSeriesCount: 1, 256 | SpikeMultiplier: 1.5, 257 | SeriesChangeRate: 1, 258 | MetricLength: 1, 259 | LabelLength: 1, 260 | ValueInterval: 100, 261 | SeriesInterval: 100, 262 | MetricInterval: 100, 263 | SeriesChangeInterval: 3, 264 | SeriesOperationMode: doubleHalveOpMode, 265 | ConstLabels: []string{"constLabel=test"}, 266 | } 267 | assert.NoError(t, testCfg.Validate()) 268 | 269 | reg := prometheus.NewRegistry() 270 | coll := NewCollector(testCfg) 271 | reg.MustRegister(coll) 272 | 273 | go coll.Run() 274 | t.Cleanup(func() { 275 | coll.Stop(nil) 276 | }) 277 | 278 | time.Sleep(2 * time.Second) 279 | for i := 0; i < 4; i++ { 280 | time.Sleep(time.Duration(testCfg.SeriesChangeInterval) * time.Second) 281 | if i%2 == 0 { // Expecting halved series count 282 | currentCount := countSeries(t, reg) 283 | expectedCount := testCfg.SeriesCount 284 | assert.Equal(t, expectedCount, currentCount, "Halved series count should be %d but got %d", expectedCount, currentCount) 285 | } else { // Expecting doubled series count 286 | currentCount := countSeries(t, reg) 287 | expectedCount := testCfg.SeriesCount * 2 288 | assert.Equal(t, expectedCount, currentCount, "Doubled series count should be %d but got %d", expectedCount, currentCount) 289 | } 290 | } 291 | } 292 | 293 | func TestRunMetricsGradualChange(t *testing.T) { 294 | testCfg := Config{ 295 | GaugeMetricCount: 1, 296 | LabelCount: 1, 297 | SeriesCount: 100, // Initial. 298 | MaxSeriesCount: 30, 299 | MinSeriesCount: 10, 300 | SpikeMultiplier: 1.5, 301 | SeriesChangeRate: 10, 302 | MetricLength: 1, 303 | LabelLength: 1, 304 | ValueInterval: 100, 305 | SeriesInterval: 100, 306 | MetricInterval: 100, 307 | SeriesChangeInterval: 3, 308 | SeriesOperationMode: gradualChangeOpMode, 309 | ConstLabels: []string{"constLabel=test"}, 310 | } 311 | assert.NoError(t, testCfg.Validate()) 312 | 313 | reg := prometheus.NewRegistry() 314 | coll := NewCollector(testCfg) 315 | reg.MustRegister(coll) 316 | 317 | go coll.Run() 318 | t.Cleanup(func() { 319 | coll.Stop(nil) 320 | }) 321 | 322 | time.Sleep(2 * time.Second) 323 | currentCount := countSeries(t, reg) 324 | fmt.Println("seriesCount: ", currentCount) 325 | assert.Equal(t, testCfg.MinSeriesCount, currentCount, "Initial series count should be minSeriesCount %d but got %d", testCfg.MinSeriesCount, currentCount) 326 | 327 | assert.Eventually(t, func() bool { 328 | graduallyIncreasedCount := countSeries(t, reg) 329 | fmt.Println("seriesCount: ", graduallyIncreasedCount) 330 | if graduallyIncreasedCount > testCfg.MaxSeriesCount { 331 | t.Fatalf("Gradually increased series count should be less than maxSeriesCount %d but got %d", testCfg.MaxSeriesCount, graduallyIncreasedCount) 332 | } 333 | if currentCount > graduallyIncreasedCount { 334 | t.Fatalf("Gradually increased series count should be greater than initial series count %d but got %d", currentCount, graduallyIncreasedCount) 335 | } else { 336 | currentCount = graduallyIncreasedCount 337 | } 338 | 339 | return graduallyIncreasedCount == testCfg.MaxSeriesCount 340 | }, 15*time.Second, time.Duration(testCfg.SeriesChangeInterval)*time.Second, "Did not receive update notification for series count gradual increase in time") 341 | 342 | assert.Eventually(t, func() bool { 343 | graduallyIncreasedCount := countSeries(t, reg) 344 | fmt.Println("seriesCount: ", graduallyIncreasedCount) 345 | if graduallyIncreasedCount < testCfg.MinSeriesCount { 346 | t.Fatalf("Gradually increased series count should be less than maxSeriesCount %d but got %d", testCfg.MaxSeriesCount, graduallyIncreasedCount) 347 | } 348 | 349 | return graduallyIncreasedCount == testCfg.MinSeriesCount 350 | }, 15*time.Second, time.Duration(testCfg.SeriesChangeInterval)*time.Second, "Did not receive update notification for series count gradual increase in time") 351 | } 352 | 353 | func TestRunMetricsWithInvalidSeriesCounts(t *testing.T) { 354 | testCfg := Config{ 355 | GaugeMetricCount: 1, 356 | LabelCount: 1, 357 | SeriesCount: 100, 358 | MaxSeriesCount: 10, 359 | MinSeriesCount: 100, 360 | SpikeMultiplier: 1.5, 361 | SeriesChangeRate: 10, 362 | MetricLength: 1, 363 | LabelLength: 1, 364 | ValueInterval: 100, 365 | SeriesInterval: 100, 366 | MetricInterval: 100, 367 | SeriesChangeInterval: 3, 368 | SeriesOperationMode: gradualChangeOpMode, 369 | ConstLabels: []string{"constLabel=test"}, 370 | } 371 | assert.Error(t, testCfg.Validate()) 372 | } 373 | 374 | func TestRunMetricsSpikeChange(t *testing.T) { 375 | testCfg := Config{ 376 | GaugeMetricCount: 1, 377 | LabelCount: 1, 378 | SeriesCount: 100, 379 | MaxSeriesCount: 30, 380 | MinSeriesCount: 10, 381 | SpikeMultiplier: 1.5, 382 | SeriesChangeRate: 10, 383 | MetricLength: 1, 384 | LabelLength: 1, 385 | ValueInterval: 100, 386 | SeriesInterval: 100, 387 | MetricInterval: 100, 388 | SeriesChangeInterval: 10, 389 | SeriesOperationMode: spikeOpMode, 390 | ConstLabels: []string{"constLabel=test"}, 391 | } 392 | assert.NoError(t, testCfg.Validate()) 393 | 394 | reg := prometheus.NewRegistry() 395 | coll := NewCollector(testCfg) 396 | reg.MustRegister(coll) 397 | 398 | go coll.Run() 399 | t.Cleanup(func() { 400 | coll.Stop(nil) 401 | }) 402 | 403 | time.Sleep(2 * time.Second) 404 | for i := 0; i < 4; i++ { 405 | time.Sleep(time.Duration(testCfg.SeriesChangeInterval) * time.Second) 406 | if i%2 == 0 { 407 | currentCount := countSeries(t, reg) 408 | expectedCount := testCfg.SeriesCount 409 | assert.Equal(t, expectedCount, currentCount, fmt.Sprintf("Halved series count should be %d but got %d", expectedCount, currentCount)) 410 | } else { 411 | currentCount := countSeries(t, reg) 412 | expectedCount := int(float64(testCfg.SeriesCount) * testCfg.SpikeMultiplier) 413 | assert.Equal(t, expectedCount, currentCount, fmt.Sprintf("Multiplied the series count by %.1f, should be %d but got %d", testCfg.SpikeMultiplier, expectedCount, currentCount)) 414 | } 415 | } 416 | } 417 | 418 | func TestCollectorLabels(t *testing.T) { 419 | testCfg := Config{ 420 | GaugeMetricCount: 1, 421 | LabelCount: 2, 422 | SeriesCount: 100, 423 | MaxSeriesCount: 30, 424 | MinSeriesCount: 10, 425 | SpikeMultiplier: 1.5, 426 | MetricLength: 1, 427 | LabelLength: 1, 428 | SeriesOperationMode: spikeOpMode, 429 | ConstLabels: []string{"constLabel=test"}, 430 | } 431 | 432 | assert.NoError(t, testCfg.Validate()) 433 | 434 | reg := prometheus.NewRegistry() 435 | col := NewCollector(testCfg) 436 | reg.MustRegister(col) 437 | 438 | go col.Run() 439 | t.Cleanup(func() { 440 | col.Stop(nil) 441 | }) 442 | 443 | select { 444 | case <-col.updateNotifyCh: 445 | metricsFamilies, err := reg.Gather() 446 | assert.NotEmpty(t, metricsFamilies) 447 | assert.NoError(t, err) 448 | 449 | for _, mf := range metricsFamilies { 450 | for _, m := range mf.Metric { 451 | labels := m.GetLabel() 452 | labelMap := make(map[string]string) 453 | for _, l := range labels { 454 | labelMap[l.GetName()] = l.GetValue() 455 | } 456 | assert.Equal(t, "test", labelMap["constLabel"]) 457 | assert.Contains(t, labelMap, "label_key_k_0") 458 | assert.Contains(t, labelMap, "label_key_k_1") 459 | assert.Contains(t, labelMap, "series_id") 460 | assert.Contains(t, labelMap, "cycle_id") 461 | } 462 | } 463 | case <-time.After(2 * time.Second): 464 | t.Fail() 465 | } 466 | } 467 | -------------------------------------------------------------------------------- /metrics/write.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package metrics 15 | 16 | import ( 17 | "context" 18 | "crypto/tls" 19 | "fmt" 20 | "log" 21 | "log/slog" 22 | "net/http" 23 | "net/url" 24 | "sort" 25 | "sync" 26 | "time" 27 | 28 | "github.com/prometheus/client_golang/exp/api/remote" 29 | "github.com/prometheus/client_golang/prometheus" 30 | dto "github.com/prometheus/client_model/go" 31 | "github.com/prometheus/common/model" 32 | "github.com/prometheus/prometheus/prompb" 33 | "gopkg.in/alecthomas/kingpin.v2" 34 | 35 | "github.com/prometheus-community/avalanche/pkg/errors" 36 | ) 37 | 38 | // ConfigWrite for the remote write requests. 39 | type ConfigWrite struct { 40 | URL *url.URL 41 | RequestInterval time.Duration 42 | BatchSize, 43 | RequestCount int 44 | UpdateNotify chan struct{} 45 | PprofURLs []*url.URL 46 | Tenant string 47 | TLSClientConfig tls.Config 48 | TenantHeader string 49 | OutOfOrder bool 50 | Concurrency int 51 | WriteV2 bool 52 | } 53 | 54 | func NewWriteConfigFromFlags(flagReg func(name, help string) *kingpin.FlagClause) *ConfigWrite { 55 | cfg := &ConfigWrite{} 56 | flagReg("remote-url", "URL to send samples via remote_write API. By default, path is set to api/v1/write"). 57 | URLVar(&cfg.URL) 58 | flagReg("remote-concurrency-limit", "how many concurrent writes can happen at any given time").Default("20"). 59 | IntVar(&cfg.Concurrency) 60 | flagReg("remote-batch-size", "how many samples to send with each remote_write API request.").Default("2000"). 61 | IntVar(&cfg.BatchSize) 62 | flagReg("remote-requests-count", "How many requests to send in total to the remote_write API. Set to -1 to run indefinitely.").Default("100"). 63 | IntVar(&cfg.RequestCount) 64 | flagReg("remote-write-interval", "delay between each remote write request.").Default("100ms"). 65 | DurationVar(&cfg.RequestInterval) 66 | flagReg("remote-tenant", "Tenant ID to include in remote_write send").Default("0"). 67 | StringVar(&cfg.Tenant) 68 | flagReg("tls-client-insecure", "Skip certificate check on tls connection").Default("false"). 69 | BoolVar(&cfg.TLSClientConfig.InsecureSkipVerify) 70 | flagReg("remote-tenant-header", "Tenant ID to include in remote_write send. The default, is the default tenant header expected by Cortex.").Default("X-Scope-OrgID"). 71 | StringVar(&cfg.TenantHeader) 72 | // TODO(bwplotka): Make this a non-bool flag (e.g. out-of-order-min-time). 73 | flagReg("remote-out-of-order", "Enable out-of-order timestamps in remote write requests").Default("true"). 74 | BoolVar(&cfg.OutOfOrder) 75 | flagReg("remote-write-v2", "Send remote write v2 format requests.").Default("false"). 76 | BoolVar(&cfg.WriteV2) 77 | 78 | return cfg 79 | } 80 | 81 | func (c *ConfigWrite) Validate() error { 82 | if c.URL != nil { 83 | if c.URL.Host == "" || c.URL.Scheme == "" { 84 | return fmt.Errorf("remote host and scheme can't be empty") 85 | } 86 | 87 | if c.BatchSize <= 0 { 88 | return fmt.Errorf("remote send batch sizemust be greater than 0, got %v", c.BatchSize) 89 | } 90 | 91 | if c.RequestInterval <= 0 { 92 | return fmt.Errorf("--remote-write-interval must be greater than 0, got %v", c.RequestInterval) 93 | } 94 | } 95 | return nil 96 | } 97 | 98 | // Writer for remote write requests. 99 | type Writer struct { 100 | logger *slog.Logger 101 | timeout time.Duration 102 | config *ConfigWrite 103 | gatherer prometheus.Gatherer 104 | remoteAPI *remote.API 105 | } 106 | 107 | // RunRemoteWriting initializes a http client and starts a Writer for remote writing metrics to a prometheus compatible remote endpoint. 108 | func RunRemoteWriting(ctx context.Context, logger *slog.Logger, cfg *ConfigWrite, gatherer prometheus.Gatherer) error { 109 | var rt http.RoundTripper = &http.Transport{ 110 | TLSClientConfig: &cfg.TLSClientConfig, 111 | } 112 | rt = &tenantRoundTripper{tenant: cfg.Tenant, tenantHeader: cfg.TenantHeader, rt: rt} 113 | rt = &userAgentRoundTripper{userAgent: "avalanche", rt: rt} 114 | httpClient := &http.Client{Transport: rt} 115 | 116 | remoteAPI, err := remote.NewAPI( 117 | cfg.URL.String(), 118 | remote.WithAPIHTTPClient(httpClient), 119 | remote.WithAPILogger(logger.With("component", "remote_write_api")), 120 | ) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | writer := Writer{ 126 | logger: logger, 127 | timeout: time.Minute, 128 | config: cfg, 129 | gatherer: gatherer, 130 | remoteAPI: remoteAPI, 131 | } 132 | 133 | if cfg.WriteV2 { 134 | return writer.writeV2(ctx) 135 | } 136 | 137 | return writer.write(ctx) 138 | } 139 | 140 | // Add the tenant ID header 141 | func (rt *tenantRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 142 | req = cloneRequest(req) 143 | req.Header.Set(rt.tenantHeader, rt.tenant) 144 | return rt.rt.RoundTrip(req) 145 | } 146 | 147 | type tenantRoundTripper struct { 148 | tenant string 149 | tenantHeader string 150 | rt http.RoundTripper 151 | } 152 | 153 | type userAgentRoundTripper struct { 154 | userAgent string 155 | rt http.RoundTripper 156 | } 157 | 158 | func (rt *userAgentRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 159 | req = cloneRequest(req) 160 | req.Header.Set("User-Agent", rt.userAgent) 161 | return rt.rt.RoundTrip(req) 162 | } 163 | 164 | // cloneRequest returns a clone of the provided *http.Request. 165 | // The clone is a shallow copy of the struct and its Header map. 166 | func cloneRequest(r *http.Request) *http.Request { 167 | // Shallow copy of the struct. 168 | r2 := new(http.Request) 169 | *r2 = *r 170 | // Deep copy of the Header. 171 | r2.Header = make(http.Header) 172 | for k, s := range r.Header { 173 | r2.Header[k] = s 174 | } 175 | return r2 176 | } 177 | 178 | func (w *Writer) write(ctx context.Context) error { 179 | select { 180 | // Wait for update first as write and collector.Run runs simultaneously. 181 | case <-w.config.UpdateNotify: 182 | case <-ctx.Done(): 183 | return ctx.Err() 184 | } 185 | 186 | tss, err := collectMetrics(w.gatherer, w.config.OutOfOrder) 187 | if err != nil { 188 | return err 189 | } 190 | 191 | var ( 192 | totalTime time.Duration 193 | totalSamplesExp = len(tss) * w.config.RequestCount 194 | totalSamplesAct int 195 | mtx sync.Mutex 196 | wgMetrics sync.WaitGroup 197 | merr = &errors.MultiError{} 198 | ) 199 | 200 | shouldRunForever := w.config.RequestCount == -1 201 | if shouldRunForever { 202 | log.Printf("Sending: %v timeseries infinitely, %v timeseries per request, %v delay between requests\n", 203 | len(tss), w.config.BatchSize, w.config.RequestInterval) 204 | } else { 205 | log.Printf("Sending: %v timeseries, %v times, %v timeseries per request, %v delay between requests\n", 206 | len(tss), w.config.RequestCount, w.config.BatchSize, w.config.RequestInterval) 207 | } 208 | 209 | ticker := time.NewTicker(w.config.RequestInterval) 210 | defer ticker.Stop() 211 | 212 | concurrencyLimitCh := make(chan struct{}, w.config.Concurrency) 213 | 214 | for i := 0; ; { 215 | if ctx.Err() != nil { 216 | return ctx.Err() 217 | } 218 | 219 | if !shouldRunForever { 220 | if i >= w.config.RequestCount { 221 | break 222 | } 223 | i++ 224 | } 225 | 226 | <-ticker.C 227 | select { 228 | case <-w.config.UpdateNotify: 229 | log.Println("updating remote write metrics") 230 | tss, err = collectMetrics(w.gatherer, w.config.OutOfOrder) 231 | if err != nil { 232 | merr.Add(err) 233 | } 234 | default: 235 | tss = updateTimetamps(tss) 236 | } 237 | 238 | start := time.Now() 239 | for i := 0; i < len(tss); i += w.config.BatchSize { 240 | wgMetrics.Add(1) 241 | concurrencyLimitCh <- struct{}{} 242 | go func(i int) { 243 | defer func() { 244 | <-concurrencyLimitCh 245 | }() 246 | defer wgMetrics.Done() 247 | end := i + w.config.BatchSize 248 | if end > len(tss) { 249 | end = len(tss) 250 | } 251 | req := &prompb.WriteRequest{ 252 | Timeseries: tss[i:end], 253 | } 254 | 255 | if _, err := w.remoteAPI.Write(ctx, remote.WriteV1MessageType, req); err != nil { 256 | merr.Add(err) 257 | w.logger.Error("error writing metrics", "error", err) 258 | return 259 | } 260 | 261 | mtx.Lock() 262 | totalSamplesAct += len(tss[i:end]) 263 | mtx.Unlock() 264 | }(i) 265 | } 266 | wgMetrics.Wait() 267 | totalTime += time.Since(start) 268 | if merr.Count() > 20 { 269 | merr.Add(fmt.Errorf("too many errors")) 270 | return merr.Err() 271 | } 272 | } 273 | if w.config.RequestCount*len(tss) != totalSamplesAct { 274 | merr.Add(fmt.Errorf("total samples mismatch, exp:%v , act:%v", totalSamplesExp, totalSamplesAct)) 275 | } 276 | w.logger.Info("metrics summary", 277 | "total_time", totalTime.Round(time.Second), 278 | "total_samples", totalSamplesAct, 279 | "samples_per_sec", int(float64(totalSamplesAct)/totalTime.Seconds()), 280 | "errors", merr.Count()) 281 | return merr.Err() 282 | } 283 | 284 | func updateTimetamps(tss []prompb.TimeSeries) []prompb.TimeSeries { 285 | t := int64(model.Now()) 286 | for i := range tss { 287 | tss[i].Samples[0].Timestamp = t 288 | } 289 | return tss 290 | } 291 | 292 | func collectMetrics(gatherer prometheus.Gatherer, outOfOrder bool) ([]prompb.TimeSeries, error) { 293 | metricFamilies, err := gatherer.Gather() 294 | if err != nil { 295 | return nil, err 296 | } 297 | tss := ToTimeSeriesSlice(metricFamilies) 298 | if outOfOrder { 299 | tss = shuffleTimestamps(tss) 300 | } 301 | return tss, nil 302 | } 303 | 304 | func shuffleTimestamps(tss []prompb.TimeSeries) []prompb.TimeSeries { 305 | now := time.Now().UnixMilli() 306 | offsets := []int64{0, -60 * 1000, -5 * 60 * 1000} 307 | 308 | for i := range tss { 309 | offset := offsets[i%len(offsets)] 310 | tss[i].Samples[0].Timestamp = now + offset 311 | } 312 | return tss 313 | } 314 | 315 | // ToTimeSeriesSlice converts a slice of metricFamilies containing samples into a slice of TimeSeries 316 | func ToTimeSeriesSlice(metricFamilies []*dto.MetricFamily) []prompb.TimeSeries { 317 | tss := make([]prompb.TimeSeries, 0, len(metricFamilies)*10) 318 | timestamp := int64(model.Now()) // Not using metric.TimestampMs because it is (always?) nil. Is this right? 319 | 320 | skippedSamples := 0 321 | for _, metricFamily := range metricFamilies { 322 | for _, metric := range metricFamily.Metric { 323 | labels := prompbLabels(*metricFamily.Name, metric.Label) 324 | ts := prompb.TimeSeries{ 325 | Labels: labels, 326 | } 327 | switch *metricFamily.Type { 328 | case dto.MetricType_COUNTER: 329 | ts.Samples = []prompb.Sample{{ 330 | Value: *metric.Counter.Value, 331 | Timestamp: timestamp, 332 | }} 333 | tss = append(tss, ts) 334 | case dto.MetricType_GAUGE: 335 | ts.Samples = []prompb.Sample{{ 336 | Value: *metric.Gauge.Value, 337 | Timestamp: timestamp, 338 | }} 339 | tss = append(tss, ts) 340 | default: 341 | skippedSamples++ 342 | } 343 | } 344 | } 345 | if skippedSamples > 0 { 346 | log.Printf("WARN: Skipping %v samples; sending only %v samples, given only gauge and counters are currently implemented\n", skippedSamples, len(tss)) 347 | } 348 | return tss 349 | } 350 | 351 | func prompbLabels(name string, label []*dto.LabelPair) []prompb.Label { 352 | ret := make([]prompb.Label, 0, len(label)+1) 353 | ret = append(ret, prompb.Label{ 354 | Name: model.MetricNameLabel, 355 | Value: name, 356 | }) 357 | for _, pair := range label { 358 | ret = append(ret, prompb.Label{ 359 | Name: *pair.Name, 360 | Value: *pair.Value, 361 | }) 362 | } 363 | sort.Slice(ret, func(i, j int) bool { 364 | return ret[i].Name < ret[j].Name 365 | }) 366 | return ret 367 | } 368 | -------------------------------------------------------------------------------- /metrics/write_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package metrics 15 | 16 | import ( 17 | "testing" 18 | "time" 19 | 20 | "github.com/prometheus/prometheus/prompb" 21 | ) 22 | 23 | func TestShuffleTimestamps(t *testing.T) { 24 | now := time.Now().UnixMilli() 25 | 26 | tss := []prompb.TimeSeries{ 27 | {Samples: []prompb.Sample{{Timestamp: now}}}, 28 | {Samples: []prompb.Sample{{Timestamp: now}}}, 29 | {Samples: []prompb.Sample{{Timestamp: now}}}, 30 | } 31 | 32 | shuffledTSS := shuffleTimestamps(tss) 33 | 34 | offsets := []int64{0, -60 * 1000, -5 * 60 * 1000} 35 | for _, ts := range shuffledTSS { 36 | timestampValid := false 37 | for _, offset := range offsets { 38 | expectedTimestamp := now + offset 39 | if ts.Samples[0].Timestamp == expectedTimestamp { 40 | timestampValid = true 41 | break 42 | } 43 | } 44 | if !timestampValid { 45 | t.Errorf("Timestamp %v is not in the expected offsets: %v", ts.Samples[0].Timestamp, offsets) 46 | } 47 | } 48 | 49 | outOfOrder := false 50 | for i := 1; i < len(shuffledTSS); i++ { 51 | if shuffledTSS[i].Samples[0].Timestamp < shuffledTSS[i-1].Samples[0].Timestamp { 52 | outOfOrder = true 53 | break 54 | } 55 | } 56 | 57 | if !outOfOrder { 58 | t.Error("Timestamps are not out of order") 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /metrics/writev2.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package metrics 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | "log" 20 | "sync" 21 | "time" 22 | 23 | "github.com/prometheus/client_golang/exp/api/remote" 24 | writev2 "github.com/prometheus/client_golang/exp/api/remote/genproto/v2" 25 | "github.com/prometheus/client_golang/prometheus" 26 | dto "github.com/prometheus/client_model/go" 27 | "github.com/prometheus/common/model" 28 | 29 | "github.com/prometheus-community/avalanche/pkg/errors" 30 | ) 31 | 32 | func (w *Writer) writeV2(ctx context.Context) error { 33 | select { 34 | // Wait for update first as write and collector.Run runs simultaneously. 35 | case <-w.config.UpdateNotify: 36 | case <-ctx.Done(): 37 | return ctx.Err() 38 | } 39 | 40 | tss, st, err := collectMetricsV2(w.gatherer, w.config.OutOfOrder) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | var ( 46 | totalTime time.Duration 47 | totalSamplesExp = len(tss) * w.config.RequestCount 48 | totalSamplesAct int 49 | mtx sync.Mutex 50 | wgMetrics sync.WaitGroup 51 | merr = &errors.MultiError{} 52 | ) 53 | 54 | shouldRunForever := w.config.RequestCount == -1 55 | if shouldRunForever { 56 | log.Printf("Sending: %v timeseries infinitely, %v timeseries per request, %v delay between requests\n", 57 | len(tss), w.config.BatchSize, w.config.RequestInterval) 58 | } else { 59 | log.Printf("Sending: %v timeseries, %v times, %v timeseries per request, %v delay between requests\n", 60 | len(tss), w.config.RequestCount, w.config.BatchSize, w.config.RequestInterval) 61 | } 62 | 63 | ticker := time.NewTicker(w.config.RequestInterval) 64 | defer ticker.Stop() 65 | 66 | concurrencyLimitCh := make(chan struct{}, w.config.Concurrency) 67 | 68 | for i := 0; ; { 69 | if ctx.Err() != nil { 70 | return ctx.Err() 71 | } 72 | 73 | if !shouldRunForever { 74 | if i >= w.config.RequestCount { 75 | break 76 | } 77 | i++ 78 | } 79 | 80 | <-ticker.C 81 | select { 82 | case <-w.config.UpdateNotify: 83 | log.Println("updating remote write metrics") 84 | tss, st, err = collectMetricsV2(w.gatherer, w.config.OutOfOrder) 85 | if err != nil { 86 | merr.Add(err) 87 | } 88 | default: 89 | tss = updateTimestampsV2(tss) 90 | } 91 | 92 | start := time.Now() 93 | for i := 0; i < len(tss); i += w.config.BatchSize { 94 | wgMetrics.Add(1) 95 | concurrencyLimitCh <- struct{}{} 96 | go func(i int) { 97 | defer func() { 98 | <-concurrencyLimitCh 99 | }() 100 | defer wgMetrics.Done() 101 | end := i + w.config.BatchSize 102 | if end > len(tss) { 103 | end = len(tss) 104 | } 105 | req := &writev2.Request{ 106 | Timeseries: tss[i:end], 107 | Symbols: st.Symbols(), // We pass full symbols table to each request for now 108 | } 109 | 110 | if _, err := w.remoteAPI.Write(ctx, remote.WriteV2MessageType, req); err != nil { 111 | merr.Add(err) 112 | w.logger.Error("error writing metrics", "error", err) 113 | return 114 | } 115 | 116 | mtx.Lock() 117 | totalSamplesAct += len(tss[i:end]) 118 | mtx.Unlock() 119 | }(i) 120 | } 121 | wgMetrics.Wait() 122 | totalTime += time.Since(start) 123 | if merr.Count() > 20 { 124 | merr.Add(fmt.Errorf("too many errors")) 125 | return merr.Err() 126 | } 127 | } 128 | if w.config.RequestCount*len(tss) != totalSamplesAct { 129 | merr.Add(fmt.Errorf("total samples mismatch, exp:%v , act:%v", totalSamplesExp, totalSamplesAct)) 130 | } 131 | w.logger.Info("metrics summary", 132 | "total_time", totalTime.Round(time.Second), 133 | "total_samples", totalSamplesAct, 134 | "samples_per_sec", int(float64(totalSamplesAct)/totalTime.Seconds()), 135 | "errors", merr.Count()) 136 | return merr.Err() 137 | } 138 | 139 | func updateTimestampsV2(tss []*writev2.TimeSeries) []*writev2.TimeSeries { 140 | now := time.Now().UnixMilli() 141 | for i := range tss { 142 | tss[i].Samples[0].Timestamp = now 143 | } 144 | return tss 145 | } 146 | 147 | func shuffleTimestampsV2(tss []*writev2.TimeSeries) []*writev2.TimeSeries { 148 | now := time.Now().UnixMilli() 149 | offsets := []int64{0, -60 * 1000, -5 * 60 * 1000} 150 | for i := range tss { 151 | offset := offsets[i%len(offsets)] 152 | tss[i].Samples[0].Timestamp = now + offset 153 | } 154 | return tss 155 | } 156 | 157 | func collectMetricsV2(gatherer prometheus.Gatherer, outOfOrder bool) ([]*writev2.TimeSeries, writev2.SymbolsTable, error) { 158 | metricFamilies, err := gatherer.Gather() 159 | if err != nil { 160 | return nil, writev2.SymbolsTable{}, err 161 | } 162 | tss, st := ToTimeSeriesSliceV2(metricFamilies) 163 | if outOfOrder { 164 | tss = shuffleTimestampsV2(tss) 165 | } 166 | return tss, st, nil 167 | } 168 | 169 | // ToTimeSeriesSliceV2 converts a slice of metricFamilies containing samples into a slice of writev2.TimeSeries. 170 | func ToTimeSeriesSliceV2(metricFamilies []*dto.MetricFamily) ([]*writev2.TimeSeries, writev2.SymbolsTable) { 171 | st := writev2.NewSymbolTable() 172 | timestamp := int64(model.Now()) 173 | tss := make([]*writev2.TimeSeries, 0, len(metricFamilies)*10) 174 | 175 | skippedSamples := 0 176 | for _, metricFamily := range metricFamilies { 177 | for _, metric := range metricFamily.Metric { 178 | labels := prompbLabels(*metricFamily.Name, metric.Label) 179 | labelRefs := make([]uint32, 0, len(labels)) 180 | for _, label := range labels { 181 | labelRefs = append(labelRefs, st.Symbolize(label.Name)) 182 | labelRefs = append(labelRefs, st.Symbolize(label.Value)) 183 | } 184 | ts := &writev2.TimeSeries{ 185 | LabelsRefs: labelRefs, 186 | } 187 | switch *metricFamily.Type { 188 | case dto.MetricType_COUNTER: 189 | ts.Samples = []*writev2.Sample{{ 190 | Value: *metric.Counter.Value, 191 | Timestamp: timestamp, 192 | }} 193 | tss = append(tss, ts) 194 | case dto.MetricType_GAUGE: 195 | ts.Samples = []*writev2.Sample{{ 196 | Value: *metric.Gauge.Value, 197 | Timestamp: timestamp, 198 | }} 199 | tss = append(tss, ts) 200 | default: 201 | skippedSamples++ 202 | } 203 | } 204 | } 205 | if skippedSamples > 0 { 206 | log.Printf("WARN: Skipping %v samples; sending only %v samples, given only gauge and counters are currently implemented\n", skippedSamples, len(tss)) 207 | } 208 | return tss, st 209 | } 210 | -------------------------------------------------------------------------------- /pkg/errors/errors.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package errors 15 | 16 | import ( 17 | "bytes" 18 | "fmt" 19 | "sync" 20 | ) 21 | 22 | // MultiError type implements the error interface, and contains the 23 | // Errors used to construct it. 24 | type MultiError struct { 25 | errors []error 26 | mtx sync.Mutex 27 | } 28 | 29 | // Error returns a concatenated string of the contained errors 30 | func (es *MultiError) Error() string { 31 | es.mtx.Lock() 32 | defer es.mtx.Unlock() 33 | var buf bytes.Buffer 34 | 35 | if len(es.errors) > 1 { 36 | fmt.Fprintf(&buf, "%d errors: ", len(es.errors)) 37 | } 38 | 39 | for i, err := range es.errors { 40 | if i != 0 { 41 | buf.WriteString("; ") 42 | } 43 | buf.WriteString(err.Error()) 44 | } 45 | 46 | return buf.String() 47 | } 48 | 49 | // Add adds the error to the error list if it is not nil. 50 | func (es *MultiError) Add(err error) { 51 | es.mtx.Lock() 52 | defer es.mtx.Unlock() 53 | if err == nil { 54 | return 55 | } 56 | if merr, ok := err.(*MultiError); ok { 57 | es.errors = append(es.errors, merr.errors...) 58 | } else { 59 | es.errors = append(es.errors, err) 60 | } 61 | } 62 | 63 | // Err returns the error list as an error or nil if it is empty. 64 | func (es *MultiError) Err() error { 65 | es.mtx.Lock() 66 | defer es.mtx.Unlock() 67 | if len(es.errors) == 0 { 68 | return nil 69 | } 70 | return es 71 | } 72 | 73 | // Count shows current errors count. 74 | func (es *MultiError) Count() int { 75 | es.mtx.Lock() 76 | defer es.mtx.Unlock() 77 | return len(es.errors) 78 | } 79 | -------------------------------------------------------------------------------- /scripts/errcheck_excludes.txt: -------------------------------------------------------------------------------- 1 | // Don't flag lines such as "io.Copy(ioutil.Discard, resp.Body)". 2 | io.Copy 3 | // The next two are used in HTTP handlers, any error is handled by the server itself. 4 | io.WriteString 5 | (net/http.ResponseWriter).Write 6 | // No need to check for errors on server's shutdown. 7 | (*net/http.Server).Shutdown 8 | 9 | // Never check for logger errors. 10 | (github.com/go-kit/log.Logger).Log 11 | 12 | // Never check for rollback errors as Rollback() is called when a previous error was detected. 13 | (github.com/prometheus/prometheus/storage.Appender).Rollback 14 | --------------------------------------------------------------------------------