├── .circleci └── config.yml ├── .github ├── dependabot.yml └── workflows │ ├── container_description.yml │ └── golangci-lint.yml ├── .gitignore ├── .golangci.yml ├── .promu.yml ├── .yamllint ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── Makefile ├── Makefile.common ├── README.md ├── SECURITY.md ├── VERSION ├── cmd ├── client │ ├── main.go │ └── main_test.go └── proxy │ ├── coordinator.go │ └── main.go ├── docs └── sequence.svg ├── end-to-end-test.sh ├── go.mod ├── go.sum └── util ├── proxy.go └── proxy_test.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2.1 3 | orbs: 4 | prometheus: prometheus/prometheus@0.17.1 5 | executors: 6 | golang: 7 | docker: 8 | - image: cimg/go:1.23 9 | - image: quay.io/prometheus/node-exporter:latest 10 | 11 | jobs: 12 | test: 13 | executor: golang 14 | environment: 15 | prom_ver: 2.54.1 16 | steps: 17 | - prometheus/setup_environment 18 | - run: wget https://github.com/prometheus/prometheus/releases/download/v${prom_ver}/prometheus-${prom_ver}.linux-amd64.tar.gz 19 | - run: tar xzf prometheus-${prom_ver}.linux-amd64.tar.gz 20 | - run: sudo cp -v prometheus-${prom_ver}.linux-amd64/prometheus /bin/prometheus 21 | - run: 22 | name: Configure Prometheus 23 | command: | 24 | cat \<< EOF > prometheus.yml 25 | global: 26 | scrape_interval: 1s 27 | scrape_configs: 28 | - job_name: pushprox 29 | proxy_url: http://127.0.0.1:8080 30 | static_configs: 31 | - targets: ['$(hostname):9100'] 32 | EOF 33 | - run: make 34 | - run: 35 | name: Run everything and test that Prometheus can scrape node_exporter via pushprox 36 | command: ./end-to-end-test.sh 37 | workflows: 38 | version: 2 39 | PushProx: 40 | jobs: 41 | - test: 42 | filters: 43 | tags: 44 | only: /.*/ 45 | - prometheus/build: 46 | name: build 47 | filters: 48 | tags: 49 | only: /.*/ 50 | - prometheus/publish_master: 51 | context: org-context 52 | docker_hub_organization: prometheuscommunity 53 | quay_io_organization: prometheuscommunity 54 | requires: 55 | - test 56 | - build 57 | filters: 58 | branches: 59 | only: master 60 | - prometheus/publish_release: 61 | context: org-context 62 | docker_hub_organization: prometheuscommunity 63 | quay_io_organization: prometheuscommunity 64 | requires: 65 | - test 66 | - build 67 | filters: 68 | tags: 69 | only: /^v.*/ 70 | branches: 71 | ignore: /.*/ 72 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | -------------------------------------------------------------------------------- /.github/workflows/container_description.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Push README to Docker Hub 3 | on: 4 | push: 5 | paths: 6 | - "README.md" 7 | - "README-containers.md" 8 | - ".github/workflows/container_description.yml" 9 | branches: [ main, master ] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | PushDockerHubReadme: 16 | runs-on: ubuntu-latest 17 | name: Push README to Docker Hub 18 | if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. 19 | steps: 20 | - name: git checkout 21 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 22 | - name: Set docker hub repo name 23 | run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV 24 | - name: Push README to Dockerhub 25 | uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1 26 | env: 27 | DOCKER_USER: ${{ secrets.DOCKER_HUB_LOGIN }} 28 | DOCKER_PASS: ${{ secrets.DOCKER_HUB_PASSWORD }} 29 | with: 30 | destination_container_repo: ${{ env.DOCKER_REPO_NAME }} 31 | provider: dockerhub 32 | short_description: ${{ env.DOCKER_REPO_NAME }} 33 | # Empty string results in README-containers.md being pushed if it 34 | # exists. Otherwise, README.md is pushed. 35 | readme_file: '' 36 | 37 | PushQuayIoReadme: 38 | runs-on: ubuntu-latest 39 | name: Push README to quay.io 40 | if: github.repository_owner == 'prometheus' || github.repository_owner == 'prometheus-community' # Don't run this workflow on forks. 41 | steps: 42 | - name: git checkout 43 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 44 | - name: Set quay.io org name 45 | run: echo "DOCKER_REPO=$(echo quay.io/${GITHUB_REPOSITORY_OWNER} | tr -d '-')" >> $GITHUB_ENV 46 | - name: Set quay.io repo name 47 | run: echo "DOCKER_REPO_NAME=$(make docker-repo-name)" >> $GITHUB_ENV 48 | - name: Push README to quay.io 49 | uses: christian-korneck/update-container-description-action@d36005551adeaba9698d8d67a296bd16fa91f8e8 # v1 50 | env: 51 | DOCKER_APIKEY: ${{ secrets.QUAY_IO_API_TOKEN }} 52 | with: 53 | destination_container_repo: ${{ env.DOCKER_REPO_NAME }} 54 | provider: quay 55 | # Empty string results in README-containers.md being pushed if it 56 | # exists. Otherwise, README.md is pushed. 57 | readme_file: '' 58 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This action is synced from https://github.com/prometheus/prometheus 3 | name: golangci-lint 4 | on: 5 | push: 6 | paths: 7 | - "go.sum" 8 | - "go.mod" 9 | - "**.go" 10 | - "scripts/errcheck_excludes.txt" 11 | - ".github/workflows/golangci-lint.yml" 12 | - ".golangci.yml" 13 | pull_request: 14 | 15 | permissions: # added using https://github.com/step-security/secure-repo 16 | contents: read 17 | 18 | jobs: 19 | golangci: 20 | permissions: 21 | contents: read # for actions/checkout to fetch code 22 | pull-requests: read # for golangci/golangci-lint-action to fetch pull requests 23 | name: lint 24 | runs-on: ubuntu-latest 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 28 | - name: Install Go 29 | uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 30 | with: 31 | go-version: 1.23.x 32 | - name: Install snmp_exporter/generator dependencies 33 | run: sudo apt-get update && sudo apt-get -y install libsnmp-dev 34 | if: github.repository == 'prometheus/snmp_exporter' 35 | - name: Lint 36 | uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 37 | with: 38 | args: --verbose 39 | version: v1.61.0 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.build/ 2 | /.release/ 3 | /.tarballs/ 4 | /node_exporter* 5 | /prometheus* 6 | /data/ 7 | /pushprox-client 8 | /pushprox-proxy 9 | 10 | /vendor 11 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | linters: 3 | enable: 4 | - misspell 5 | - revive 6 | - sloglint 7 | 8 | linters-settings: 9 | errcheck: 10 | exclude-functions: 11 | # Don't flag lines such as "io.Copy(io.Discard, resp.Body)". 12 | - io.Copy 13 | # Used in HTTP handlers, any error is handled by the server itself. 14 | - (net/http.ResponseWriter).Write 15 | revive: 16 | rules: 17 | # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter 18 | - name: unused-parameter 19 | severity: warning 20 | disabled: true 21 | -------------------------------------------------------------------------------- /.promu.yml: -------------------------------------------------------------------------------- 1 | go: 2 | # This must match .circle/config.yml. 3 | version: 1.23 4 | repository: 5 | path: github.com/prometheus-community/pushprox 6 | build: 7 | binaries: 8 | - name: pushprox-client 9 | path: ./cmd/client 10 | - name: pushprox-proxy 11 | path: ./cmd/proxy 12 | tarball: 13 | files: 14 | - LICENSE 15 | -------------------------------------------------------------------------------- /.yamllint: -------------------------------------------------------------------------------- 1 | --- 2 | extends: default 3 | ignore: | 4 | **/node_modules 5 | 6 | rules: 7 | braces: 8 | max-spaces-inside: 1 9 | level: error 10 | brackets: 11 | max-spaces-inside: 1 12 | level: error 13 | commas: disable 14 | comments: disable 15 | comments-indentation: disable 16 | document-start: disable 17 | indentation: 18 | spaces: consistent 19 | indent-sequences: consistent 20 | key-duplicates: 21 | ignore: | 22 | config/testdata/section_key_dup.bad.yml 23 | line-length: disable 24 | truthy: 25 | check-keys: false 26 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## master / unreleased 2 | 3 | ## 0.2.0 / 2024-03-28 4 | 5 | * [FEATURE] Implement flags to control retry delays #83 6 | * [ENHANCEMENT] Add scrape_id to error log #120 7 | * [BUGFIX] /clients endpoint return application/json as Content-Type #121 8 | 9 | ## 0.1.0 / 2019-07-29 10 | 11 | * [CHANGE] Initial release 12 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Prometheus Community Code of Conduct 2 | 3 | Prometheus follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md). 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Requires `promu crossbuild` artifacts. 2 | ARG ARCH="amd64" 3 | ARG OS="linux" 4 | FROM quay.io/prometheus/busybox-${OS}-${ARCH}:glibc 5 | 6 | ARG ARCH="amd64" 7 | ARG OS="linux" 8 | COPY .build/${OS}-${ARCH}/pushprox-proxy /app/pushprox-proxy 9 | COPY .build/${OS}-${ARCH}/pushprox-client /app/pushprox-client 10 | 11 | # The default startup is the proxy. 12 | # This can be overridden with the docker --entrypoint flag or the command 13 | # field in Kubernetes container v1 API. 14 | USER nobody 15 | ENTRYPOINT ["/app/pushprox-proxy"] 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2020 The Prometheus Authors 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | # Needs to be defined before including Makefile.common to auto-generate targets 15 | DOCKER_ARCHS ?= amd64 armv7 arm64 ppc64le s390x 16 | DOCKER_REPO ?= prometheuscommunity 17 | 18 | include Makefile.common 19 | 20 | DOCKER_IMAGE_NAME ?= pushprox 21 | -------------------------------------------------------------------------------- /Makefile.common: -------------------------------------------------------------------------------- 1 | # Copyright 2018 The Prometheus Authors 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | 15 | # A common Makefile that includes rules to be reused in different prometheus projects. 16 | # !!! Open PRs only against the prometheus/prometheus/Makefile.common repository! 17 | 18 | # Example usage : 19 | # Create the main Makefile in the root project directory. 20 | # include Makefile.common 21 | # customTarget: 22 | # @echo ">> Running customTarget" 23 | # 24 | 25 | # Ensure GOBIN is not set during build so that promu is installed to the correct path 26 | unexport GOBIN 27 | 28 | GO ?= go 29 | GOFMT ?= $(GO)fmt 30 | FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH))) 31 | GOOPTS ?= 32 | GOHOSTOS ?= $(shell $(GO) env GOHOSTOS) 33 | GOHOSTARCH ?= $(shell $(GO) env GOHOSTARCH) 34 | 35 | GO_VERSION ?= $(shell $(GO) version) 36 | GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) 37 | PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') 38 | 39 | PROMU := $(FIRST_GOPATH)/bin/promu 40 | pkgs = ./... 41 | 42 | ifeq (arm, $(GOHOSTARCH)) 43 | GOHOSTARM ?= $(shell GOARM= $(GO) env GOARM) 44 | GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH)v$(GOHOSTARM) 45 | else 46 | GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) 47 | endif 48 | 49 | GOTEST := $(GO) test 50 | GOTEST_DIR := 51 | ifneq ($(CIRCLE_JOB),) 52 | ifneq ($(shell command -v gotestsum 2> /dev/null),) 53 | GOTEST_DIR := test-results 54 | GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml -- 55 | endif 56 | endif 57 | 58 | PROMU_VERSION ?= 0.17.0 59 | PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz 60 | 61 | SKIP_GOLANGCI_LINT := 62 | GOLANGCI_LINT := 63 | GOLANGCI_LINT_OPTS ?= 64 | GOLANGCI_LINT_VERSION ?= v1.61.0 65 | # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. 66 | # windows isn't included here because of the path separator being different. 67 | ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) 68 | ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386 arm64)) 69 | # If we're in CI and there is an Actions file, that means the linter 70 | # is being run in Actions, so we don't need to run it here. 71 | ifneq (,$(SKIP_GOLANGCI_LINT)) 72 | GOLANGCI_LINT := 73 | else ifeq (,$(CIRCLE_JOB)) 74 | GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint 75 | else ifeq (,$(wildcard .github/workflows/golangci-lint.yml)) 76 | GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint 77 | endif 78 | endif 79 | endif 80 | 81 | PREFIX ?= $(shell pwd) 82 | BIN_DIR ?= $(shell pwd) 83 | DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) 84 | DOCKERFILE_PATH ?= ./Dockerfile 85 | DOCKERBUILD_CONTEXT ?= ./ 86 | DOCKER_REPO ?= prom 87 | 88 | DOCKER_ARCHS ?= amd64 89 | 90 | BUILD_DOCKER_ARCHS = $(addprefix common-docker-,$(DOCKER_ARCHS)) 91 | PUBLISH_DOCKER_ARCHS = $(addprefix common-docker-publish-,$(DOCKER_ARCHS)) 92 | TAG_DOCKER_ARCHS = $(addprefix common-docker-tag-latest-,$(DOCKER_ARCHS)) 93 | 94 | SANITIZED_DOCKER_IMAGE_TAG := $(subst +,-,$(DOCKER_IMAGE_TAG)) 95 | 96 | ifeq ($(GOHOSTARCH),amd64) 97 | ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux freebsd darwin windows)) 98 | # Only supported on amd64 99 | test-flags := -race 100 | endif 101 | endif 102 | 103 | # This rule is used to forward a target like "build" to "common-build". This 104 | # allows a new "build" target to be defined in a Makefile which includes this 105 | # one and override "common-build" without override warnings. 106 | %: common-% ; 107 | 108 | .PHONY: common-all 109 | common-all: precheck style check_license lint yamllint unused build test 110 | 111 | .PHONY: common-style 112 | common-style: 113 | @echo ">> checking code style" 114 | @fmtRes=$$($(GOFMT) -d $$(find . -path ./vendor -prune -o -name '*.go' -print)); \ 115 | if [ -n "$${fmtRes}" ]; then \ 116 | echo "gofmt checking failed!"; echo "$${fmtRes}"; echo; \ 117 | echo "Please ensure you are using $$($(GO) version) for formatting code."; \ 118 | exit 1; \ 119 | fi 120 | 121 | .PHONY: common-check_license 122 | common-check_license: 123 | @echo ">> checking license header" 124 | @licRes=$$(for file in $$(find . -type f -iname '*.go' ! -path './vendor/*') ; do \ 125 | awk 'NR<=3' $$file | grep -Eq "(Copyright|generated|GENERATED)" || echo $$file; \ 126 | done); \ 127 | if [ -n "$${licRes}" ]; then \ 128 | echo "license header checking failed:"; echo "$${licRes}"; \ 129 | exit 1; \ 130 | fi 131 | 132 | .PHONY: common-deps 133 | common-deps: 134 | @echo ">> getting dependencies" 135 | $(GO) mod download 136 | 137 | .PHONY: update-go-deps 138 | update-go-deps: 139 | @echo ">> updating Go dependencies" 140 | @for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ 141 | $(GO) get -d $$m; \ 142 | done 143 | $(GO) mod tidy 144 | 145 | .PHONY: common-test-short 146 | common-test-short: $(GOTEST_DIR) 147 | @echo ">> running short tests" 148 | $(GOTEST) -short $(GOOPTS) $(pkgs) 149 | 150 | .PHONY: common-test 151 | common-test: $(GOTEST_DIR) 152 | @echo ">> running all tests" 153 | $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs) 154 | 155 | $(GOTEST_DIR): 156 | @mkdir -p $@ 157 | 158 | .PHONY: common-format 159 | common-format: 160 | @echo ">> formatting code" 161 | $(GO) fmt $(pkgs) 162 | 163 | .PHONY: common-vet 164 | common-vet: 165 | @echo ">> vetting code" 166 | $(GO) vet $(GOOPTS) $(pkgs) 167 | 168 | .PHONY: common-lint 169 | common-lint: $(GOLANGCI_LINT) 170 | ifdef GOLANGCI_LINT 171 | @echo ">> running golangci-lint" 172 | $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs) 173 | endif 174 | 175 | .PHONY: common-lint-fix 176 | common-lint-fix: $(GOLANGCI_LINT) 177 | ifdef GOLANGCI_LINT 178 | @echo ">> running golangci-lint fix" 179 | $(GOLANGCI_LINT) run --fix $(GOLANGCI_LINT_OPTS) $(pkgs) 180 | endif 181 | 182 | .PHONY: common-yamllint 183 | common-yamllint: 184 | @echo ">> running yamllint on all YAML files in the repository" 185 | ifeq (, $(shell command -v yamllint 2> /dev/null)) 186 | @echo "yamllint not installed so skipping" 187 | else 188 | yamllint . 189 | endif 190 | 191 | # For backward-compatibility. 192 | .PHONY: common-staticcheck 193 | common-staticcheck: lint 194 | 195 | .PHONY: common-unused 196 | common-unused: 197 | @echo ">> running check for unused/missing packages in go.mod" 198 | $(GO) mod tidy 199 | @git diff --exit-code -- go.sum go.mod 200 | 201 | .PHONY: common-build 202 | common-build: promu 203 | @echo ">> building binaries" 204 | $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) 205 | 206 | .PHONY: common-tarball 207 | common-tarball: promu 208 | @echo ">> building release tarball" 209 | $(PROMU) tarball --prefix $(PREFIX) $(BIN_DIR) 210 | 211 | .PHONY: common-docker-repo-name 212 | common-docker-repo-name: 213 | @echo "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)" 214 | 215 | .PHONY: common-docker $(BUILD_DOCKER_ARCHS) 216 | common-docker: $(BUILD_DOCKER_ARCHS) 217 | $(BUILD_DOCKER_ARCHS): common-docker-%: 218 | docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" \ 219 | -f $(DOCKERFILE_PATH) \ 220 | --build-arg ARCH="$*" \ 221 | --build-arg OS="linux" \ 222 | $(DOCKERBUILD_CONTEXT) 223 | 224 | .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) 225 | common-docker-publish: $(PUBLISH_DOCKER_ARCHS) 226 | $(PUBLISH_DOCKER_ARCHS): common-docker-publish-%: 227 | docker push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" 228 | 229 | DOCKER_MAJOR_VERSION_TAG = $(firstword $(subst ., ,$(shell cat VERSION))) 230 | .PHONY: common-docker-tag-latest $(TAG_DOCKER_ARCHS) 231 | common-docker-tag-latest: $(TAG_DOCKER_ARCHS) 232 | $(TAG_DOCKER_ARCHS): common-docker-tag-latest-%: 233 | docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:latest" 234 | docker tag "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(SANITIZED_DOCKER_IMAGE_TAG)" "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:v$(DOCKER_MAJOR_VERSION_TAG)" 235 | 236 | .PHONY: common-docker-manifest 237 | common-docker-manifest: 238 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create -a "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" $(foreach ARCH,$(DOCKER_ARCHS),$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$(ARCH):$(SANITIZED_DOCKER_IMAGE_TAG)) 239 | DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME):$(SANITIZED_DOCKER_IMAGE_TAG)" 240 | 241 | .PHONY: promu 242 | promu: $(PROMU) 243 | 244 | $(PROMU): 245 | $(eval PROMU_TMP := $(shell mktemp -d)) 246 | curl -s -L $(PROMU_URL) | tar -xvzf - -C $(PROMU_TMP) 247 | mkdir -p $(FIRST_GOPATH)/bin 248 | cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu 249 | rm -r $(PROMU_TMP) 250 | 251 | .PHONY: proto 252 | proto: 253 | @echo ">> generating code from proto files" 254 | @./scripts/genproto.sh 255 | 256 | ifdef GOLANGCI_LINT 257 | $(GOLANGCI_LINT): 258 | mkdir -p $(FIRST_GOPATH)/bin 259 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/$(GOLANGCI_LINT_VERSION)/install.sh \ 260 | | sed -e '/install -d/d' \ 261 | | sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION) 262 | endif 263 | 264 | .PHONY: precheck 265 | precheck:: 266 | 267 | define PRECHECK_COMMAND_template = 268 | precheck:: $(1)_precheck 269 | 270 | PRECHECK_COMMAND_$(1) ?= $(1) $$(strip $$(PRECHECK_OPTIONS_$(1))) 271 | .PHONY: $(1)_precheck 272 | $(1)_precheck: 273 | @if ! $$(PRECHECK_COMMAND_$(1)) 1>/dev/null 2>&1; then \ 274 | echo "Execution of '$$(PRECHECK_COMMAND_$(1))' command failed. Is $(1) installed?"; \ 275 | exit 1; \ 276 | fi 277 | endef 278 | 279 | govulncheck: install-govulncheck 280 | govulncheck ./... 281 | 282 | install-govulncheck: 283 | command -v govulncheck > /dev/null || go install golang.org/x/vuln/cmd/govulncheck@latest 284 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PushProx [![CircleCI](https://circleci.com/gh/prometheus-community/PushProx.svg?style=shield)](https://circleci.com/gh/prometheus-community/PushProx) 2 | 3 | PushProx is a client and proxy that allows transversing of NAT and other 4 | similar network topologies by Prometheus, while still following the pull model. 5 | 6 | While this is reasonably robust in practice, this is a work in progress. 7 | 8 | ## Running 9 | 10 | First build the proxy and client: 11 | 12 | ``` 13 | git clone https://github.com/prometheus-community/pushprox.git 14 | cd pushprox 15 | make build 16 | ``` 17 | 18 | Run the proxy somewhere both Prometheus and the clients can get to: 19 | 20 | ``` 21 | ./pushprox-proxy 22 | ``` 23 | 24 | On every target machine run the client, pointing it at the proxy: 25 | ``` 26 | ./pushprox-client --proxy-url=http://proxy:8080/ 27 | ``` 28 | 29 | In Prometheus, use the proxy as a `proxy_url`: 30 | 31 | ``` 32 | scrape_configs: 33 | - job_name: node 34 | proxy_url: http://proxy:8080/ 35 | static_configs: 36 | - targets: ['client:9100'] # Presuming the FQDN of the client is "client". 37 | ``` 38 | 39 | If the target must be scraped over SSL/TLS, add: 40 | ``` 41 | params: 42 | _scheme: [https] 43 | ``` 44 | rather than the usual `scheme: https`. Only the default `scheme: http` works with the proxy, 45 | so this workaround is required. 46 | 47 | ## Service Discovery 48 | 49 | The `/clients` endpoint will return a list of all registered clients in the format 50 | used by `file_sd_configs`. You could use wget in a cronjob to put it somewhere 51 | file\_sd\_configs can read and then then relabel as needed. 52 | 53 | ## How It Works 54 | 55 | ![Sequence diagram](./docs/sequence.svg) 56 | 57 | Clients perform scrapes in a network environment that's not directly accessible by Prometheus. 58 | The Proxy is accessible by both the Clients and Prometheus. 59 | Each client is identified by its fqdn. 60 | 61 | For example, the following sequence is performed when Prometheus scrapes target `fqdn-x` via PushProx. 62 | First, a Client polls the Proxy for scrape requests, and includes its fqdn in the poll (1). 63 | The Proxy does not respond yet. 64 | Next, Prometheus tries to scrape the target with hostname `fqdn-x` via the Proxy (2). 65 | Using the fqdn received in (1), the Proxy now routes the scrape to the correct Client: the scrape request is in the response body of the poll (3). 66 | This scrape request is executed by the client (4), the response containing metrics (5) is posted to the Proxy (6). 67 | On its turn, the Proxy returns this to Prometheus (7) as a reponse to the initial scrape of (2). 68 | 69 | PushProx passes all HTTP headers transparently, features like compression and accept encoding are up to the scraping Prometheus server. 70 | 71 | ## Security 72 | 73 | There is no authentication or authorisation included, a reverse proxy can be 74 | put in front though to add these. 75 | 76 | Running the client allows those with access to the proxy or the client to access 77 | all network services on the machine hosting the client. 78 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting a security issue 2 | 3 | The Prometheus security policy, including how to report vulnerabilities, can be 4 | found here: 5 | 6 | 7 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.2.0 2 | -------------------------------------------------------------------------------- /cmd/client/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "bufio" 18 | "bytes" 19 | "context" 20 | "crypto/tls" 21 | "crypto/x509" 22 | "errors" 23 | "fmt" 24 | "io" 25 | "log/slog" 26 | "net" 27 | "net/http" 28 | "net/url" 29 | "os" 30 | "strings" 31 | "time" 32 | 33 | "github.com/Showmax/go-fqdn" 34 | "github.com/alecthomas/kingpin/v2" 35 | "github.com/cenkalti/backoff/v4" 36 | "github.com/prometheus-community/pushprox/util" 37 | "github.com/prometheus/client_golang/prometheus" 38 | "github.com/prometheus/client_golang/prometheus/promhttp" 39 | "github.com/prometheus/common/promslog" 40 | "github.com/prometheus/common/promslog/flag" 41 | ) 42 | 43 | var ( 44 | myFqdn = kingpin.Flag("fqdn", "FQDN to register with").Default(fqdn.Get()).String() 45 | proxyURL = kingpin.Flag("proxy-url", "Push proxy to talk to.").Required().String() 46 | caCertFile = kingpin.Flag("tls.cacert", " CA certificate to verify peer against").String() 47 | tlsCert = kingpin.Flag("tls.cert", " Client certificate file").String() 48 | tlsKey = kingpin.Flag("tls.key", " Private key file").String() 49 | metricsAddr = kingpin.Flag("metrics-addr", "Serve Prometheus metrics at this address").Default(":9369").String() 50 | 51 | retryInitialWait = kingpin.Flag("proxy.retry.initial-wait", "Amount of time to wait after proxy failure").Default("1s").Duration() 52 | retryMaxWait = kingpin.Flag("proxy.retry.max-wait", "Maximum amount of time to wait between proxy poll retries").Default("5s").Duration() 53 | ) 54 | 55 | var ( 56 | scrapeErrorCounter = prometheus.NewCounter( 57 | prometheus.CounterOpts{ 58 | Name: "pushprox_client_scrape_errors_total", 59 | Help: "Number of scrape errors", 60 | }, 61 | ) 62 | pushErrorCounter = prometheus.NewCounter( 63 | prometheus.CounterOpts{ 64 | Name: "pushprox_client_push_errors_total", 65 | Help: "Number of push errors", 66 | }, 67 | ) 68 | pollErrorCounter = prometheus.NewCounter( 69 | prometheus.CounterOpts{ 70 | Name: "pushprox_client_poll_errors_total", 71 | Help: "Number of poll errors", 72 | }, 73 | ) 74 | ) 75 | 76 | func init() { 77 | prometheus.MustRegister(pushErrorCounter, pollErrorCounter, scrapeErrorCounter) 78 | } 79 | 80 | func newBackOffFromFlags() backoff.BackOff { 81 | b := backoff.NewExponentialBackOff() 82 | b.InitialInterval = *retryInitialWait 83 | b.Multiplier = 1.5 84 | b.MaxInterval = *retryMaxWait 85 | b.MaxElapsedTime = time.Duration(0) 86 | return b 87 | } 88 | 89 | // Coordinator for scrape requests and responses 90 | type Coordinator struct { 91 | logger *slog.Logger 92 | } 93 | 94 | func (c *Coordinator) handleErr(request *http.Request, client *http.Client, err error) { 95 | c.logger.Error("Coordinator error", "error", err) 96 | scrapeErrorCounter.Inc() 97 | resp := &http.Response{ 98 | StatusCode: http.StatusInternalServerError, 99 | Body: io.NopCloser(strings.NewReader(err.Error())), 100 | Header: http.Header{}, 101 | } 102 | if err = c.doPush(resp, request, client); err != nil { 103 | pushErrorCounter.Inc() 104 | c.logger.Warn("Failed to push failed scrape response:", "err", err) 105 | return 106 | } 107 | c.logger.Info("Pushed failed scrape response") 108 | } 109 | 110 | func (c *Coordinator) doScrape(request *http.Request, client *http.Client) { 111 | logger := c.logger.With("scrape_id", request.Header.Get("id")) 112 | timeout, err := util.GetHeaderTimeout(request.Header) 113 | if err != nil { 114 | c.handleErr(request, client, err) 115 | return 116 | } 117 | ctx, cancel := context.WithTimeout(request.Context(), timeout) 118 | defer cancel() 119 | request = request.WithContext(ctx) 120 | // We cannot handle https requests at the proxy, as we would only 121 | // see a CONNECT, so use a URL parameter to trigger it. 122 | params := request.URL.Query() 123 | if params.Get("_scheme") == "https" { 124 | request.URL.Scheme = "https" 125 | params.Del("_scheme") 126 | request.URL.RawQuery = params.Encode() 127 | } 128 | 129 | if request.URL.Hostname() != *myFqdn { 130 | c.handleErr(request, client, errors.New("scrape target doesn't match client fqdn")) 131 | return 132 | } 133 | 134 | scrapeResp, err := client.Do(request) 135 | if err != nil { 136 | c.handleErr(request, client, fmt.Errorf("failed to scrape %s: %w", request.URL.String(), err)) 137 | return 138 | } 139 | logger.Info("Retrieved scrape response") 140 | if err = c.doPush(scrapeResp, request, client); err != nil { 141 | pushErrorCounter.Inc() 142 | logger.Warn("Failed to push scrape response:", "err", err) 143 | return 144 | } 145 | logger.Info("Pushed scrape result") 146 | } 147 | 148 | // Report the result of the scrape back up to the proxy. 149 | func (c *Coordinator) doPush(resp *http.Response, origRequest *http.Request, client *http.Client) error { 150 | resp.Header.Set("id", origRequest.Header.Get("id")) // Link the request and response 151 | // Remaining scrape deadline. 152 | deadline, _ := origRequest.Context().Deadline() 153 | resp.Header.Set("X-Prometheus-Scrape-Timeout", fmt.Sprintf("%f", float64(time.Until(deadline))/1e9)) 154 | 155 | base, err := url.Parse(*proxyURL) 156 | if err != nil { 157 | return err 158 | } 159 | u, err := url.Parse("push") 160 | if err != nil { 161 | return err 162 | } 163 | url := base.ResolveReference(u) 164 | 165 | buf := &bytes.Buffer{} 166 | //nolint:errcheck // https://github.com/prometheus-community/PushProx/issues/111 167 | resp.Write(buf) 168 | request := &http.Request{ 169 | Method: "POST", 170 | URL: url, 171 | Body: io.NopCloser(buf), 172 | ContentLength: int64(buf.Len()), 173 | } 174 | request = request.WithContext(origRequest.Context()) 175 | if _, err = client.Do(request); err != nil { 176 | return err 177 | } 178 | return nil 179 | } 180 | 181 | func (c *Coordinator) doPoll(client *http.Client) error { 182 | base, err := url.Parse(*proxyURL) 183 | if err != nil { 184 | c.logger.Error("Error parsing url:", "err", err) 185 | return fmt.Errorf("error parsing url: %w", err) 186 | } 187 | u, err := url.Parse("poll") 188 | if err != nil { 189 | c.logger.Error("Error parsing url:", "err", err) 190 | return fmt.Errorf("error parsing url poll: %w", err) 191 | } 192 | url := base.ResolveReference(u) 193 | resp, err := client.Post(url.String(), "", strings.NewReader(*myFqdn)) 194 | if err != nil { 195 | c.logger.Error("Error polling:", "err", err) 196 | return fmt.Errorf("error polling: %w", err) 197 | } 198 | defer resp.Body.Close() 199 | 200 | request, err := http.ReadRequest(bufio.NewReader(resp.Body)) 201 | if err != nil { 202 | c.logger.Error("Error reading request:", "err", err) 203 | return fmt.Errorf("error reading request: %w", err) 204 | } 205 | c.logger.Info("Got scrape request", "scrape_id", request.Header.Get("id"), "url", request.URL) 206 | 207 | request.RequestURI = "" 208 | 209 | go c.doScrape(request, client) 210 | 211 | return nil 212 | } 213 | 214 | func (c *Coordinator) loop(bo backoff.BackOff, client *http.Client) { 215 | op := func() error { 216 | return c.doPoll(client) 217 | } 218 | 219 | for { 220 | if err := backoff.RetryNotify(op, bo, func(err error, _ time.Duration) { 221 | pollErrorCounter.Inc() 222 | }); err != nil { 223 | c.logger.Error("backoff returned error", "error", err) 224 | } 225 | } 226 | } 227 | 228 | func main() { 229 | promslogConfig := promslog.Config{} 230 | flag.AddFlags(kingpin.CommandLine, &promslogConfig) 231 | kingpin.HelpFlag.Short('h') 232 | kingpin.Parse() 233 | logger := promslog.New(&promslogConfig) 234 | coordinator := Coordinator{logger: logger} 235 | 236 | if *proxyURL == "" { 237 | coordinator.logger.Error("--proxy-url flag must be specified.") 238 | os.Exit(1) 239 | } 240 | // Make sure proxyURL ends with a single '/' 241 | *proxyURL = strings.TrimRight(*proxyURL, "/") + "/" 242 | coordinator.logger.Info("URL and FQDN info", "proxy_url", *proxyURL, "fqdn", *myFqdn) 243 | 244 | tlsConfig := &tls.Config{} 245 | if *tlsCert != "" { 246 | cert, err := tls.LoadX509KeyPair(*tlsCert, *tlsKey) 247 | if err != nil { 248 | coordinator.logger.Error("Certificate or Key is invalid", "err", err) 249 | os.Exit(1) 250 | } 251 | 252 | // Setup HTTPS client 253 | tlsConfig.Certificates = []tls.Certificate{cert} 254 | } 255 | 256 | if *caCertFile != "" { 257 | caCert, err := os.ReadFile(*caCertFile) 258 | if err != nil { 259 | coordinator.logger.Error("Not able to read cacert file", "err", err) 260 | os.Exit(1) 261 | } 262 | caCertPool := x509.NewCertPool() 263 | if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { 264 | coordinator.logger.Error("Failed to use cacert file as ca certificate") 265 | os.Exit(1) 266 | } 267 | 268 | tlsConfig.RootCAs = caCertPool 269 | } 270 | 271 | if *metricsAddr != "" { 272 | go func() { 273 | if err := http.ListenAndServe(*metricsAddr, promhttp.Handler()); err != nil { 274 | coordinator.logger.Warn("ListenAndServe", "err", err) 275 | } 276 | }() 277 | } 278 | 279 | transport := &http.Transport{ 280 | Proxy: http.ProxyFromEnvironment, 281 | DialContext: (&net.Dialer{ 282 | Timeout: 30 * time.Second, 283 | KeepAlive: 30 * time.Second, 284 | DualStack: true, 285 | }).DialContext, 286 | MaxIdleConns: 100, 287 | IdleConnTimeout: 90 * time.Second, 288 | TLSHandshakeTimeout: 10 * time.Second, 289 | ExpectContinueTimeout: 1 * time.Second, 290 | TLSClientConfig: tlsConfig, 291 | } 292 | 293 | client := &http.Client{Transport: transport} 294 | 295 | coordinator.loop(newBackOffFromFlags(), client) 296 | } 297 | -------------------------------------------------------------------------------- /cmd/client/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "errors" 18 | "fmt" 19 | "net/http" 20 | "net/http/httptest" 21 | "testing" 22 | 23 | "github.com/prometheus/common/promslog" 24 | ) 25 | 26 | func prepareTest() (*httptest.Server, Coordinator) { 27 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 28 | w.WriteHeader(http.StatusOK) 29 | fmt.Fprintln(w, "GET /index.html HTTP/1.0\n\nOK") 30 | })) 31 | c := Coordinator{logger: promslog.NewNopLogger()} 32 | *proxyURL = ts.URL 33 | return ts, c 34 | } 35 | 36 | func TestDoScrape(t *testing.T) { 37 | ts, c := prepareTest() 38 | defer ts.Close() 39 | 40 | req, err := http.NewRequest("GET", ts.URL, nil) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | req.Header.Add("X-Prometheus-Scrape-Timeout-Seconds", "10.0") 45 | *myFqdn = ts.URL 46 | c.doScrape(req, ts.Client()) 47 | } 48 | 49 | func TestHandleErr(t *testing.T) { 50 | ts, c := prepareTest() 51 | defer ts.Close() 52 | 53 | req, err := http.NewRequest("GET", ts.URL, nil) 54 | if err != nil { 55 | t.Fatal(err) 56 | } 57 | c.handleErr(req, ts.Client(), errors.New("test error")) 58 | } 59 | 60 | func TestLoop(t *testing.T) { 61 | ts, c := prepareTest() 62 | defer ts.Close() 63 | if err := c.doPoll(ts.Client()); err != nil { 64 | t.Fatal(err) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /cmd/proxy/coordinator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "context" 18 | "fmt" 19 | "log/slog" 20 | "net/http" 21 | "sync" 22 | "time" 23 | 24 | "github.com/alecthomas/kingpin/v2" 25 | "github.com/google/uuid" 26 | "github.com/prometheus-community/pushprox/util" 27 | "github.com/prometheus/client_golang/prometheus" 28 | "github.com/prometheus/client_golang/prometheus/promauto" 29 | ) 30 | 31 | var ( 32 | registrationTimeout = kingpin.Flag("registration.timeout", "After how long a registration expires.").Default("5m").Duration() 33 | ) 34 | 35 | // Coordinator metrics. 36 | var ( 37 | knownClients = promauto.NewGauge( 38 | prometheus.GaugeOpts{ 39 | Namespace: namespace, 40 | Name: "clients", 41 | Help: "Number of known pushprox clients.", 42 | }, 43 | ) 44 | ) 45 | 46 | // Coordinator for scrape requests and responses 47 | type Coordinator struct { 48 | mu sync.Mutex 49 | 50 | // Clients waiting for a scrape. 51 | waiting map[string]chan *http.Request 52 | // Responses from clients. 53 | responses map[string]chan *http.Response 54 | // Clients we know about and when they last contacted us. 55 | known map[string]time.Time 56 | 57 | logger *slog.Logger 58 | } 59 | 60 | // NewCoordinator initiates the coordinator and starts the client cleanup routine 61 | func NewCoordinator(logger *slog.Logger) (*Coordinator, error) { 62 | c := &Coordinator{ 63 | waiting: map[string]chan *http.Request{}, 64 | responses: map[string]chan *http.Response{}, 65 | known: map[string]time.Time{}, 66 | logger: logger, 67 | } 68 | 69 | go c.gc() 70 | return c, nil 71 | } 72 | 73 | // Generate a unique ID 74 | func (c *Coordinator) genID() (string, error) { 75 | id, err := uuid.NewRandom() 76 | return id.String(), err 77 | } 78 | 79 | func (c *Coordinator) getRequestChannel(fqdn string) chan *http.Request { 80 | c.mu.Lock() 81 | defer c.mu.Unlock() 82 | ch, ok := c.waiting[fqdn] 83 | if !ok { 84 | ch = make(chan *http.Request) 85 | c.waiting[fqdn] = ch 86 | } 87 | return ch 88 | } 89 | 90 | func (c *Coordinator) getResponseChannel(id string) chan *http.Response { 91 | c.mu.Lock() 92 | defer c.mu.Unlock() 93 | ch, ok := c.responses[id] 94 | if !ok { 95 | ch = make(chan *http.Response) 96 | c.responses[id] = ch 97 | } 98 | return ch 99 | } 100 | 101 | // Remove a response channel. Idempotent. 102 | func (c *Coordinator) removeResponseChannel(id string) { 103 | c.mu.Lock() 104 | defer c.mu.Unlock() 105 | delete(c.responses, id) 106 | } 107 | 108 | // DoScrape requests a scrape. 109 | func (c *Coordinator) DoScrape(ctx context.Context, r *http.Request) (*http.Response, error) { 110 | id, err := c.genID() 111 | if err != nil { 112 | return nil, err 113 | } 114 | c.logger.Info("DoScrape", "scrape_id", id, "url", r.URL.String()) 115 | r.Header.Add("Id", id) 116 | select { 117 | case <-ctx.Done(): 118 | return nil, fmt.Errorf("Timeout reached for %q: %s", r.URL.String(), ctx.Err()) 119 | case c.getRequestChannel(r.URL.Hostname()) <- r: 120 | } 121 | 122 | respCh := c.getResponseChannel(id) 123 | defer c.removeResponseChannel(id) 124 | 125 | select { 126 | case <-ctx.Done(): 127 | return nil, ctx.Err() 128 | case resp := <-respCh: 129 | return resp, nil 130 | } 131 | } 132 | 133 | // WaitForScrapeInstruction registers a client waiting for a scrape result 134 | func (c *Coordinator) WaitForScrapeInstruction(fqdn string) (*http.Request, error) { 135 | c.logger.Info("WaitForScrapeInstruction", "fqdn", fqdn) 136 | 137 | c.addKnownClient(fqdn) 138 | // TODO: What if the client times out? 139 | ch := c.getRequestChannel(fqdn) 140 | 141 | // exhaust existing poll request (eg. timeouted queues) 142 | select { 143 | case ch <- nil: 144 | // 145 | default: 146 | break 147 | } 148 | 149 | for { 150 | request := <-ch 151 | if request == nil { 152 | return nil, fmt.Errorf("request is expired") 153 | } 154 | 155 | select { 156 | case <-request.Context().Done(): 157 | // Request has timed out, get another one. 158 | default: 159 | return request, nil 160 | } 161 | } 162 | } 163 | 164 | // ScrapeResult send by client 165 | func (c *Coordinator) ScrapeResult(r *http.Response) error { 166 | id := r.Header.Get("Id") 167 | c.logger.Info("ScrapeResult", "scrape_id", id) 168 | ctx, cancel := context.WithTimeout(context.Background(), util.GetScrapeTimeout(maxScrapeTimeout, defaultScrapeTimeout, r.Header)) 169 | defer cancel() 170 | // Don't expose internal headers. 171 | r.Header.Del("Id") 172 | r.Header.Del("X-Prometheus-Scrape-Timeout-Seconds") 173 | select { 174 | case c.getResponseChannel(id) <- r: 175 | return nil 176 | case <-ctx.Done(): 177 | c.removeResponseChannel(id) 178 | return ctx.Err() 179 | } 180 | } 181 | 182 | func (c *Coordinator) addKnownClient(fqdn string) { 183 | c.mu.Lock() 184 | defer c.mu.Unlock() 185 | 186 | c.known[fqdn] = time.Now() 187 | knownClients.Set(float64(len(c.known))) 188 | } 189 | 190 | // KnownClients returns a list of alive clients 191 | func (c *Coordinator) KnownClients() []string { 192 | c.mu.Lock() 193 | defer c.mu.Unlock() 194 | 195 | limit := time.Now().Add(-*registrationTimeout) 196 | known := make([]string, 0, len(c.known)) 197 | for k, t := range c.known { 198 | if limit.Before(t) { 199 | known = append(known, k) 200 | } 201 | } 202 | return known 203 | } 204 | 205 | // Garbagee collect old clients. 206 | func (c *Coordinator) gc() { 207 | for range time.Tick(1 * time.Minute) { 208 | func() { 209 | c.mu.Lock() 210 | defer c.mu.Unlock() 211 | limit := time.Now().Add(-*registrationTimeout) 212 | deleted := 0 213 | for k, ts := range c.known { 214 | if ts.Before(limit) { 215 | delete(c.known, k) 216 | deleted++ 217 | } 218 | } 219 | c.logger.Info("GC of clients completed", "deleted", deleted, "remaining", len(c.known)) 220 | knownClients.Set(float64(len(c.known))) 221 | }() 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /cmd/proxy/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package main 15 | 16 | import ( 17 | "bufio" 18 | "bytes" 19 | "context" 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | "log/slog" 24 | "net/http" 25 | "os" 26 | "strings" 27 | 28 | "github.com/alecthomas/kingpin/v2" 29 | "github.com/prometheus/client_golang/prometheus" 30 | "github.com/prometheus/client_golang/prometheus/promhttp" 31 | "github.com/prometheus/common/promslog" 32 | "github.com/prometheus/common/promslog/flag" 33 | 34 | "github.com/prometheus-community/pushprox/util" 35 | ) 36 | 37 | const ( 38 | namespace = "pushprox_proxy" // For Prometheus metrics. 39 | ) 40 | 41 | var ( 42 | listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for proxy and client requests.").Default(":8080").String() 43 | maxScrapeTimeout = kingpin.Flag("scrape.max-timeout", "Any scrape with a timeout higher than this will have to be clamped to this.").Default("5m").Duration() 44 | defaultScrapeTimeout = kingpin.Flag("scrape.default-timeout", "If a scrape lacks a timeout, use this value.").Default("15s").Duration() 45 | ) 46 | 47 | var ( 48 | httpAPICounter = prometheus.NewCounterVec( 49 | prometheus.CounterOpts{ 50 | Name: "pushprox_http_requests_total", 51 | Help: "Number of http api requests.", 52 | }, []string{"code", "path"}, 53 | ) 54 | 55 | httpProxyCounter = prometheus.NewCounterVec( 56 | prometheus.CounterOpts{ 57 | Name: "pushproxy_proxied_requests_total", 58 | Help: "Number of http proxy requests.", 59 | }, []string{"code"}, 60 | ) 61 | httpPathHistogram = prometheus.NewHistogramVec( 62 | prometheus.HistogramOpts{ 63 | Name: "pushprox_http_duration_seconds", 64 | Help: "Time taken by path", 65 | }, []string{"path"}) 66 | ) 67 | 68 | func init() { 69 | prometheus.MustRegister(httpAPICounter, httpProxyCounter, httpPathHistogram) 70 | } 71 | 72 | func copyHTTPResponse(resp *http.Response, w http.ResponseWriter) { 73 | for k, v := range resp.Header { 74 | w.Header()[k] = v 75 | } 76 | w.WriteHeader(resp.StatusCode) 77 | io.Copy(w, resp.Body) 78 | } 79 | 80 | type targetGroup struct { 81 | Targets []string `json:"targets"` 82 | Labels map[string]string `json:"labels"` 83 | } 84 | 85 | type httpHandler struct { 86 | logger *slog.Logger 87 | coordinator *Coordinator 88 | mux http.Handler 89 | proxy http.Handler 90 | } 91 | 92 | func newHTTPHandler(logger *slog.Logger, coordinator *Coordinator, mux *http.ServeMux) *httpHandler { 93 | h := &httpHandler{logger: logger, coordinator: coordinator, mux: mux} 94 | 95 | // api handlers 96 | handlers := map[string]http.HandlerFunc{ 97 | "/push": h.handlePush, 98 | "/poll": h.handlePoll, 99 | "/clients": h.handleListClients, 100 | "/metrics": promhttp.Handler().ServeHTTP, 101 | } 102 | for path, handlerFunc := range handlers { 103 | counter := httpAPICounter.MustCurryWith(prometheus.Labels{"path": path}) 104 | handler := promhttp.InstrumentHandlerCounter(counter, http.HandlerFunc(handlerFunc)) 105 | histogram := httpPathHistogram.MustCurryWith(prometheus.Labels{"path": path}) 106 | handler = promhttp.InstrumentHandlerDuration(histogram, handler) 107 | mux.Handle(path, handler) 108 | counter.WithLabelValues("200") 109 | if path == "/push" { 110 | counter.WithLabelValues("500") 111 | } 112 | if path == "/poll" { 113 | counter.WithLabelValues("408") 114 | } 115 | } 116 | 117 | // proxy handler 118 | h.proxy = promhttp.InstrumentHandlerCounter(httpProxyCounter, http.HandlerFunc(h.handleProxy)) 119 | 120 | return h 121 | } 122 | 123 | // handlePush handles scrape responses from client. 124 | func (h *httpHandler) handlePush(w http.ResponseWriter, r *http.Request) { 125 | buf := &bytes.Buffer{} 126 | io.Copy(buf, r.Body) 127 | scrapeResult, err := http.ReadResponse(bufio.NewReader(buf), nil) 128 | if err != nil { 129 | h.logger.Error("Error reading pushed response:", "err", err) 130 | http.Error(w, fmt.Sprintf("Error pushing: %s", err.Error()), 500) 131 | return 132 | } 133 | scrapeId := scrapeResult.Header.Get("Id") 134 | h.logger.Info("Got /push", "scrape_id", scrapeId) 135 | err = h.coordinator.ScrapeResult(scrapeResult) 136 | if err != nil { 137 | h.logger.Error("Error pushing:", "err", err, "scrape_id", scrapeId) 138 | http.Error(w, fmt.Sprintf("Error pushing: %s", err.Error()), 500) 139 | } 140 | } 141 | 142 | // handlePoll handles clients registering and asking for scrapes. 143 | func (h *httpHandler) handlePoll(w http.ResponseWriter, r *http.Request) { 144 | fqdn, _ := io.ReadAll(r.Body) 145 | request, err := h.coordinator.WaitForScrapeInstruction(strings.TrimSpace(string(fqdn))) 146 | if err != nil { 147 | h.logger.Info("Error WaitForScrapeInstruction:", "err", err) 148 | http.Error(w, fmt.Sprintf("Error WaitForScrapeInstruction: %s", err.Error()), 408) 149 | return 150 | } 151 | //nolint:errcheck // https://github.com/prometheus-community/PushProx/issues/111 152 | request.WriteProxy(w) // Send full request as the body of the response. 153 | h.logger.Info("Responded to /poll", "url", request.URL.String(), "scrape_id", request.Header.Get("Id")) 154 | } 155 | 156 | // handleListClients handles requests to list available clients as a JSON array. 157 | func (h *httpHandler) handleListClients(w http.ResponseWriter, r *http.Request) { 158 | known := h.coordinator.KnownClients() 159 | targets := make([]*targetGroup, 0, len(known)) 160 | for _, k := range known { 161 | targets = append(targets, &targetGroup{Targets: []string{k}}) 162 | } 163 | w.Header().Set("Content-Type", "application/json") 164 | //nolint:errcheck // https://github.com/prometheus-community/PushProx/issues/111 165 | json.NewEncoder(w).Encode(targets) 166 | h.logger.Info("Responded to /clients", "client_count", len(known)) 167 | } 168 | 169 | // handleProxy handles proxied scrapes from Prometheus. 170 | func (h *httpHandler) handleProxy(w http.ResponseWriter, r *http.Request) { 171 | ctx, cancel := context.WithTimeout(r.Context(), util.GetScrapeTimeout(maxScrapeTimeout, defaultScrapeTimeout, r.Header)) 172 | defer cancel() 173 | request := r.WithContext(ctx) 174 | request.RequestURI = "" 175 | 176 | resp, err := h.coordinator.DoScrape(ctx, request) 177 | if err != nil { 178 | h.logger.Error("Error scraping:", "err", err, "url", request.URL.String()) 179 | http.Error(w, fmt.Sprintf("Error scraping %q: %s", request.URL.String(), err.Error()), 500) 180 | return 181 | } 182 | defer resp.Body.Close() 183 | copyHTTPResponse(resp, w) 184 | } 185 | 186 | // ServeHTTP discriminates between proxy requests (e.g. from Prometheus) and other requests (e.g. from the Client). 187 | func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 188 | if r.URL.Host != "" { // Proxy request 189 | h.proxy.ServeHTTP(w, r) 190 | } else { // Non-proxy requests 191 | h.mux.ServeHTTP(w, r) 192 | } 193 | } 194 | 195 | func main() { 196 | promslogConfig := promslog.Config{} 197 | flag.AddFlags(kingpin.CommandLine, &promslogConfig) 198 | kingpin.HelpFlag.Short('h') 199 | kingpin.Parse() 200 | logger := promslog.New(&promslogConfig) 201 | coordinator, err := NewCoordinator(logger) 202 | if err != nil { 203 | logger.Error("Coordinator initialization failed", "err", err) 204 | os.Exit(1) 205 | } 206 | 207 | mux := http.NewServeMux() 208 | handler := newHTTPHandler(logger, coordinator, mux) 209 | 210 | logger.Info("Listening", "address", *listenAddress) 211 | if err := http.ListenAndServe(*listenAddress, handler); err != nil { 212 | logger.Error("Listening failed", "err", err) 213 | os.Exit(1) 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /docs/sequence.svg: -------------------------------------------------------------------------------- 1 | 2 |
Some network barrier
Some network barrier
Client
fqdn=fqdn-x
Client<br>fqdn=fqdn-x<br>
scrape: GET ...
scrape: GET ...
Proxy
Proxy
POST fqdn to /poll
POST fqdn to /poll
Prometheus
Prometheus
GET fqdn-x:9100/metrics
GET <span>fqdn-x</span>:9100/metrics
fqdn-x:9100
fqdn-x:9100
POST metrics to /push
POST metrics to /push
GET fqdn-x:9100 / metrics
GET fqdn-x:9100 / metrics
metrics
metrics
1
1
4
[Not supported by viewer]
3
3
2
2
metrics
metrics
5
[Not supported by viewer]
6
[Not supported by viewer]
7
7 -------------------------------------------------------------------------------- /end-to-end-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -u -o pipefail 4 | 5 | tmpdir=$(mktemp -d /tmp/pushprox_e2e_test.XXXXXX) 6 | 7 | cleanup() { 8 | for f in "${tmpdir}"/*.pid ; do 9 | kill -9 "$(< $f)" 10 | done 11 | rm -r "${tmpdir}" 12 | } 13 | 14 | trap cleanup EXIT 15 | 16 | if type node_exporter > /dev/null 2>&1 ; then 17 | node_exporter & 18 | echo $! > "${tmpdir}/node_exporter.pid" 19 | fi 20 | while ! curl -s -f -L http://localhost:9100; do 21 | echo 'Waiting for node_exporter' 22 | sleep 2 23 | done 24 | 25 | ./pushprox-proxy --log.level=debug & 26 | echo $! > "${tmpdir}/proxy.pid" 27 | while ! curl -s -f -L http://localhost:8080/clients; do 28 | echo 'Waiting for proxy' 29 | sleep 2 30 | done 31 | 32 | ./pushprox-client --log.level=debug --proxy-url=http://localhost:8080 & 33 | echo $! > "${tmpdir}/client.pid" 34 | while [ "$(curl -s -L 'http://localhost:8080/clients' | jq 'length')" != '1' ] ; do 35 | echo 'Waiting for client' 36 | sleep 2 37 | done 38 | 39 | prometheus --config.file=prometheus.yml --log.level=debug & 40 | echo $! > "${tmpdir}/prometheus.pid" 41 | while ! curl -s -f -L http://localhost:9090/-/ready; do 42 | echo 'Waiting for Prometheus' 43 | sleep 2 44 | done 45 | sleep 15 46 | 47 | query='http://localhost:9090/api/v1/query?query=node_exporter_build_info' 48 | while [ $(curl -s -L "${query}" | jq '.data.result | length') != '1' ]; do 49 | echo 'Waiting for results' 50 | sleep 2 51 | done 52 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/prometheus-community/pushprox 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/Showmax/go-fqdn v1.0.0 7 | github.com/alecthomas/kingpin/v2 v2.4.0 8 | github.com/cenkalti/backoff/v4 v4.3.0 9 | github.com/google/uuid v1.6.0 10 | github.com/prometheus/client_golang v1.21.0 11 | github.com/prometheus/common v0.62.0 12 | ) 13 | 14 | require ( 15 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 16 | github.com/beorn7/perks v1.0.1 // indirect 17 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 18 | github.com/klauspost/compress v1.17.11 // indirect 19 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 20 | github.com/prometheus/client_model v0.6.1 // indirect 21 | github.com/prometheus/procfs v0.15.1 // indirect 22 | github.com/xhit/go-str2duration/v2 v2.1.0 // indirect 23 | golang.org/x/sys v0.28.0 // indirect 24 | google.golang.org/protobuf v1.36.1 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Showmax/go-fqdn v1.0.0 h1:0rG5IbmVliNT5O19Mfuvna9LL7zlHyRfsSvBPZmF9tM= 2 | github.com/Showmax/go-fqdn v1.0.0/go.mod h1:SfrFBzmDCtCGrnHhoDjuvFnKsWjEQX/Q9ARZvOrJAko= 3 | github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= 4 | github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= 5 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= 6 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 10 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 11 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 12 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 13 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 15 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 16 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 17 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 18 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 19 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 20 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 21 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 22 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 23 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 24 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 25 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 26 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 27 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 28 | github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= 29 | github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= 30 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 31 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 32 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 33 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 34 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 35 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 36 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 37 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 38 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 39 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 40 | github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= 41 | github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= 42 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 43 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 44 | google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= 45 | google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 46 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 47 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 48 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 49 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 50 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 51 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 52 | -------------------------------------------------------------------------------- /util/proxy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "net/http" 18 | "strconv" 19 | "time" 20 | ) 21 | 22 | func GetScrapeTimeout(maxScrapeTimeout, defaultScrapeTimeout *time.Duration, h http.Header) time.Duration { 23 | timeout := *defaultScrapeTimeout 24 | headerTimeout, err := GetHeaderTimeout(h) 25 | if err == nil { 26 | timeout = headerTimeout 27 | } 28 | if timeout > *maxScrapeTimeout { 29 | timeout = *maxScrapeTimeout 30 | } 31 | return timeout 32 | } 33 | 34 | func GetHeaderTimeout(h http.Header) (time.Duration, error) { 35 | timeoutSeconds, err := strconv.ParseFloat(h.Get("X-Prometheus-Scrape-Timeout-Seconds"), 64) 36 | if err != nil { 37 | return time.Duration(0 * time.Second), err 38 | } 39 | 40 | return time.Duration(timeoutSeconds * 1e9), nil 41 | } 42 | -------------------------------------------------------------------------------- /util/proxy_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | package util 15 | 16 | import ( 17 | "net/http" 18 | "testing" 19 | "time" 20 | ) 21 | 22 | func TestGetScrapeTimeout(t *testing.T) { 23 | // With header set 24 | maxScrapeTimeout := time.Duration(5 * time.Minute) 25 | defaultScrapeTimeout := time.Duration(10 * time.Second) 26 | header := http.Header{"X-Prometheus-Scrape-Timeout-Seconds": []string{"5.0"}} 27 | timeout := GetScrapeTimeout(&maxScrapeTimeout, &defaultScrapeTimeout, header) 28 | if timeout != time.Duration(5*time.Second) { 29 | t.Errorf("Expected 5s, got %s", timeout) 30 | } 31 | 32 | // With header unset 33 | header = http.Header{} 34 | timeout = GetScrapeTimeout(&maxScrapeTimeout, &defaultScrapeTimeout, header) 35 | if timeout != time.Duration(10*time.Second) { 36 | t.Errorf("Expected 10s, got %s", timeout) 37 | } 38 | 39 | // With header set empty 40 | header = http.Header{"X-Prometheus-Scrape-Timeout-Seconds": []string{}} 41 | timeout = GetScrapeTimeout(&maxScrapeTimeout, &defaultScrapeTimeout, header) 42 | if timeout != time.Duration(10*time.Second) { 43 | t.Errorf("Expected 10s, got %s", timeout) 44 | } 45 | 46 | // With header set higher than maxScrapeTimeout 47 | header = http.Header{"X-Prometheus-Scrape-Timeout-Seconds": []string{"600.0"}} 48 | timeout = GetScrapeTimeout(&maxScrapeTimeout, &defaultScrapeTimeout, header) 49 | if timeout != time.Duration(5*time.Minute) { 50 | t.Errorf("Expected 5m0s, got %s", timeout) 51 | } 52 | 53 | // With header set higher than defaultScrapeTimeout, lower than maxScrapeTimeout 54 | header = http.Header{"X-Prometheus-Scrape-Timeout-Seconds": []string{"30.0"}} 55 | defaultScrapeTimeout = time.Duration(10 * time.Second) 56 | timeout = GetScrapeTimeout(&maxScrapeTimeout, &defaultScrapeTimeout, header) 57 | if timeout != time.Duration(30*time.Second) { 58 | t.Errorf("Expected 30s, got %s", timeout) 59 | } 60 | } 61 | 62 | func TestGetHeaderTimeout(t *testing.T) { 63 | // With header set 64 | header := http.Header{"X-Prometheus-Scrape-Timeout-Seconds": []string{"5.0"}} 65 | timeout, err := GetHeaderTimeout(header) 66 | if err != nil { 67 | t.Errorf("Expected no error, got %v", err) 68 | } 69 | if timeout != time.Duration(5*time.Second) { 70 | t.Errorf("Expected 5s, got %s", timeout) 71 | } 72 | 73 | // With header unset 74 | header = http.Header{} 75 | timeout, err = GetHeaderTimeout(header) 76 | if err == nil { 77 | t.Error("Expected error, got none") 78 | } 79 | if timeout != time.Duration(0*time.Second) { 80 | t.Errorf("Expected 0s, got %s", timeout) 81 | } 82 | 83 | } 84 | --------------------------------------------------------------------------------