├── .github └── workflows │ └── pipeline.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cli └── cmd │ ├── embed.go │ ├── main.go │ ├── main_test.go │ ├── meta.go │ ├── promdump.tar.gz │ ├── restore.go │ └── root.go ├── core └── cmd │ └── main.go ├── go.mod ├── go.sum ├── img ├── demo_http_requests_total_dev_00.png └── demo_http_requests_total_dev_01.png ├── pkg ├── config │ └── config.go ├── download │ ├── download.go │ └── download_test.go ├── k8s │ ├── clientset.go │ ├── exec.go │ ├── exec_test.go │ ├── fake │ │ └── executor.go │ └── reader.go ├── log │ └── log.go └── tsdb │ ├── testdata │ └── wal │ │ ├── 00000037 │ │ ├── 00000038 │ │ ├── 00000039 │ │ └── checkpoint.00000036 │ │ └── 00000000 │ ├── tsdb.go │ └── tsdb_test.go └── plugins └── promdump.yaml /.github/workflows/pipeline.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | tags: 6 | - v* 7 | pull_request: 8 | branches: 9 | - main 10 | jobs: 11 | lint: 12 | runs-on: ubuntu-22.04 13 | steps: 14 | - name: checkout code 15 | uses: actions/checkout@v2 16 | - name: run linter 17 | run: | 18 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./ v1.48.0 19 | make lint PATH=$PATH:`pwd` 20 | build: 21 | runs-on: ubuntu-22.04 22 | outputs: 23 | version: ${{ steps.build.outputs.version }} 24 | steps: 25 | - name: install Go 26 | uses: actions/setup-go@v3 27 | with: 28 | go-version: 1.18.5 29 | - name: checkout code 30 | uses: actions/checkout@v2 31 | - name: test 32 | run: | 33 | make test 34 | - name: build 35 | id: build 36 | run: | 37 | version=$(echo "${{ github.ref }}" | awk -F'/' '{print $3}') 38 | make build VERSION="${version}" 39 | echo "::set-output name=version::${version}" 40 | image: 41 | if: ${{ startsWith(github.ref, 'refs/tags') }} 42 | needs: 43 | - lint 44 | - build 45 | runs-on: ubuntu-22.04 46 | steps: 47 | - name: checkout code 48 | uses: actions/checkout@v2 49 | - uses: docker/login-action@v2 50 | with: 51 | registry: ghcr.io 52 | username: ${{ github.actor }} 53 | password: ${{ secrets.GITHUB_TOKEN }} 54 | - run: make image VERSION=${{ needs.build.outputs.version }} IMAGE_PUSH="true" 55 | release: 56 | if: ${{ startsWith(github.ref, 'refs/tags') }} 57 | needs: 58 | - build 59 | - image 60 | runs-on: ubuntu-22.04 61 | steps: 62 | - name: install Go 63 | uses: actions/setup-go@v3 64 | with: 65 | go-version: 1.18.5 66 | - name: checkout code 67 | uses: actions/checkout@v2 68 | - name: release 69 | run: | 70 | make release VERSION=${{ needs.build.outputs.version }} 71 | - name: publish 72 | if: ${{ success() }} 73 | uses: softprops/action-gh-release@v1 74 | env: 75 | GITHUB_TOKEN: ${{ github.token }} 76 | with: 77 | files: | 78 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-darwin-amd64-${{ needs.build.outputs.version }}.tar.gz 79 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-darwin-amd64-${{ needs.build.outputs.version }}.tar.gz.sha256 80 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-darwin-arm64-${{ needs.build.outputs.version }}.tar.gz 81 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-darwin-arm64-${{ needs.build.outputs.version }}.tar.gz.sha256 82 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-linux-amd64-${{ needs.build.outputs.version }}.tar.gz 83 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-linux-amd64-${{ needs.build.outputs.version }}.tar.gz.sha256 84 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-windows-amd64-${{ needs.build.outputs.version }}.tar.gz 85 | target/releases/${{ needs.build.outputs.version }}/plugins/kubectl-promdump-windows-amd64-${{ needs.build.outputs.version }}.tar.gz.sha256 86 | target/releases/${{ needs.build.outputs.version }}/promdump 87 | target/releases/${{ needs.build.outputs.version }}/promdump.sha256 88 | - name: krew-index 89 | if: ${{ success() }} 90 | uses: rajatjindal/krew-release-bot@v0.0.43 91 | with: 92 | krew_template_file: plugins/promdump.yaml 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.swp 2 | /promdump* 3 | target/ 4 | /server 5 | /pkg/tsdb/chunks_head/ 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bookworm-20220622 2 | ARG KUBECTL_VERSION=v1.24.0 \ 3 | KREW_VERSION=v0.4.3 4 | RUN apt update -y && \ 5 | apt install -y curl git && \ 6 | curl -LO "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \ 7 | curl -LO "https://github.com/kubernetes-sigs/krew/releases/download/${KREW_VERSION}/krew-linux_amd64.tar.gz" && \ 8 | install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \ 9 | tar -C /usr/local/bin -xvf krew-linux_amd64.tar.gz && \ 10 | rm krew-linux_amd64.tar.gz && \ 11 | useradd -rm -d /home/promdump -s /bin/bash promdump 12 | USER promdump 13 | WORKDIR /home/promdump 14 | RUN /usr/local/bin/krew-linux_amd64 install krew 15 | ENV PATH="/home/promdump/.krew/bin:${PATH}" 16 | RUN kubectl krew update && \ 17 | kubectl krew install promdump 18 | CMD ["kubectl", "promdump", "-h"] 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL ?= /bin/bash 2 | 3 | BUILD_OS ?= linux 4 | BUILD_ARCH ?= amd64 5 | 6 | VERSION = $(shell git describe --abbrev=0) 7 | GIT_COMMIT=$(shell git rev-parse --short HEAD) 8 | 9 | BASE_DIR = $(shell pwd) 10 | TARGET_DIR = $(BASE_DIR)/target 11 | TARGET_BIN_DIR = $(TARGET_DIR)/bin 12 | TARGET_RELEASE_DIR = $(TARGET_DIR)/releases/$(VERSION) 13 | TARGET_PLUGINS_DIR = $(TARGET_RELEASE_DIR)/plugins 14 | 15 | IMAGE_REPO ?= ghcr.io/ihcsim/krew-promdump 16 | 17 | all: test lint build 18 | 19 | build: prebuild core cli 20 | prebuild: 21 | rm -rf $(TARGET_BIN_DIR) 22 | mkdir -p $(TARGET_BIN_DIR) 23 | 24 | .PHONY: test 25 | test: test-core test-cli 26 | go test ./... 27 | 28 | test-%: 29 | go test ./$*/... 30 | 31 | lint-%: 32 | cd ./$* && golangci-lint run --timeout 5m 33 | 34 | lint: lint-core lint-cli 35 | golangci-lint run 36 | 37 | tidy: tidy-core tidy-cli test 38 | 39 | tidy-%: 40 | cd ./$* && go mod tidy 41 | 42 | .PHONY: core 43 | core: 44 | CGO_ENABLED=0 GOOS="$(BUILD_OS)" GOARCH="$(BUILD_ARCH)" go build -o "$(TARGET_BIN_DIR)/promdump" ./core/cmd 45 | shasum -a256 "$(TARGET_BIN_DIR)/promdump" | awk '{print $$1}' > "$(TARGET_BIN_DIR)/promdump.sha256" 46 | tar -C "$(TARGET_BIN_DIR)" -czvf "$(TARGET_BIN_DIR)/promdump.tar.gz" promdump 47 | cp "$(TARGET_BIN_DIR)/promdump.tar.gz" ./cli/cmd/ 48 | 49 | .PHONY: cli 50 | cli: 51 | if [ "$(BUILD_OS)" = "windows" ]; then \ 52 | extension=".exe" ;\ 53 | fi && \ 54 | CGO_ENABLED=0 GOOS="$(BUILD_OS)" GOARCH="$(BUILD_ARCH)" go build -ldflags="-X 'main.Version=$(VERSION)' -X 'main.Commit=$(GIT_COMMIT)'" -o "$(TARGET_BIN_DIR)/cli-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION)$${extension}" ./cli/cmd &&\ 55 | shasum -a256 "$(TARGET_BIN_DIR)/cli-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION)"$${extension} | awk '{print $$1}' > "$(TARGET_BIN_DIR)/cli-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION).sha256" 56 | 57 | .PHONY: release 58 | release: 59 | rm -rf "$(TARGET_RELEASE_DIR)" && \ 60 | mkdir -p "$(TARGET_RELEASE_DIR)" && \ 61 | for os in linux darwin windows ; do \ 62 | $(MAKE) BUILD_OS="$${os}" BUILD_ARCH="amd64" TARGET_BIN_DIR="$(TARGET_RELEASE_DIR)" cli plugin ;\ 63 | done && \ 64 | $(MAKE) BUILD_OS="darwin" BUILD_ARCH="arm64" TARGET_BIN_DIR="$(TARGET_RELEASE_DIR)" cli plugin && \ 65 | $(MAKE) TARGET_BIN_DIR="$(TARGET_RELEASE_DIR)" core 66 | 67 | .PHONY: plugin 68 | plugin: 69 | mkdir -p "$(TARGET_PLUGINS_DIR)" && \ 70 | if [ "$(BUILD_OS)" = "windows" ]; then \ 71 | extension=".exe" ;\ 72 | fi && \ 73 | cp LICENSE "$(TARGET_PLUGINS_DIR)" && \ 74 | cp "$(TARGET_RELEASE_DIR)/cli-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION)$${extension}" "$(TARGET_PLUGINS_DIR)/kubectl-promdump$${extension}" && \ 75 | tar -C "$(TARGET_PLUGINS_DIR)" -czvf "$(TARGET_PLUGINS_DIR)/kubectl-promdump-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION).tar.gz" kubectl-promdump$${extension} LICENSE && \ 76 | rm "$(TARGET_PLUGINS_DIR)/kubectl-promdump$${extension}" && \ 77 | shasum -a256 $(TARGET_PLUGINS_DIR)/kubectl-promdump-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION).tar.gz | awk '{print $$1}' > $(TARGET_PLUGINS_DIR)/kubectl-promdump-$(BUILD_OS)-$(BUILD_ARCH)-$(VERSION).tar.gz.sha256 78 | 79 | image: 80 | docker build --rm -t $(IMAGE_REPO):$(VERSION) . 81 | if [ $${IMAGE_PUSH} ]; then \ 82 | docker push $(IMAGE_REPO):$(VERSION) ;\ 83 | fi 84 | 85 | .PHONY: hack/prometheus-repos 86 | hack/prometheus-repos: 87 | helm repo add prometheus-community https://prometheus-community.github.io/helm-charts 88 | helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics 89 | helm repo update 90 | 91 | .PHONY: hack/prometheus 92 | hack/prometheus: hack/prometheus-repos 93 | helm install prometheus prometheus-community/prometheus 94 | 95 | HACK_NAMESPACE ?= default 96 | HACK_DATA_DIR ?= /data 97 | 98 | .PHONY: hack/deploy 99 | hack/deploy: 100 | pod="$$(kubectl get pods --namespace $(HACK_NAMESPACE) -l "app=prometheus,component=server" -o jsonpath="{.items[0].metadata.name}")" ;\ 101 | kubectl -n "$(HACK_NAMESPACE)" cp -c prometheus-server "$(TARGET_BIN_DIR)/promdump" "$${pod}:$(HACK_DATA_DIR)" ;\ 102 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # promdump 2 | 3 | ![pipeline](https://github.com/ihcsim/promdump/actions/workflows/pipeline.yaml/badge.svg) 4 | 5 | promdump dumps the head and persistent blocks of Prometheus. It supports 6 | filtering the persistent blocks by time range. 7 | 8 | * [Why This Tool](#why-this-tool) 9 | * [How It Works](#how-it-works) 10 | * [Getting Started](#getting-started) 11 | * [FAQ](#faq) 12 | * [Limitations](#limitations) 13 | * [Development](#development) 14 | * [License](#license) 15 | 16 | ## Why This Tool 17 | 18 | When debugging Kubernetes clusters, I often find it helpful to get access to the 19 | in-cluster Prometheus metrics. Since it is unlikely that the users will grant me 20 | direct access to their Prometheus instance, I have to ask them to export the 21 | data. To reduce the amount of back-and-forth with the users (due to missing 22 | metrics, incorrect labels etc.), it makes sense to ask the users to _"get me 23 | everything around the time of the incident"_. 24 | 25 | The most common way to achieve this is to use commands like `kubectl exec` and 26 | `kubectl cp` to compress and dump Prometheus' entire data directory. On 27 | non-trivial clusters, the resulting compressed file can be very large. To 28 | import the data into a local test instance, I will need at least the same amount 29 | of disk space. 30 | 31 | promdump is a tool that can be used to dump Prometheus data blocks. It is 32 | different from the `promtool tsdb dump` command in such a way that its output 33 | can be re-used in another Prometheus instance. See this 34 | [issue](https://github.com/prometheus/prometheus/issues/8281) for a discussion 35 | on the limitation on the output of `promtool tsdb dump`. And unlike the 36 | Promethues TSDB `snapshot` API, promdump doesn't require Prometheus to be 37 | started with the `--web.enable-admin-api` option. Instead of dumping the entire 38 | TSDB, promdump offers the flexibility to filter persistent blocks by time range. 39 | 40 | ## How It Works 41 | 42 | The promdump kubectl plugin uploads the compressed, embeded promdump to the 43 | targeted Prometheus container, via the pod's `exec` subresource. 44 | 45 | Within the Prometheus container, promdump queries the Prometheus TSDB using the 46 | [`tsdb`](https://pkg.go.dev/github.com/prometheus/prometheus/tsdb) package. It 47 | reads and streams the WAL files, head block and persistent blocks to stdout, 48 | which can be redirected to a file on your local file system. To regulate the 49 | size of the dump, persistent blocks can be filtered by time range. 50 | 51 | ⭐ _promdump performs read-only operations on the TSDB._ 52 | 53 | When the data dump is completed, the promdump binary will be automatically 54 | deleted from your Prometheus container. 55 | 56 | The `restore` subcommand can then be used to copy this dump file to another 57 | Prometheus container. When this container is restarted, it will reconstruct its 58 | in-memory index and chunks using the restored on-disk memory-mapped chunks and 59 | WAL. 60 | 61 | The `--debug` option can be used to output more verbose logs for each command. 62 | 63 | ## Getting Started 64 | 65 | Install promdump as a `kubectl` plugin: 66 | ```sh 67 | kubectl krew update 68 | 69 | kubectl krew install promdump 70 | 71 | kubectl promdump --version 72 | ``` 73 | 74 | For demonstration purposes, use [kind](https://kind.sigs.k8s.io/) to create two 75 | K8s clusters: 76 | ```sh 77 | for i in {0..1}; do \ 78 | kind create cluster --name dev-0$i ;\ 79 | done 80 | ``` 81 | 82 | Install Prometheus on both clusters using the community 83 | [Helm chart](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus): 84 | ```sh 85 | for i in {0..1}; do \ 86 | helm --kube-context=kind-dev-0$i install prometheus prometheus-community/prometheus ;\ 87 | done 88 | ``` 89 | 90 | Deploy a custom controller to cluster `dev-00`. This controller is annotated for 91 | metrics scraping: 92 | ```sh 93 | kubectl --context=kind-dev-00 apply -f https://raw.githubusercontent.com/ihcsim/controllers/master/podlister/deployment.yaml 94 | ``` 95 | 96 | Port-forward to the Prometheus pod to find the custom `demo_http_requests_total` 97 | metric. 98 | 99 | 📝 _Later, we will use promdump to copy the samples of this metric over to the 100 | `dev-01` cluster._ 101 | 102 | ```sh 103 | CONTEXT="kind-dev-00" 104 | POD_NAME=$(kubectl --context "${CONTEXT}" get pods --namespace default -l "app=prometheus,component=server" -o jsonpath="{.items[0].metadata.name}") 105 | kubectl --context="${CONTEXT}" port-forward "${POD_NAME}" 9090 106 | ``` 107 | 108 | ![Demo controller metrics](img/demo_http_requests_total_dev_00.png) 109 | 110 | 📝 _In subsequent commands, the `-c` and `-d` options can be used to change 111 | the container name and data directoy._ 112 | 113 | Dump the data from the first cluster: 114 | ```sh 115 | # check the tsdb metadata. 116 | # if it's reported that there are no persistent blocks yet, then no data dump 117 | # will be captured, until Prometheus persists the data. 118 | kubectl promdump meta --context=$CONTEXT -p $POD_NAME 119 | Head Block Metadata 120 | ------------------------ 121 | Minimum time (UTC): | 2021-04-18 18:00:03 122 | Maximum time (UTC): | 2021-04-18 20:34:48 123 | Number of series | 18453 124 | 125 | Persistent Blocks Metadata 126 | ---------------------------- 127 | Minimum time (UTC): | 2021-04-15 03:19:10 128 | Maximum time (UTC): | 2021-04-18 18:00:00 129 | Total number of blocks | 9 130 | Total number of samples | 92561234 131 | Total number of series | 181304 132 | Total size | 139272005 133 | 134 | # capture the data dump 135 | TARFILE="dump-`date +%s`.tar.gz" 136 | kubectl promdump \ 137 | --context "${CONTEXT}" \ 138 | -p "${POD_NAME}" \ 139 | --min-time "2021-04-15 03:19:10" \ 140 | --max-time "2021-04-18 20:34:48" > "${TARFILE}" 141 | 142 | # view the content of the tar file. expect to see the 'chunk_heads', 'wal' and 143 | # persistent blocks directories. 144 | $ tar -tf "${TARFILE}" 145 | ``` 146 | 147 | Restore the data dump to the Prometheus pod on the `dev-01` cluster, where we 148 | don't have the custom controller: 149 | ```sh 150 | CONTEXT="kind-dev-01" 151 | POD_NAME=$(kubectl --context "${CONTEXT}" get pods --namespace default -l "app=prometheus,component=server" -o jsonpath="{.items[0].metadata.name}") 152 | 153 | # check the tsdb metadata 154 | kubectl promdump meta --context "${CONTEXT}" -p "${POD_NAME}" 155 | Head Block Metadata 156 | ------------------------ 157 | Minimum time (UTC): | 2021-04-18 20:39:21 158 | Maximum time (UTC): | 2021-04-18 20:47:30 159 | Number of series | 20390 160 | 161 | No persistent blocks found 162 | 163 | # restore the data dump found at ${TARFILE} 164 | kubectl promdump restore \ 165 | --context="${CONTEXT}" \ 166 | -p "${POD_NAME}" \ 167 | -t "${TARFILE}" 168 | 169 | # check the metadata again. it should match that of the dev-00 cluster 170 | kubectl promdump meta --context "${CONTEXT}" -p "${POD_NAME}" 171 | Head Block Metadata 172 | ------------------------ 173 | Minimum time (UTC): | 2021-04-18 18:00:03 174 | Maximum time (UTC): | 2021-04-18 20:35:48 175 | Number of series | 18453 176 | 177 | Persistent Blocks Metadata 178 | ---------------------------- 179 | Minimum time (UTC): | 2021-04-15 03:19:10 180 | Maximum time (UTC): | 2021-04-18 18:00:00 181 | Total number of blocks | 9 182 | Total number of samples | 92561234 183 | Total number of series | 181304 184 | Total size | 139272005 185 | 186 | # confirm that the WAL, head and persistent blocks are copied to the targeted 187 | # Prometheus server 188 | kubectl --context="${CONTEXT}" exec "${POD_NAME}" -c prometheus-server -- ls -al /data 189 | ``` 190 | 191 | Restart the Prometheus pod: 192 | ```sh 193 | kubectl --context="${CONTEXT}" delete po "${POD_NAME}" 194 | ``` 195 | 196 | Port-forward to the pod to confirm that the samples of 197 | the `demo_http_requests_total` metric have been copied over: 198 | ```sh 199 | kubectl --context="${CONTEXT}" port-forward "${POD_NAME}" 9091:9090 200 | ``` 201 | 202 | Make sure that time frame of your query matches that of the restored data. 203 | 204 | ![Restored metrics](img/demo_http_requests_total_dev_01.png) 205 | 206 | ## FAQ 207 | 208 | Q: The `promdump meta` subcommand shows that the time range of the restored 209 | persistent data blocks is different from the ones I specified. 210 | 211 | A: There isn't a way to fetch partial data blocks from the TSDB. If the time 212 | range you specified spans across multiple data blocks, then all of them need 213 | to be retrieved. The amount of excessive data retrieved is dependent on the 214 | span of the data blocks. 215 | 216 | The time range reported by the `promdump meta` subcommand should cover the one 217 | you specified. 218 | 219 | ---- 220 | Q: I am not seeing the restored data 221 | 222 | A: There are a few things you can check: 223 | 224 | * When generating the dump, make sure the start and end date times are 225 | specified in the UTC time zone. 226 | * If using the Prometheus console, make sure the time filter falls within the 227 | time range of your data dump. You can confirm your restored data time range 228 | using the `promdump meta` subcommand. 229 | * Compare the TSDB metadata of the target Prometheus with the source Prometheus 230 | to see if their time range match, using the `promdump meta` subcommand. 231 | The head block metadata may deviate slightly depending on how old your data dump 232 | is. 233 | * Use the `kubectl exec` command to run commands likes `ls -al ` 234 | and `cat //meta.json` to confirm the data range of a 235 | particular data block. 236 | * Try restarting the target Prometheus pod after the restoration to let 237 | Prometheus replay the restored WALs. The restored data must be persisted to 238 | survive a restart. 239 | * Check Prometheus logs to see if there are any errors due to corrupted data 240 | blocks, and report any [issues](https://github.com/ihcsim/promdump/issues/new). 241 | * Run the `promdump restore` subcommand with the `--debug` flag to see 242 | if it provides more hints. 243 | 244 | ---- 245 | Q: The `promdump meta` and `promdump restore` subcommands are failing with this 246 | error: 247 | ```sh 248 | found unsequential head chunk files 249 | ``` 250 | 251 | A: This happens when there are out-of-sequence files in the `chunk_heads` folder 252 | of the source Prometheus instance. 253 | 254 | The `promdump` command can still be used to generate the dump `.tar.gz` file 255 | because it doesn't parse the folder content, using the `tsdb` API. It simply 256 | adds the the entire `chunk_heads` folder to the dump `.tar.gz` file. 257 | 258 | E.g, a dump file with 2 out-of-sequence head files may look like this: 259 | ```sh 260 | $ tar -tf dump.tar.gz 261 | ./ 262 | ./chunks_head/ 263 | ./chunks_head/000027 # out-of-sequence 264 | ./chunks_head/000029 # out-of-sequence 265 | ./chunks_head/000033 266 | ./chunks_head/000034 267 | ./01F5ETH5T4MKTXJ1PEHQ71758P/ 268 | ./01F5ETH5T4MKTXJ1PEHQ71758P/index 269 | ./01F5ETH5T4MKTXJ1PEHQ71758P/chunks/ 270 | ./01F5ETH5T4MKTXJ1PEHQ71758P/chunks/000001 271 | ./01F5ETH5T4MKTXJ1PEHQ71758P/meta.json 272 | ./01F5ETH5T4MKTXJ1PEHQ71758P/tombstones 273 | ... 274 | ``` 275 | 276 | Any attempts to restore this dump file will crash the target Prometheus with the 277 | above error, complaining that files `000027` and `000028` are out-of-sequence. 278 | 279 | To fix this dump file, we will have to manually delete those offending files: 280 | ```sh 281 | mkdir temp 282 | 283 | tar -xvfz dump.tar.gz -C temp 284 | 285 | rm temp/chunks_head/000027 chunks_head/000029 286 | 287 | tar -C temp -czvf restored.tar.gz . 288 | ``` 289 | 290 | Now you can restore the `restored.tar.gz` file to your target Prometheus with: 291 | ``` 292 | kubectl promdump restore -p $POD_NAME -t restored.tar.gz 293 | ``` 294 | 295 | Note that deleting those head files may cause some head data to be lost. 296 | 297 | ---- 298 | ## Limitations 299 | 300 | promdump is still in its experimental phase. SREs can use it to copy data blocks 301 | from one Prometheus instance to another development instance, while debugging 302 | cluster issues. 303 | 304 | Before restoring the data dump, promdump will erase the content of the data 305 | folder in the target Prometheus instance, to avoid corrupting the data blocks 306 | due to conflicting segment error such as: 307 | 308 | ```sh 309 | opening storage failed: get segment range: segments are not sequential 310 | ``` 311 | 312 | Restoring a data dump containing out-of-sequence head blocks will crash the 313 | target Prometheus. See [FAQ](#faq) on how to fix the data dump. 314 | 315 | promdump not suitable for production backup/restore operation. 316 | 317 | Like `kubectl cp`, promdump requires the `tar` binary to be installed in the 318 | Prometheus container. 319 | 320 | ## Development 321 | 322 | To run linters and unit test: 323 | ```sh 324 | make lint test 325 | ``` 326 | 327 | To produce local builds: 328 | ```sh 329 | # the promdump core 330 | make core 331 | 332 | # the kubectl CLI plugin 333 | make cli 334 | ``` 335 | 336 | To test the `kubectl` plugin locally, the plugin manifest at 337 | `plugins/promdump.yaml` must be updated with the new checksums. Then run: 338 | ```sh 339 | kubectl krew install --manifest=plugins/promdump.yaml --archive=target/releases//plugins/kubectl-promdump-linux-amd64-.tar.gz 340 | ``` 341 | 342 | 343 | All the binaries can be found in the local `target/bin` folder. 344 | 345 | To install Prometheus via Helm: 346 | ```sh 347 | make hack/prometheus 348 | ``` 349 | 350 | To create a release candidate: 351 | 352 | ```sh 353 | make release VERSION= 354 | ``` 355 | 356 | To do a release: 357 | ```sh 358 | git tag -a v$version 359 | 360 | make release 361 | ``` 362 | All the release artifacts can be found in the local `target/releases` folder. 363 | Note that the GitHub Actions pipeline uses the same make release targets. 364 | 365 | ## License 366 | 367 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 368 | these files except in compliance with the License. You may obtain a copy of the 369 | License at: 370 | 371 | ``` 372 | http://www.apache.org/licenses/LICENSE-2.0 373 | ``` 374 | 375 | Unless required by applicable law or agreed to in writing, software distributed 376 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 377 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 378 | specific language governing permissions and limitations under the License. 379 | -------------------------------------------------------------------------------- /cli/cmd/embed.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import _ "embed" 4 | 5 | //go:embed promdump.tar.gz 6 | var promdumpBin []byte 7 | -------------------------------------------------------------------------------- /cli/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | _ "k8s.io/client-go/plugin/pkg/client/auth" 8 | ) 9 | 10 | func main() { 11 | rootCmd, err := initRootCmd() 12 | if err != nil { 13 | exitWithErr(err) 14 | } 15 | 16 | _ = initMetaCmd(rootCmd) 17 | if _, err := initRestoreCmd(rootCmd); err != nil { 18 | exitWithErr(err) 19 | } 20 | 21 | if err := rootCmd.Execute(); err != nil { 22 | exitWithErr(err) 23 | } 24 | } 25 | 26 | func exitWithErr(err error) { 27 | fmt.Fprintln(os.Stderr, err) 28 | os.Exit(1) 29 | } 30 | -------------------------------------------------------------------------------- /cli/cmd/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/ihcsim/promdump/pkg/config" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/pflag" 12 | ) 13 | 14 | var cmd *cobra.Command 15 | 16 | func TestConfig(t *testing.T) { 17 | if err := initFixtures(); err != nil { 18 | t.Fatal("unexpected error: ", err) 19 | } 20 | 21 | if err := cmd.Execute(); err != nil { 22 | t.Fatal("unexpected error: ", err) 23 | } 24 | 25 | args := buildArgsFromFlags(cmd, t) 26 | cmd.SetArgs(args) 27 | 28 | if err := cmd.Execute(); err != nil { 29 | t.Fatal("unexpected error: ", err) 30 | } 31 | 32 | assertConfig(cmd, appConfig, t) 33 | } 34 | 35 | func initFixtures() error { 36 | var err error 37 | cmd, err = initRootCmd() 38 | if err != nil { 39 | return err 40 | } 41 | 42 | cmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { 43 | appConfig, err = config.New(cmd.Flags()) 44 | if err != nil { 45 | return fmt.Errorf("failed to init viper config: %w", err) 46 | } 47 | 48 | return nil 49 | } 50 | 51 | cmd.RunE = func(cmd *cobra.Command, args []string) error { 52 | return nil // omit irrelevant streaming function 53 | } 54 | 55 | // preset required fields with no default value 56 | if err := cmd.PersistentFlags().Set("pod", "test-pod"); err != nil { 57 | return err 58 | } 59 | 60 | cmd.SetOutput(io.Discard) 61 | return nil 62 | } 63 | 64 | func buildArgsFromFlags(cmd *cobra.Command, t *testing.T) []string { 65 | // construct the CLI arguments 66 | var args []string 67 | cmd.Flags().VisitAll(func(f *pflag.Flag) { 68 | if skipFlag(f) { 69 | return 70 | } 71 | 72 | args = append(args, testArgs(f, t)...) 73 | }) 74 | 75 | return args 76 | } 77 | 78 | func skipFlag(f *pflag.Flag) bool { 79 | return f.Name == "help" || f.Name == "version" 80 | } 81 | 82 | // build the arguments to be passed to the command for testing. 83 | // returns a slice in the form of {"--flag", "test-flag"}. 84 | func testArgs(f *pflag.Flag, t *testing.T) []string { 85 | args := []string{fmt.Sprintf("--%s", f.Name)} 86 | 87 | switch f.Value.Type() { 88 | case "bool": 89 | return args 90 | case "string": 91 | args = append(args, fmt.Sprintf("test-%s", f.Name)) 92 | case "stringArray": 93 | args = append(args, fmt.Sprintf("test-%s-00", f.Name), 94 | fmt.Sprintf("--%s", f.Name), 95 | fmt.Sprintf("test-%s-01", f.Name)) 96 | default: 97 | t.Fatalf("unsupported type: %s (flag: %s)", f.Value.Type(), f.Name) 98 | } 99 | 100 | return args 101 | } 102 | 103 | func assertConfig(cmd *cobra.Command, appConfig *config.Config, t *testing.T) { 104 | // verify flags config 105 | cmd.Flags().VisitAll(func(f *pflag.Flag) { 106 | if skipFlag(f) { 107 | return 108 | } 109 | 110 | t.Run(f.Name, func(t *testing.T) { 111 | expected := expectedValue(f, t) 112 | if actual := appConfig.Get(f.Name); !reflect.DeepEqual(expected, actual) { 113 | t.Errorf("mismatch config: %s. expected: %v (%T), actual: %v (%T)", f.Name, expected, expected, actual, actual) 114 | } 115 | }) 116 | }) 117 | } 118 | 119 | func expectedValue(f *pflag.Flag, t *testing.T) interface{} { 120 | switch f.Value.Type() { 121 | case "bool": 122 | return true 123 | case "string": 124 | return fmt.Sprintf("test-%s", f.Name) 125 | case "stringArray": 126 | return fmt.Sprintf("[test-%s-00,test-%s-01]", f.Name, f.Name) 127 | default: 128 | t.Fatalf("unsupported type: %s (%s)", f.Value.Type(), f.Name) 129 | } 130 | 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /cli/cmd/meta.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/ihcsim/promdump/pkg/config" 9 | "github.com/ihcsim/promdump/pkg/k8s" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func initMetaCmd(rootCmd *cobra.Command) *cobra.Command { 14 | metaCmd := &cobra.Command{ 15 | Use: "meta -p POD [-n NAMESPACE] [-c CONTAINER] [-d DATA_DIR]", 16 | Short: "Shows the metadata of the Prometheus TSDB.", 17 | Example: `# show the metadata of all the data blocks in the Prometheus pod in 18 | # namespace 19 | kubectl promdump meta -p -n `, 20 | SilenceErrors: true, // let main() handles errors 21 | SilenceUsage: true, 22 | RunE: func(cmd *cobra.Command, args []string) error { 23 | if err := setMissingDefaults(cmd); err != nil { 24 | return fmt.Errorf("can't set missing defaults: %w", err) 25 | } 26 | 27 | if err := validateMetaOptions(cmd); err != nil { 28 | return fmt.Errorf("validation failed: %w", err) 29 | } 30 | 31 | if err := clientset.CanExec(); err != nil { 32 | return fmt.Errorf("exec operation denied: %w", err) 33 | } 34 | 35 | return runMeta(cmd, appConfig, clientset) 36 | }, 37 | } 38 | 39 | rootCmd.AddCommand(metaCmd) 40 | return metaCmd 41 | } 42 | 43 | func runMeta(cmd *cobra.Command, config *config.Config, clientset *k8s.Clientset) error { 44 | r := bytes.NewBuffer(promdumpBin) 45 | if err := uploadToContainer(r, config, clientset); err != nil { 46 | return err 47 | } 48 | defer func() { 49 | _ = clean(config, clientset) 50 | }() 51 | 52 | return printMeta(config, clientset) 53 | } 54 | 55 | func validateMetaOptions(cmd *cobra.Command) error { 56 | return nil 57 | } 58 | 59 | func printMeta(config *config.Config, clientset *k8s.Clientset) error { 60 | dataDir := config.GetString("data-dir") 61 | execCmd := []string{fmt.Sprintf("%s/promdump", dataDir), "-meta", 62 | "-data-dir", dataDir} 63 | if config.GetBool("debug") { 64 | execCmd = append(execCmd, "-debug") 65 | } 66 | 67 | return clientset.ExecPod(execCmd, os.Stdin, os.Stdout, os.Stderr, false) 68 | } 69 | -------------------------------------------------------------------------------- /cli/cmd/promdump.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/cli/cmd/promdump.tar.gz -------------------------------------------------------------------------------- /cli/cmd/restore.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/ihcsim/promdump/pkg/config" 10 | "github.com/ihcsim/promdump/pkg/k8s" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | func initRestoreCmd(rootCmd *cobra.Command) (*cobra.Command, error) { 15 | restoreCmd := &cobra.Command{ 16 | Use: "restore -p POD [-n NAMESPACE] [-c CONTAINER] [-d DATA_DIR]", 17 | Short: "Restores data dump to a Prometheus instance.", 18 | Example: `# copy and restore the data dump in the dump.tar.gz file to the Prometheus 19 | # in namespace . 20 | kubectl promdump restore -p -n -t dump.tar.gz`, 21 | SilenceErrors: true, // let main() handles errors 22 | SilenceUsage: true, 23 | RunE: func(cmd *cobra.Command, args []string) error { 24 | if err := setMissingDefaults(cmd); err != nil { 25 | return fmt.Errorf("can't set missing defaults: %w", err) 26 | } 27 | 28 | if err := clientset.CanExec(); err != nil { 29 | return fmt.Errorf("exec operation denied: %w", err) 30 | } 31 | 32 | return runRestore(appConfig, clientset) 33 | }, 34 | } 35 | 36 | restoreCmd.Flags().StringP("dump-file", "t", "", "path to the sample dump TAR file") 37 | if err := restoreCmd.MarkFlagRequired("dump-file"); err != nil { 38 | return nil, err 39 | } 40 | 41 | rootCmd.AddCommand(restoreCmd) 42 | return restoreCmd, nil 43 | } 44 | 45 | func runRestore(config *config.Config, clientset *k8s.Clientset) error { 46 | filename := config.GetString("dump-file") 47 | dumpFile, err := os.Open(filename) 48 | if err != nil { 49 | return fmt.Errorf("can't open dump file: %w", err) 50 | } 51 | 52 | data, err := io.ReadAll(dumpFile) 53 | if err != nil { 54 | return fmt.Errorf("can't read sample dump file: %w", err) 55 | } 56 | 57 | dataDir := config.GetString("data-dir") 58 | execCmd := []string{"sh", "-c", fmt.Sprintf("rm -rf %s/*", dataDir)} 59 | if err := clientset.ExecPod(execCmd, os.Stdin, os.Stdout, os.Stderr, false); err != nil { 60 | return err 61 | } 62 | 63 | return uploadToContainer(bytes.NewBuffer(data), config, clientset) 64 | } 65 | -------------------------------------------------------------------------------- /cli/cmd/root.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/ihcsim/promdump/pkg/log" 13 | 14 | "github.com/ihcsim/promdump/pkg/config" 15 | "github.com/ihcsim/promdump/pkg/k8s" 16 | "github.com/spf13/cobra" 17 | "github.com/spf13/pflag" 18 | k8scliopts "k8s.io/cli-runtime/pkg/genericclioptions" 19 | "k8s.io/client-go/rest" 20 | "k8s.io/client-go/tools/clientcmd" 21 | ) 22 | 23 | const ( 24 | timeFormat = "2006-01-02 15:04:05" 25 | ) 26 | 27 | var ( 28 | defaultContainer = "prometheus-server" 29 | defaultDataDir = "/data" 30 | defaultDebugEnabled = false 31 | defaultMaxTime = time.Now() 32 | defaultLogLevel = "error" 33 | defaultNamespace = "default" 34 | defaultMinTime = defaultMaxTime.Add(-1 * time.Hour) 35 | defaultRequestTimeout = "10s" 36 | 37 | appConfig *config.Config 38 | clientset *k8s.Clientset 39 | k8sConfigFlags *k8scliopts.ConfigFlags 40 | 41 | logger *log.Logger 42 | 43 | // Version is the version of the CLI, set during build time 44 | Version = "v0.1.0" 45 | Commit = "unknown" 46 | ) 47 | 48 | func initRootCmd() (*cobra.Command, error) { 49 | rootCmd := &cobra.Command{ 50 | Use: `promdump -p POD --min-time "yyyy-mm-dd hh:mm:ss" --max-time "yyyy-mm-dd hh:mm:ss" [-n NAMESPACE] [-c CONTAINER] [-d DATA_DIR]`, 51 | Short: "promdump dumps the head and persistent blocks of Prometheus", 52 | Example: `# dumps the head block and persistent blocks between 53 | # 2021-01-01 00:00:00 and 2021-04-02 16:59:00, from the Prometheus in the 54 | # namespace. 55 | kubectl promdump -p -n --min-time "2021-01-01 00:00:00" --max-time "2021-04-02 16:59:00" > dump.tar.gz`, 56 | Long: `promdump dumps the head and persistent blocks of Prometheus. It supports 57 | filtering the persistent blocks by time range. 58 | 59 | promdump is a tool that can be used to dump Prometheus data blocks. It is 60 | different from the 'promtool tsdb dump' command in such a way that its output 61 | can be re-used in another Prometheus instance. And unlike the Promethues TSDB 62 | 'snapshot' API, promdump doesn't require Prometheus to be started with the 63 | '--web.enable-admin-api' option. Instead of dumping the entire TSDB, promdump 64 | offers the flexibility to filter persistent blocks by time range. 65 | 66 | For more information on how promdump works, see 67 | https://github.com/ihcsim/promdump. 68 | `, 69 | Version: fmt.Sprintf("%s+%s", Version, Commit), 70 | SilenceErrors: true, // let main() handles errors 71 | SilenceUsage: true, 72 | RunE: func(cmd *cobra.Command, args []string) error { 73 | if err := setMissingDefaults(cmd); err != nil { 74 | return fmt.Errorf("can't set missing defaults: %w", err) 75 | } 76 | 77 | if err := validateRootOptions(cmd); err != nil { 78 | return fmt.Errorf("validation failed: %w", err) 79 | } 80 | 81 | if err := clientset.CanExec(); err != nil { 82 | return fmt.Errorf("exec operation denied: %w", err) 83 | } 84 | 85 | return run(cmd, appConfig, clientset) 86 | }, 87 | PersistentPreRunE: func(cmd *cobra.Command, args []string) error { 88 | var err error 89 | appConfig, err = config.New(cmd.Flags()) 90 | if err != nil { 91 | return fmt.Errorf("failed to init viper config: %w", err) 92 | } 93 | 94 | k8sConfig, err := k8sConfig(k8sConfigFlags, cmd.Flags()) 95 | if err != nil { 96 | return fmt.Errorf("failed to init k8s config: %w", err) 97 | } 98 | 99 | initLogger() 100 | clientset, err = k8s.NewClientset(appConfig, k8sConfig, logger) 101 | if err != nil { 102 | return fmt.Errorf("failed to init k8s client: %w", err) 103 | } 104 | 105 | return nil 106 | }, 107 | } 108 | 109 | // add default k8s client flags 110 | k8sConfigFlags = k8scliopts.NewConfigFlags(true) 111 | k8sConfigFlags.AddFlags(rootCmd.PersistentFlags()) 112 | 113 | rootCmd.PersistentFlags().StringP("pod", "p", "", "Prometheus pod name") 114 | rootCmd.PersistentFlags().StringP("container", "c", defaultContainer, "Prometheus container name") 115 | rootCmd.PersistentFlags().StringP("data-dir", "d", defaultDataDir, "Prometheus data directory") 116 | rootCmd.PersistentFlags().Bool("debug", defaultDebugEnabled, "run promdump in debug mode") 117 | rootCmd.Flags().String("min-time", defaultMinTime.Format(timeFormat), "min time (UTC) of the samples (yyyy-mm-dd hh:mm:ss)") 118 | rootCmd.Flags().String("max-time", defaultMaxTime.Format(timeFormat), "max time (UTC) of the samples (yyyy-mm-dd hh:mm:ss)") 119 | 120 | rootCmd.Flags().SortFlags = false 121 | if err := rootCmd.MarkPersistentFlagRequired("pod"); err != nil { 122 | return nil, err 123 | } 124 | 125 | setPluginUsageTemplate(rootCmd) 126 | 127 | return rootCmd, nil 128 | } 129 | 130 | func setMissingDefaults(cmd *cobra.Command) error { 131 | ns, err := cmd.Flags().GetString("namespace") 132 | if err != nil { 133 | return err 134 | } 135 | 136 | if ns == "" { 137 | if err := cmd.Flags().Set("namespace", defaultNamespace); err != nil { 138 | return err 139 | } 140 | } 141 | 142 | timeout, err := cmd.Flags().GetString("request-timeout") 143 | if err != nil { 144 | return err 145 | } 146 | 147 | if timeout == "0" { 148 | if err := cmd.Flags().Set("request-timeout", defaultRequestTimeout); err != nil { 149 | return err 150 | } 151 | } 152 | 153 | return nil 154 | } 155 | 156 | func validateRootOptions(cmd *cobra.Command) error { 157 | argMinTime, err := cmd.Flags().GetString("min-time") 158 | if err != nil { 159 | return err 160 | } 161 | 162 | argMaxTime, err := cmd.Flags().GetString("max-time") 163 | if err != nil { 164 | return err 165 | } 166 | 167 | minTime, err := time.Parse(timeFormat, argMinTime) 168 | if err != nil { 169 | return err 170 | } 171 | 172 | maxTime, err := time.Parse(timeFormat, argMaxTime) 173 | if err != nil { 174 | return err 175 | } 176 | 177 | if minTime.After(maxTime) { 178 | return fmt.Errorf("min time (%s) cannot be after max time (%s)", argMinTime, argMaxTime) 179 | } 180 | 181 | now := time.Now().UTC() 182 | if minTime.After(now) { 183 | return fmt.Errorf("min time (%s) cannot be after now (%s)", argMinTime, now.Format(timeFormat)) 184 | } 185 | 186 | if maxTime.After(now) { 187 | return fmt.Errorf("max time (%s) cannot be after now (%s)", argMaxTime, now.Format(timeFormat)) 188 | } 189 | 190 | return nil 191 | } 192 | 193 | func k8sConfig(k8sConfigFlags *k8scliopts.ConfigFlags, fs *pflag.FlagSet) (*rest.Config, error) { 194 | // read from CLI flags first 195 | // then if empty, load defaults from config loader 196 | currentContext, err := fs.GetString("context") 197 | if err != nil { 198 | return nil, err 199 | } 200 | 201 | configLoader := k8sConfigFlags.ToRawKubeConfigLoader() 202 | if currentContext == "" { 203 | rawConfig, err := configLoader.RawConfig() 204 | if err != nil { 205 | return nil, err 206 | } 207 | currentContext = rawConfig.CurrentContext 208 | } 209 | 210 | timeout, err := fs.GetString("request-timeout") 211 | if err != nil { 212 | return nil, err 213 | } 214 | 215 | if timeout == "" { 216 | clientConfig, err := configLoader.ClientConfig() 217 | if err != nil { 218 | return nil, err 219 | } 220 | timeout = clientConfig.Timeout.String() 221 | } 222 | 223 | return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( 224 | &clientcmd.ClientConfigLoadingRules{ 225 | ExplicitPath: configLoader.ConfigAccess().GetDefaultFilename(), 226 | WarnIfAllMissing: true, 227 | }, 228 | &clientcmd.ConfigOverrides{ 229 | CurrentContext: currentContext, 230 | Timeout: timeout, 231 | }).ClientConfig() 232 | } 233 | 234 | func run(cmd *cobra.Command, config *config.Config, clientset *k8s.Clientset) error { 235 | r := bytes.NewReader(promdumpBin) 236 | if err := uploadToContainer(r, config, clientset); err != nil { 237 | return err 238 | } 239 | defer func() { 240 | _ = clean(config, clientset) 241 | }() 242 | 243 | return dumpSamples(config, clientset) 244 | } 245 | 246 | func uploadToContainer(bin io.Reader, config *config.Config, clientset *k8s.Clientset) error { 247 | dataDir := config.GetString("data-dir") 248 | execCmd := []string{"tar", "-C", dataDir, "-xzvf", "-"} 249 | return clientset.ExecPod(execCmd, bin, io.Discard, os.Stderr, false) 250 | } 251 | 252 | func dumpSamples(config *config.Config, clientset *k8s.Clientset) error { 253 | dataDir := config.GetString("data-dir") 254 | maxTime, err := time.Parse(timeFormat, config.GetString("max-time")) 255 | if err != nil { 256 | return err 257 | } 258 | minTime, err := time.Parse(timeFormat, config.GetString("min-time")) 259 | if err != nil { 260 | return err 261 | } 262 | maxTimestamp := strconv.FormatInt(maxTime.UnixNano(), 10) 263 | minTimestamp := strconv.FormatInt(minTime.UnixNano(), 10) 264 | 265 | execCmd := []string{fmt.Sprintf("%s/promdump", dataDir), 266 | "-min-time", minTimestamp, 267 | "-max-time", maxTimestamp, 268 | "-data-dir", dataDir} 269 | if config.GetBool("debug") { 270 | execCmd = append(execCmd, "-debug") 271 | } 272 | 273 | return clientset.ExecPod(execCmd, os.Stdin, os.Stdout, os.Stderr, false) 274 | } 275 | 276 | func clean(config *config.Config, clientset *k8s.Clientset) error { 277 | dataDir := config.GetString("data-dir") 278 | execCmd := []string{"rm", "-f", fmt.Sprintf("%s/promdump", dataDir)} 279 | return clientset.ExecPod(execCmd, os.Stdin, os.Stdout, os.Stderr, false) 280 | } 281 | 282 | func initLogger() { 283 | logLevel := defaultLogLevel 284 | if appConfig.GetBool("debug") { 285 | logLevel = "debug" 286 | } 287 | 288 | logger = log.New(logLevel, os.Stderr) 289 | } 290 | 291 | func setPluginUsageTemplate(cmd *cobra.Command) { 292 | defaultTmpl := cmd.UsageTemplate() 293 | newTmpl := strings.ReplaceAll(defaultTmpl, "{{.CommandPath}}", "kubectl {{.CommandPath}}") 294 | newTmpl = strings.ReplaceAll(newTmpl, "{{.UseLine}}", "kubectl {{.UseLine}}") 295 | cmd.SetUsageTemplate(newTmpl) 296 | } 297 | -------------------------------------------------------------------------------- /core/cmd/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "compress/gzip" 7 | "flag" 8 | "fmt" 9 | "io" 10 | "os" 11 | "path/filepath" 12 | "time" 13 | 14 | "github.com/go-kit/kit/log/level" 15 | "github.com/ihcsim/promdump/pkg/log" 16 | "github.com/ihcsim/promdump/pkg/tsdb" 17 | promtsdb "github.com/prometheus/prometheus/tsdb" 18 | ) 19 | 20 | const ( 21 | defaultLogLevel = "error" 22 | 23 | timeFormatFile = "2006-01-02-150405" 24 | timeFormatOut = "2006-01-02 15:04:05" 25 | ) 26 | 27 | var ( 28 | logger *log.Logger 29 | msgNoHeadBlock = "No head block found" 30 | msgNoPersistentBlocks = "No persistent blocks found" 31 | targetDir = os.TempDir() 32 | ) 33 | 34 | func main() { 35 | var ( 36 | defaultMaxTime = time.Now().UTC() 37 | defaultMinTime = defaultMaxTime.Add(-2 * time.Hour) 38 | 39 | dataDir = flag.String("data-dir", "/data", "path to the Prometheus data directory") 40 | minTime = flag.Int64("min-time", defaultMinTime.UnixNano(), "lower bound of the timestamp range (in nanoseconds)") 41 | maxTime = flag.Int64("max-time", defaultMaxTime.UnixNano(), "upper bound of the timestamp range (in nanoseconds)") 42 | debug = flag.Bool("debug", false, "run promdump in debug mode") 43 | showMeta = flag.Bool("meta", false, "retrieve the Promtheus TSDB metadata") 44 | help = flag.Bool("help", false, "show usage") 45 | ) 46 | flag.Parse() 47 | 48 | if *help { 49 | flag.Usage() 50 | return 51 | } 52 | 53 | logLevel := defaultLogLevel 54 | if *debug { 55 | logLevel = "debug" 56 | } 57 | logger = log.New(logLevel, os.Stderr) 58 | 59 | if err := validateTimestamp(*minTime, *maxTime); err != nil { 60 | exit(err) 61 | } 62 | 63 | tsdb, err := tsdb.New(*dataDir, logger) 64 | if err != nil { 65 | exit(err) 66 | } 67 | defer tsdb.Close() 68 | 69 | if *showMeta { 70 | headMeta, blockMeta, err := tsdb.Meta() 71 | if err != nil { 72 | exit(err) 73 | } 74 | 75 | if _, err := writeMeta(headMeta, blockMeta); err != nil { 76 | exit(err) 77 | } 78 | 79 | return 80 | } 81 | 82 | blocks, err := tsdb.Blocks(*minTime, *maxTime) 83 | if err != nil { 84 | exit(err) 85 | } 86 | 87 | nbr, err := writeBlocks(*dataDir, blocks, os.Stdout) 88 | if err != nil { 89 | exit(err) 90 | } 91 | 92 | _ = level.Info(logger.Logger).Log("message", "operation completed", "numBytesRead", nbr) 93 | } 94 | 95 | func writeMeta(headMeta *tsdb.HeadMeta, blockMeta *tsdb.BlockMeta) (int64, error) { 96 | if headMeta.MinTime.IsZero() && headMeta.MaxTime.IsZero() { 97 | buf := bytes.NewBuffer([]byte(msgNoHeadBlock)) 98 | return buf.WriteTo(os.Stdout) 99 | } 100 | 101 | head := fmt.Sprintf(`Head Block Metadata 102 | ------------------------ 103 | Minimum time (UTC): | %s 104 | Maximum time (UTC): | %s 105 | Number of series | %d 106 | `, 107 | headMeta.MinTime.Format(timeFormatOut), 108 | headMeta.MaxTime.Format(timeFormatOut), 109 | headMeta.NumSeries) 110 | 111 | buf := bytes.NewBuffer([]byte(head)) 112 | if blockMeta.MinTime.IsZero() && blockMeta.MaxTime.IsZero() { 113 | if _, err := buf.Write([]byte("\n" + msgNoPersistentBlocks)); err != nil { 114 | return 0, err 115 | } 116 | return buf.WriteTo(os.Stdout) 117 | } 118 | 119 | blocks := fmt.Sprintf(` 120 | Persistent Blocks Metadata 121 | ---------------------------- 122 | Minimum time (UTC): | %s 123 | Maximum time (UTC): | %s 124 | Total number of blocks | %d 125 | Total number of samples | %d 126 | Total number of series | %d 127 | Total size | %d 128 | `, 129 | blockMeta.MinTime.Format(timeFormatOut), 130 | blockMeta.MaxTime.Format(timeFormatOut), 131 | blockMeta.BlockCount, 132 | blockMeta.NumSamples, 133 | blockMeta.NumSeries, 134 | blockMeta.Size) 135 | 136 | if _, err := buf.Write([]byte(blocks)); err != nil { 137 | return 0, err 138 | } 139 | 140 | return buf.WriteTo(os.Stdout) 141 | } 142 | 143 | func writeBlocks(dataDir string, blocks []*promtsdb.Block, w io.Writer) (int64, error) { 144 | if len(blocks) == 0 { 145 | buf := bytes.NewBuffer([]byte(msgNoPersistentBlocks)) 146 | return buf.WriteTo(os.Stdout) 147 | } 148 | 149 | pipeReader, pipeWriter := io.Pipe() 150 | defer pipeReader.Close() 151 | 152 | go func() { 153 | defer pipeWriter.Close() 154 | if err := compressed(dataDir, blocks, pipeWriter); err != nil { 155 | _ = level.Error(logger.Logger).Log("message", "error closing pipeWriter", "reason", err) 156 | } 157 | }() 158 | 159 | return io.Copy(w, pipeReader) 160 | } 161 | 162 | func compressed(dataDir string, blocks []*promtsdb.Block, writer *io.PipeWriter) error { 163 | var ( 164 | buf = &bytes.Buffer{} 165 | tw = tar.NewWriter(buf) 166 | ) 167 | 168 | dirs := []string{ 169 | filepath.Join(dataDir, "chunks_head"), 170 | filepath.Join(dataDir, "wal"), 171 | } 172 | for _, block := range blocks { 173 | dirs = append(dirs, block.Dir()) 174 | } 175 | 176 | // walk all the block directories 177 | for _, dir := range dirs { 178 | if err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { 179 | if err != nil { 180 | return err 181 | } 182 | 183 | var link string 184 | if info.Mode()&os.ModeSymlink == os.ModeSymlink { 185 | if link, err = os.Readlink(path); err != nil { 186 | return err 187 | } 188 | } 189 | 190 | header, err := tar.FileInfoHeader(info, link) 191 | if err != nil { 192 | return err 193 | } 194 | 195 | header.Name = path[len(dataDir)+1:] 196 | if err = tw.WriteHeader(header); err != nil { 197 | return err 198 | } 199 | 200 | if !info.Mode().IsRegular() { 201 | return nil 202 | } 203 | 204 | data, err := os.ReadFile(path) 205 | if err != nil { 206 | return fmt.Errorf("failed to read data file: %w", err) 207 | } 208 | 209 | buf := bytes.NewBuffer(data) 210 | if _, err := io.Copy(tw, buf); err != nil { 211 | return fmt.Errorf("failed to write compressed file: %w", err) 212 | } 213 | 214 | return nil 215 | }); err != nil { 216 | _ = level.Error(logger.Logger).Log("errors", err) 217 | } 218 | } 219 | 220 | if err := tw.Close(); err != nil { 221 | return err 222 | } 223 | 224 | now := time.Now() 225 | filename := fmt.Sprintf(filepath.Join(targetDir, "promdump-%s.tar.gz"), now.Format(timeFormatFile)) 226 | 227 | gwriter := gzip.NewWriter(writer) 228 | defer gwriter.Close() 229 | 230 | gwriter.Header = gzip.Header{ 231 | Name: filename, 232 | ModTime: now, 233 | OS: 255, 234 | } 235 | 236 | if _, err := gwriter.Write(buf.Bytes()); err != nil { 237 | return err 238 | } 239 | 240 | return nil 241 | } 242 | 243 | func validateTimestamp(minTime, maxTime int64) error { 244 | if minTime > maxTime { 245 | return fmt.Errorf("min-time (%d) cannot exceed max-time (%d)", minTime, maxTime) 246 | } 247 | 248 | now := time.Now().UnixNano() 249 | if minTime > now { 250 | return fmt.Errorf("min-time (%d) cannot exceed now (%d)", minTime, now) 251 | } 252 | 253 | if maxTime > now { 254 | return fmt.Errorf("max-time (%d) cannot exceed now (%d)", maxTime, now) 255 | } 256 | 257 | return nil 258 | } 259 | 260 | func exit(err error) { 261 | _ = level.Error(logger.Logger).Log("error", err) 262 | os.Exit(1) 263 | } 264 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ihcsim/promdump 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/go-kit/kit v0.10.0 7 | github.com/prometheus/prometheus v1.8.2-0.20201015110737-0a7fdd3b7696 8 | github.com/spf13/cobra v1.1.1 9 | github.com/spf13/pflag v1.0.5 10 | github.com/spf13/viper v1.7.0 11 | k8s.io/api v0.20.5 12 | k8s.io/apimachinery v0.20.5 13 | k8s.io/cli-runtime v0.20.5 14 | k8s.io/client-go v0.20.5 15 | ) 16 | 17 | require ( 18 | cloud.google.com/go v0.81.0 // indirect 19 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 20 | github.com/Azure/go-autorest/autorest v0.11.18 // indirect 21 | github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect 22 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 23 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 24 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 25 | github.com/PuerkitoBio/purell v1.1.1 // indirect 26 | github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect 27 | github.com/beorn7/perks v1.0.1 // indirect 28 | github.com/cespare/xxhash v1.1.0 // indirect 29 | github.com/cespare/xxhash/v2 v2.1.1 // indirect 30 | github.com/davecgh/go-spew v1.1.1 // indirect 31 | github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 // indirect 32 | github.com/emicklei/go-restful v2.9.5+incompatible // indirect 33 | github.com/evanphx/json-patch v4.9.0+incompatible // indirect 34 | github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect 35 | github.com/fsnotify/fsnotify v1.4.9 // indirect 36 | github.com/ghodss/yaml v1.0.0 // indirect 37 | github.com/go-logfmt/logfmt v0.5.0 // indirect 38 | github.com/go-logr/logr v0.2.0 // indirect 39 | github.com/go-openapi/jsonpointer v0.19.3 // indirect 40 | github.com/go-openapi/jsonreference v0.19.3 // indirect 41 | github.com/go-openapi/spec v0.19.8 // indirect 42 | github.com/go-openapi/swag v0.19.9 // indirect 43 | github.com/gogo/protobuf v1.3.1 // indirect 44 | github.com/golang/protobuf v1.5.2 // indirect 45 | github.com/golang/snappy v0.0.2 // indirect 46 | github.com/google/btree v1.0.0 // indirect 47 | github.com/google/gofuzz v1.1.0 // indirect 48 | github.com/googleapis/gnostic v0.4.1 // indirect 49 | github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect 50 | github.com/hashicorp/hcl v1.0.0 // indirect 51 | github.com/imdario/mergo v0.3.5 // indirect 52 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 53 | github.com/json-iterator/go v1.1.10 // indirect 54 | github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect 55 | github.com/magiconair/properties v1.8.1 // indirect 56 | github.com/mailru/easyjson v0.7.1 // indirect 57 | github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect 58 | github.com/mitchellh/mapstructure v1.2.2 // indirect 59 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 60 | github.com/modern-go/reflect2 v1.0.1 // indirect 61 | github.com/oklog/ulid v1.3.1 // indirect 62 | github.com/pelletier/go-toml v1.4.0 // indirect 63 | github.com/peterbourgon/diskv v2.0.1+incompatible // indirect 64 | github.com/pkg/errors v0.9.1 // indirect 65 | github.com/prometheus/client_golang v1.7.1 // indirect 66 | github.com/prometheus/client_model v0.2.0 // indirect 67 | github.com/prometheus/common v0.14.0 // indirect 68 | github.com/prometheus/procfs v0.2.0 // indirect 69 | github.com/spf13/afero v1.2.2 // indirect 70 | github.com/spf13/cast v1.3.0 // indirect 71 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 72 | github.com/subosito/gotenv v1.2.0 // indirect 73 | go.uber.org/atomic v1.7.0 // indirect 74 | golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect 75 | golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect 76 | golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c // indirect 77 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 78 | golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 // indirect 79 | golang.org/x/text v0.3.6 // indirect 80 | golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect 81 | google.golang.org/appengine v1.6.7 // indirect 82 | google.golang.org/protobuf v1.26.0 // indirect 83 | gopkg.in/inf.v0 v0.9.1 // indirect 84 | gopkg.in/ini.v1 v1.51.0 // indirect 85 | gopkg.in/yaml.v2 v2.3.0 // indirect 86 | k8s.io/klog/v2 v2.4.0 // indirect 87 | k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect 88 | k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect 89 | sigs.k8s.io/kustomize v2.0.3+incompatible // indirect 90 | sigs.k8s.io/structured-merge-diff/v4 v4.0.2 // indirect 91 | sigs.k8s.io/yaml v1.2.0 // indirect 92 | ) 93 | 94 | replace ( 95 | k8s.io/api => k8s.io/api v0.20.5 96 | k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.20.5 97 | k8s.io/apimachinery => k8s.io/apimachinery v0.20.5 98 | k8s.io/apiserver => k8s.io/apiserver v0.20.5 99 | k8s.io/cli-runtime => k8s.io/cli-runtime v0.20.5 100 | k8s.io/client-go => k8s.io/client-go v0.20.5 101 | k8s.io/cloud-provider => k8s.io/cloud-provider v0.20.5 102 | k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.20.5 103 | k8s.io/code-generator => k8s.io/code-generator v0.20.5 104 | k8s.io/component-base => k8s.io/component-base v0.20.5 105 | k8s.io/component-helpers => k8s.io/component-helpers v0.20.5 106 | k8s.io/controller-manager => k8s.io/controller-manager v0.20.5 107 | k8s.io/cri-api => k8s.io/cri-api v0.20.5 108 | k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.20.5 109 | k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.20.5 110 | k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.20.5 111 | k8s.io/kube-proxy => k8s.io/kube-proxy v0.20.5 112 | k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.20.5 113 | k8s.io/kubectl => k8s.io/kubectl v0.20.5 114 | k8s.io/kubelet => k8s.io/kubelet v0.20.5 115 | k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.20.5 116 | k8s.io/metrics => k8s.io/metrics v0.20.5 117 | k8s.io/mount-utils => k8s.io/mount-utils v0.20.5 118 | k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.20.5 119 | ) 120 | -------------------------------------------------------------------------------- /img/demo_http_requests_total_dev_00.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/img/demo_http_requests_total_dev_00.png -------------------------------------------------------------------------------- /img/demo_http_requests_total_dev_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/img/demo_http_requests_total_dev_01.png -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/spf13/pflag" 5 | "github.com/spf13/viper" 6 | ) 7 | 8 | // Config stores application configuration in a viper instance. 9 | type Config struct { 10 | *viper.Viper 11 | } 12 | 13 | // New returns a new instance of Config, bounded to the provided flagset. 14 | func New(flags *pflag.FlagSet) (*Config, error) { 15 | v := viper.New() 16 | if flags != nil { 17 | if err := v.BindPFlags(flags); err != nil { 18 | return nil, err 19 | } 20 | } 21 | 22 | return &Config{v}, nil 23 | } 24 | -------------------------------------------------------------------------------- /pkg/download/download.go: -------------------------------------------------------------------------------- 1 | package download 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "path" 11 | "path/filepath" 12 | "strings" 13 | "time" 14 | 15 | "github.com/go-kit/kit/log/level" 16 | "github.com/ihcsim/promdump/pkg/log" 17 | ) 18 | 19 | var errChecksumMismatch = fmt.Errorf("mismatch checksum") 20 | 21 | // Download can issue GET requests to download assets from a remote endpoint. 22 | type Download struct { 23 | logger *log.Logger 24 | http *http.Client 25 | localDir string 26 | } 27 | 28 | // New returns a new instance of Download. 29 | func New(localDir string, timeout time.Duration, logger *log.Logger) *Download { 30 | return &Download{ 31 | localDir: localDir, 32 | logger: logger, 33 | http: &http.Client{ 34 | Timeout: timeout, 35 | }, 36 | } 37 | } 38 | 39 | // Get issues a GET request to the remote endpoint to download the promdump TAR 40 | // file, to the localDir directory. If non-empty, it also fetches the SHA256 sum 41 | // file from the specifed endpoint, and used that to verified the content of the 42 | // downloaded TAR file. 43 | // If the file is already present on the local file system, then the download will 44 | // be skipped, unless force is set to true to trigger a re-download. 45 | // The content of the file is then read and returned. Caller is responsible for 46 | // closing the returned ReadCloser. 47 | func (d *Download) Get(force bool, remoteURI, remoteURISHA string) (io.ReadCloser, error) { 48 | var ( 49 | exists = true 50 | filename = path.Base(remoteURI) 51 | savedPath = filepath.Join(d.localDir, filename) 52 | ) 53 | 54 | tarFile, err := os.Open(savedPath) 55 | if err != nil { 56 | // file exists; errors caused by something else 57 | if !os.IsNotExist(err) { 58 | return nil, err 59 | } 60 | 61 | exists = false 62 | } 63 | 64 | if exists && !force { 65 | return tarFile, nil 66 | } 67 | 68 | if err := d.download(remoteURI, savedPath); err != nil { 69 | return nil, err 70 | } 71 | 72 | newFile, err := os.Open(savedPath) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | if err := d.checksum(newFile, remoteURISHA); err != nil { 78 | return nil, err 79 | } 80 | 81 | return newFile, nil 82 | } 83 | 84 | func (d *Download) download(remote, savedPath string) error { 85 | _ = level.Info(d.logger).Log("message", "downloading promdump", 86 | "remoteURI", remote, 87 | "timeout", d.http.Timeout, 88 | "localDir", savedPath) 89 | 90 | resp, err := d.http.Get(remote) 91 | if err != nil { 92 | return err 93 | } 94 | defer resp.Body.Close() 95 | 96 | if resp.StatusCode != http.StatusOK { 97 | return fmt.Errorf("download failed. reason: %s", resp.Status) 98 | } 99 | 100 | file, err := os.Create(savedPath) 101 | if err != nil { 102 | return err 103 | } 104 | 105 | nbr, err := io.CopyN(file, resp.Body, resp.ContentLength) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | _ = level.Info(d.logger).Log("message", "download completed", "numBytesWrite", nbr) 111 | return nil 112 | } 113 | 114 | func (d *Download) checksum(file *os.File, remote string) error { 115 | _ = level.Info(d.logger).Log("message", "verifying checksum", 116 | "endpoint", remote, 117 | "timeout", d.http.Timeout, 118 | ) 119 | 120 | resp, err := d.http.Get(remote) 121 | if err != nil { 122 | return err 123 | } 124 | defer resp.Body.Close() 125 | 126 | buf := &bytes.Buffer{} 127 | if _, err := io.CopyN(buf, resp.Body, resp.ContentLength); err != nil { 128 | return err 129 | } 130 | 131 | sha := sha256.New() 132 | if _, err := io.Copy(sha, file); err != nil { 133 | return err 134 | } 135 | if _, err := file.Seek(0, io.SeekStart); err != nil { 136 | return err 137 | } 138 | 139 | actual := fmt.Sprintf("%x", sha.Sum(nil)) 140 | expected := strings.TrimSpace(buf.String()) 141 | _ = level.Debug(d.logger).Log("message", "comparing checksum", 142 | "expected", expected, 143 | "actual", actual) 144 | if expected != actual { 145 | return fmt.Errorf("%w: expected:%s, actual:%s", errChecksumMismatch, expected, actual) 146 | } 147 | 148 | _ = level.Info(d.logger).Log("message", "confirmed checksum") 149 | return nil 150 | } 151 | -------------------------------------------------------------------------------- /pkg/download/download_test.go: -------------------------------------------------------------------------------- 1 | package download 2 | 3 | import ( 4 | "crypto/sha256" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/http/httptest" 9 | "os" 10 | "reflect" 11 | "testing" 12 | "time" 13 | 14 | "github.com/ihcsim/promdump/pkg/log" 15 | ) 16 | 17 | func TestDownload(t *testing.T) { 18 | var ( 19 | dirname = "promdump-test" 20 | downloadContent = []byte("test response data") 21 | force = false 22 | logger = log.New("debug", io.Discard) 23 | timeout = time.Second 24 | 25 | mux = http.NewServeMux() 26 | server = httptest.NewServer(mux) 27 | remoteURI = server.URL 28 | remoteURISHA = fmt.Sprintf("%s/checksum", server.URL) 29 | ) 30 | 31 | // returns the download content 32 | mux.Handle("/", http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { 33 | resp.WriteHeader(http.StatusOK) 34 | if _, err := resp.Write(downloadContent); err != nil { 35 | t.Fatal("unexpected error: ", err) 36 | } 37 | })) 38 | 39 | // returns the checksum of the download content 40 | mux.Handle("/checksum", http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { 41 | h := sha256.New() 42 | if _, err := h.Write(downloadContent); err != nil { 43 | t.Fatal("unexpected error: ", err) 44 | } 45 | d := fmt.Sprintf("%x", h.Sum(nil)) 46 | 47 | resp.WriteHeader(http.StatusOK) 48 | if _, err := resp.Write([]byte(d)); err != nil { 49 | t.Fatal("unexpected error: ", err) 50 | } 51 | })) 52 | 53 | // the downloaded content will be saved in tempDir 54 | tempDir, err := os.MkdirTemp("", dirname) 55 | if err != nil { 56 | t.Fatal("unexpected error: ", err) 57 | } 58 | defer os.RemoveAll(tempDir) 59 | 60 | d := New(tempDir, timeout, logger) 61 | reader, err := d.Get(force, remoteURI, remoteURISHA) 62 | if err != nil { 63 | t.Fatal("unexpected error: ", err) 64 | } 65 | 66 | // read and compare the downloaded content with the actual data 67 | actual, err := io.ReadAll(reader) 68 | if err != nil { 69 | t.Fatal("unexpected error: ", err) 70 | } 71 | 72 | if !reflect.DeepEqual(actual, downloadContent) { 73 | t.Errorf("mismatch response. expected:%s, actual:%s", downloadContent, actual) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/k8s/clientset.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "github.com/ihcsim/promdump/pkg/config" 5 | "github.com/ihcsim/promdump/pkg/log" 6 | "k8s.io/client-go/kubernetes" 7 | "k8s.io/client-go/rest" 8 | ) 9 | 10 | // Clientset knows how to interact with a K8s cluster. It has a reference to 11 | // user configuration stored in viper. 12 | type Clientset struct { 13 | config *config.Config 14 | k8sConfig *rest.Config 15 | logger *log.Logger 16 | kubernetes.Interface 17 | } 18 | 19 | // NewClientset returns a new Clientset for the given config. 20 | func NewClientset(appConfig *config.Config, k8sConfig *rest.Config, logger *log.Logger) (*Clientset, error) { 21 | k8sClientset, err := kubernetes.NewForConfig(k8sConfig) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | return &Clientset{ 27 | appConfig, 28 | k8sConfig, 29 | logger, 30 | k8sClientset}, nil 31 | } 32 | -------------------------------------------------------------------------------- /pkg/k8s/exec.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/url" 8 | "strings" 9 | 10 | "github.com/go-kit/kit/log/level" 11 | authzv1 "k8s.io/api/authorization/v1" 12 | corev1 "k8s.io/api/core/v1" 13 | metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 14 | "k8s.io/client-go/kubernetes/scheme" 15 | "k8s.io/client-go/rest" 16 | "k8s.io/client-go/tools/remotecommand" 17 | ) 18 | 19 | var deniedCreateExecErr = fmt.Errorf("no permissions to create exec subresource") 20 | 21 | // ExecPod issues an exec request to execute the given command to a particular 22 | // pod. 23 | func (c *Clientset) ExecPod(command []string, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { 24 | var ( 25 | ns = c.config.GetString("namespace") 26 | pod = c.config.GetString("pod") 27 | container = c.config.GetString("container") 28 | requestTimeout = c.config.GetDuration("request-timeout") 29 | minTime = c.config.GetTime("min-time") 30 | maxTime = c.config.GetTime("max-time") 31 | ) 32 | 33 | _ = level.Info(c.logger).Log("message", "sending exec request", 34 | "command", strings.Join(command, " "), 35 | "namespace", ns, 36 | "pod", pod, 37 | "container", container, 38 | "min-time", minTime, 39 | "max-time", maxTime) 40 | 41 | execRequest := c.CoreV1().RESTClient().Post(). 42 | Resource("pods"). 43 | Namespace(ns). 44 | Name(pod). 45 | SubResource("exec"). 46 | Timeout(requestTimeout) 47 | 48 | execRequest = execRequest.VersionedParams(&corev1.PodExecOptions{ 49 | Container: container, 50 | Command: command, 51 | Stdin: stdin != nil, 52 | Stdout: stdout != nil, 53 | Stderr: stderr != nil, 54 | TTY: tty, 55 | }, scheme.ParameterCodec) 56 | 57 | exec, err := newExecutor(c.k8sConfig, "POST", execRequest.URL()) 58 | if err != nil { 59 | return fmt.Errorf("failed to set up executor: %w", err) 60 | } 61 | 62 | if err := exec.Stream(remotecommand.StreamOptions{ 63 | Stdin: stdin, 64 | Stdout: stdout, 65 | Stderr: stderr, 66 | Tty: tty, 67 | }); err != nil { 68 | return fmt.Errorf("failed to exec command: %w", err) 69 | } 70 | 71 | return nil 72 | } 73 | 74 | // CanExec determines if the current user can create a exec subresource in the 75 | // given pod. 76 | func (c *Clientset) CanExec() error { 77 | var ( 78 | ns = c.config.GetString("namespace") 79 | timeout = c.config.GetDuration("request-timeout") 80 | ) 81 | selfAccessReview := &authzv1.SelfSubjectAccessReview{ 82 | Spec: authzv1.SelfSubjectAccessReviewSpec{ 83 | ResourceAttributes: &authzv1.ResourceAttributes{ 84 | Namespace: ns, 85 | Verb: "create", 86 | Group: "", 87 | Resource: "pods", 88 | Subresource: "exec", 89 | Name: "", 90 | }, 91 | }, 92 | } 93 | 94 | _ = level.Info(c.logger).Log("message", "checking for exec permissions", 95 | "namespace", ns, 96 | "request-timeout", timeout) 97 | 98 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 99 | defer cancel() 100 | 101 | response, err := c.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, selfAccessReview, metav1.CreateOptions{}) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | if !response.Status.Allowed { 107 | if response.Status.Reason != "" { 108 | return fmt.Errorf("%w. reason: %s", deniedCreateExecErr, response.Status.Reason) 109 | } 110 | return deniedCreateExecErr 111 | } 112 | 113 | _ = level.Info(c.logger).Log("message", "confirmed exec permissions", 114 | "namespace", ns, 115 | "request-timeout", timeout) 116 | return nil 117 | } 118 | 119 | var newExecutor = func(config *rest.Config, method string, url *url.URL) (remotecommand.Executor, error) { 120 | return remotecommand.NewSPDYExecutor(config, method, url) 121 | } 122 | -------------------------------------------------------------------------------- /pkg/k8s/exec_test.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "net/http" 7 | "net/url" 8 | "os" 9 | "testing" 10 | 11 | authzv1 "k8s.io/api/authorization/v1" 12 | apiruntime "k8s.io/apimachinery/pkg/runtime" 13 | "k8s.io/client-go/kubernetes" 14 | k8sfake "k8s.io/client-go/kubernetes/fake" 15 | "k8s.io/client-go/rest" 16 | fakerest "k8s.io/client-go/rest/fake" 17 | k8stesting "k8s.io/client-go/testing" 18 | "k8s.io/client-go/tools/remotecommand" 19 | 20 | "github.com/ihcsim/promdump/pkg/config" 21 | "github.com/ihcsim/promdump/pkg/k8s/fake" 22 | "github.com/ihcsim/promdump/pkg/log" 23 | "github.com/spf13/viper" 24 | ) 25 | 26 | func TestCanExec(t *testing.T) { 27 | 28 | var testCases = []struct { 29 | allowed bool 30 | reason string 31 | expected error 32 | }{ 33 | {allowed: true, reason: "allowed"}, 34 | {allowed: false, reason: "denied", expected: deniedCreateExecErr}, 35 | } 36 | 37 | for _, tc := range testCases { 38 | t.Run(tc.reason, func(t *testing.T) { 39 | reaction := func(action k8stesting.Action) (handled bool, ret apiruntime.Object, err error) { 40 | return true, 41 | &authzv1.SelfSubjectAccessReview{ 42 | Status: authzv1.SubjectAccessReviewStatus{ 43 | Allowed: tc.allowed, 44 | Reason: tc.reason, 45 | }, 46 | }, nil 47 | } 48 | 49 | k8sClientset := &k8sfake.Clientset{} 50 | k8sClientset.Fake.AddReactor("create", "selfsubjectaccessreviews", reaction) 51 | clientset := Clientset{ 52 | &config.Config{Viper: viper.New()}, 53 | &rest.Config{}, 54 | log.New("debug", io.Discard), 55 | k8sClientset, 56 | } 57 | 58 | if actual := clientset.CanExec(); !errors.Is(actual, tc.expected) { 59 | t.Errorf("mismatch errors: expected: %v, actual: %v", tc.expected, actual) 60 | } 61 | }) 62 | } 63 | } 64 | 65 | func TestExec(t *testing.T) { 66 | testConfig := &config.Config{Viper: viper.New()} 67 | testConfig.Set("namespace", "test-ns") 68 | testConfig.Set("pod", "test-pod") 69 | testConfig.Set("container", "test-container") 70 | testConfig.Set("request-timeout", "5s") 71 | testConfig.Set("min-time", "2000-01-01 00:00:00") 72 | testConfig.Set("max-time", "2000-01-02 00:00:00") 73 | 74 | execRoundTripper := func(req *http.Request) (*http.Response, error) { 75 | return &http.Response{ 76 | StatusCode: http.StatusOK, 77 | }, nil 78 | } 79 | 80 | restClient := &fakerest.RESTClient{ 81 | Client: fakerest.CreateHTTPClient(execRoundTripper), 82 | } 83 | k8sClientset := kubernetes.New(restClient) 84 | clientset := &Clientset{ 85 | testConfig, 86 | &rest.Config{}, 87 | log.New("debug", io.Discard), 88 | k8sClientset, 89 | } 90 | 91 | // fake the executor constructor so that the fake executor is used in the 92 | // ExecPod() method 93 | newExecutor = newFakeExecutor 94 | 95 | cmd := []string{} 96 | if err := clientset.ExecPod(cmd, os.Stdin, os.Stdout, os.Stderr, false); err != nil { 97 | t.Fatal("unexpected error: ", err) 98 | } 99 | } 100 | 101 | func newFakeExecutor(config *rest.Config, method string, url *url.URL) (remotecommand.Executor, error) { 102 | return fake.NewExecutor(url), nil 103 | } 104 | -------------------------------------------------------------------------------- /pkg/k8s/fake/executor.go: -------------------------------------------------------------------------------- 1 | package fake 2 | 3 | import ( 4 | "net/url" 5 | 6 | "k8s.io/client-go/tools/remotecommand" 7 | ) 8 | 9 | // Executor implements the remotecommand.Executor interface for testing 10 | // purpose. See https://pkg.go.dev/k8s.io/client-go/tools/remotecommand#Executor. 11 | type Executor struct { 12 | ServerURL *url.URL 13 | } 14 | 15 | // NewExecutor returns a new instance of Executor. 16 | func NewExecutor(serverURL *url.URL) *Executor { 17 | return &Executor{serverURL} 18 | } 19 | 20 | // Stream provides the implementation to satisfy the remotecommand.Executor 21 | // interface. 22 | func (f *Executor) Stream(options remotecommand.StreamOptions) error { 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /pkg/k8s/reader.go: -------------------------------------------------------------------------------- 1 | package k8s 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | ) 7 | 8 | // ReadCloser implements the io.ReadCloser interface. It provides a Close() 9 | // method to the io.Reader type, so that callers like k8s.ExecPod() can close 10 | // the stdin stream when done. 11 | type ReadCloser struct { 12 | *bytes.Buffer 13 | } 14 | 15 | func NewReadCloser(r io.Reader) (*ReadCloser, error) { 16 | buf := &bytes.Buffer{} 17 | if _, err := buf.ReadFrom(r); err != nil { 18 | return nil, err 19 | } 20 | 21 | return &ReadCloser{buf}, nil 22 | } 23 | 24 | // Close closes the underlying buffer of r by resetting it. 25 | func (r *ReadCloser) Close() error { 26 | r.Reset() 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | "time" 7 | 8 | "github.com/go-kit/kit/log" 9 | "github.com/go-kit/kit/log/level" 10 | ) 11 | 12 | // Logger encapsulates the underlying logging library. 13 | type Logger struct { 14 | log.Logger 15 | } 16 | 17 | // New returns a new contextual logger. Log lines will be written to out. 18 | func New(logLevel string, out io.Writer) *Logger { 19 | logger := log.NewLogfmtLogger(out) 20 | logger = log.With(logger, 21 | "time", log.TimestampFormat(now, time.RFC3339), 22 | "caller", log.DefaultCaller, 23 | ) 24 | 25 | var opt level.Option 26 | switch strings.ToLower(logLevel) { 27 | case level.DebugValue().String(): 28 | opt = level.AllowDebug() 29 | case level.ErrorValue().String(): 30 | opt = level.AllowError() 31 | case level.WarnValue().String(): 32 | opt = level.AllowWarn() 33 | case "all": 34 | opt = level.AllowAll() 35 | case "none": 36 | opt = level.AllowNone() 37 | case level.InfoValue().String(): 38 | fallthrough 39 | default: 40 | opt = level.AllowInfo() 41 | } 42 | logger = level.NewFilter(logger, opt) 43 | 44 | return &Logger{logger} 45 | } 46 | 47 | func now() time.Time { 48 | return time.Now() 49 | } 50 | 51 | // With returns a new contextual logger with keyvals prepended to those passed 52 | // to calls to Log. See https://pkg.go.dev/github.com/go-kit/kit/log#With 53 | func (l *Logger) With(keyvals ...interface{}) *Logger { 54 | l.Logger = log.With(l.Logger, keyvals...) 55 | return l 56 | } 57 | -------------------------------------------------------------------------------- /pkg/tsdb/testdata/wal/00000037: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/pkg/tsdb/testdata/wal/00000037 -------------------------------------------------------------------------------- /pkg/tsdb/testdata/wal/00000038: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/pkg/tsdb/testdata/wal/00000038 -------------------------------------------------------------------------------- /pkg/tsdb/testdata/wal/00000039: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/pkg/tsdb/testdata/wal/00000039 -------------------------------------------------------------------------------- /pkg/tsdb/testdata/wal/checkpoint.00000036/00000000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ihcsim/promdump/be920d15c8739eb2f93059415cfcad0b6e21a1ca/pkg/tsdb/testdata/wal/checkpoint.00000036/00000000 -------------------------------------------------------------------------------- /pkg/tsdb/tsdb.go: -------------------------------------------------------------------------------- 1 | package tsdb 2 | 3 | import ( 4 | "math" 5 | "path/filepath" 6 | "strings" 7 | "time" 8 | 9 | "github.com/go-kit/kit/log/level" 10 | "github.com/ihcsim/promdump/pkg/log" 11 | "github.com/prometheus/prometheus/tsdb" 12 | "github.com/prometheus/prometheus/tsdb/chunkenc" 13 | "github.com/prometheus/prometheus/tsdb/wal" 14 | ) 15 | 16 | // Tsdb knows how to access a Prometheus tsdb. 17 | type Tsdb struct { 18 | dataDir string 19 | db *tsdb.DBReadOnly 20 | logger *log.Logger 21 | } 22 | 23 | // HeadMeta contains metadata of the head block. 24 | type HeadMeta struct { 25 | *Meta 26 | NumChunks uint64 27 | } 28 | 29 | // BlockMeta contains aggregated metadata of all the persistent blocks. 30 | type BlockMeta struct { 31 | *Meta 32 | BlockCount int 33 | } 34 | 35 | // Meta contains metadata for a TSDB instance. 36 | type Meta struct { 37 | MaxTime time.Time 38 | MinTime time.Time 39 | NumSamples uint64 40 | NumSeries uint64 41 | Size int64 42 | } 43 | 44 | // New returns a new instance of Tsdb. 45 | func New(dataDir string, logger *log.Logger) (*Tsdb, error) { 46 | db, err := tsdb.OpenDBReadOnly(dataDir, logger) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return &Tsdb{dataDir, db, logger}, nil 51 | } 52 | 53 | // Close closes the underlying database connection. 54 | func (t *Tsdb) Close() error { 55 | _ = level.Debug(t.logger).Log("message", "closing connection to tsdb") 56 | return t.db.Close() 57 | } 58 | 59 | // Blocks looks for data blocks that fall within the provided time range, in the 60 | // data directory. 61 | func (t *Tsdb) Blocks(minTimeNano, maxTimeNano int64) ([]*tsdb.Block, error) { 62 | var ( 63 | minTime = time.Unix(0, minTimeNano).UTC() 64 | maxTime = time.Unix(0, maxTimeNano).UTC() 65 | ) 66 | _ = level.Debug(t.logger).Log("message", "accessing tsdb", 67 | "datadir", t.dataDir, 68 | "minTime", minTime, 69 | "maxTime", maxTime) 70 | 71 | var ( 72 | results []*tsdb.Block 73 | current string 74 | ) 75 | 76 | blocks, err := t.db.Blocks() 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | for _, block := range blocks { 82 | b, ok := block.(*tsdb.Block) 83 | if !ok { 84 | continue 85 | } 86 | 87 | var ( 88 | blMinTime = time.Unix(0, nanoseconds(b.MinTime())).UTC() 89 | blMaxTime = time.Unix(0, nanoseconds(b.MaxTime())).UTC() 90 | truncDir = b.Dir()[len(t.dataDir)+1:] 91 | blockDir = truncDir 92 | ) 93 | if i := strings.Index(truncDir, "/"); i != -1 { 94 | blockDir = truncDir[:strings.Index(truncDir, "/")] 95 | } 96 | 97 | _ = level.Debug(t.logger).Log("message", "checking block", 98 | "path", blockDir, 99 | "minTime (utc)", blMinTime, 100 | "maxTime (utc)", blMaxTime, 101 | ) 102 | 103 | if minTime.Equal(blMinTime) || maxTime.Equal(blMaxTime) || 104 | (maxTime.After(blMinTime) && maxTime.Before(blMaxTime)) || 105 | (minTime.After(blMinTime) && minTime.Before(blMaxTime)) || 106 | (minTime.Before(blMinTime) && maxTime.After(blMaxTime)) { 107 | 108 | if blockDir != current { 109 | current = blockDir 110 | _ = level.Debug(t.logger).Log("message", "adding block", "path", blockDir) 111 | } 112 | results = append(results, b) 113 | } else { 114 | _ = level.Debug(t.logger).Log("message", "skipping block", "path", blockDir) 115 | } 116 | } 117 | 118 | _ = level.Debug(t.logger).Log("message", "finish parsing persistent blocks", "numBlocksFound", len(results)) 119 | return results, nil 120 | } 121 | 122 | // BlockMeta returns metadata of the TSDB persistent blocks. 123 | func (t *Tsdb) Meta() (*HeadMeta, *BlockMeta, error) { 124 | _ = level.Debug(t.logger).Log("message", "retrieving tsdb metadata", "datadir", t.dataDir) 125 | headMeta, err := t.headMeta() 126 | if err != nil { 127 | return nil, nil, err 128 | } 129 | 130 | blockMeta, err := t.blockMeta() 131 | if err != nil { 132 | return nil, nil, err 133 | } 134 | 135 | return headMeta, blockMeta, nil 136 | } 137 | 138 | // headMeta() is based on the implementation of tsdb.FlushWAL() at 139 | // https://github.com/prometheus/prometheus/blob/80545bfb2eb8f9deeedc442130f7c4dc34525d8d/tsdb/db.go#L334 140 | func (t *Tsdb) headMeta() (*HeadMeta, error) { 141 | dir := filepath.Join(t.dataDir, "wal") 142 | _ = level.Debug(t.logger).Log("message", "retrieving head block metadata", "datadir", dir) 143 | wal, err := wal.Open(t.logger, dir) 144 | if err != nil { 145 | return nil, err 146 | } 147 | defer wal.Close() 148 | 149 | head, err := tsdb.NewHead(nil, t.logger, wal, 150 | tsdb.DefaultBlockDuration, 151 | "", 152 | chunkenc.NewPool(), 153 | tsdb.DefaultStripeSize, 154 | nil) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | blocks, err := t.db.Blocks() 160 | if err != nil { 161 | return nil, err 162 | } 163 | 164 | minValidTime := int64(math.MinInt64) 165 | if len(blocks) > 0 { 166 | minValidTime = blocks[len(blocks)-1].Meta().MaxTime 167 | } 168 | if err := head.Init(minValidTime); err != nil { 169 | return nil, err 170 | } 171 | 172 | // NumSamples and NumChunks are not populated by default. See 173 | // https://github.com/prometheus/prometheus/blob/80545bfb2eb8f9deeedc442130f7c4dc34525d8d/tsdb/head.go#L1600 174 | return &HeadMeta{ 175 | Meta: &Meta{ 176 | MaxTime: time.Unix(0, nanoseconds(head.MaxTime())).UTC(), 177 | MinTime: time.Unix(0, nanoseconds(head.MinTime())).UTC(), 178 | NumSeries: head.Meta().Stats.NumSeries, 179 | }, 180 | }, nil 181 | } 182 | 183 | func (t *Tsdb) blockMeta() (*BlockMeta, error) { 184 | _ = level.Debug(t.logger).Log("message", "retrieving persistent blocks metadata") 185 | blocks, err := t.db.Blocks() 186 | if err != nil { 187 | return nil, err 188 | } 189 | 190 | var ( 191 | maxTime time.Time 192 | minTime time.Time 193 | numSamples uint64 194 | numSeries uint64 195 | size int64 196 | ) 197 | for _, block := range blocks { 198 | b, ok := block.(*tsdb.Block) 199 | if !ok { 200 | continue 201 | } 202 | 203 | var ( 204 | blMinTime = time.Unix(0, nanoseconds(b.MinTime())).UTC() 205 | blMaxTime = time.Unix(0, nanoseconds(b.MaxTime())).UTC() 206 | truncDir = b.Dir()[len(t.dataDir)+1:] 207 | blockDir = truncDir 208 | ) 209 | if i := strings.Index(truncDir, "/"); i != -1 { 210 | blockDir = truncDir[:strings.Index(truncDir, "/")] 211 | } 212 | 213 | _ = level.Debug(t.logger).Log("message", "checking block", 214 | "path", blockDir, 215 | "minTime", blMinTime, 216 | "maxTime", blMaxTime, 217 | ) 218 | 219 | size += b.Size() 220 | numSamples += b.Meta().Stats.NumSamples 221 | numSeries += b.Meta().Stats.NumSeries 222 | 223 | if blMinTime.Before(minTime) || minTime.IsZero() { 224 | minTime = blMinTime 225 | } 226 | 227 | if blMaxTime.After(maxTime) || maxTime.IsZero() { 228 | maxTime = blMaxTime 229 | } 230 | } 231 | 232 | return &BlockMeta{ 233 | Meta: &Meta{ 234 | MaxTime: maxTime, 235 | MinTime: minTime, 236 | NumSamples: numSamples, 237 | NumSeries: numSeries, 238 | Size: size, 239 | }, 240 | BlockCount: len(blocks), 241 | }, nil 242 | } 243 | 244 | func nanoseconds(milliseconds int64) int64 { 245 | return milliseconds * 1000000 246 | } 247 | -------------------------------------------------------------------------------- /pkg/tsdb/tsdb_test.go: -------------------------------------------------------------------------------- 1 | package tsdb 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | "time" 10 | 11 | "github.com/ihcsim/promdump/pkg/log" 12 | "github.com/prometheus/prometheus/pkg/labels" 13 | promtsdb "github.com/prometheus/prometheus/tsdb" 14 | ) 15 | 16 | var series []*promtsdb.MetricSample 17 | 18 | func TestBlocks(t *testing.T) { 19 | logger := log.New("debug", io.Discard) 20 | tempDir, err := os.MkdirTemp("", "promdump-tsdb-test") 21 | if err != nil { 22 | t.Fatal("unexpected error: ", err) 23 | } 24 | defer os.RemoveAll(tempDir) 25 | 26 | // initialize head and persistent blocks in data directory 27 | if err := initHeadBlock(tempDir); err != nil { 28 | t.Fatal("unexpected error when creating head block: ", err) 29 | } 30 | blockOne, blockTwo, err := initPersistentBlocks(tempDir, logger, t) 31 | if err != nil { 32 | t.Fatal("unexpected error when creating persistent blocks: ", err) 33 | } 34 | 35 | // initialize tsdb 36 | tsdb, err := New(tempDir, logger) 37 | if err != nil { 38 | t.Fatal("unexpected error: ", err) 39 | } 40 | 41 | t.Run("meta", func(t *testing.T) { 42 | headMeta, blockMeta, err := tsdb.Meta() 43 | if err != nil { 44 | t.Fatal("unexpected error: ", err) 45 | } 46 | 47 | t.Run("head block", func(t *testing.T) { 48 | expectedNumSeries := 18171 // value is read from static test file 49 | if actual := headMeta.NumSeries; actual != uint64(expectedNumSeries) { 50 | t.Errorf("mismatch total series. expected: %d, actual: %d", expectedNumSeries, actual) 51 | } 52 | 53 | expectedMaxTime, err := time.Parse(timeFormat, "2021-04-18 18:28:21.05 UTC") 54 | if err != nil { 55 | t.Fatal("unexpected error: ", err) 56 | } 57 | if actual := headMeta.MaxTime; actual != expectedMaxTime { 58 | t.Errorf("mismatch max time. expected: %s, actual: %s", expectedMaxTime, actual) 59 | } 60 | 61 | expectedMinTime, err := time.Parse(timeFormat, "2021-04-18 13:00:05.939 UTC") 62 | if err != nil { 63 | t.Fatal("unexpected error: ", err) 64 | } 65 | if actual := headMeta.MinTime; actual != expectedMinTime { 66 | t.Errorf("mismatch min time. expected: %s, actual: %s", expectedMinTime, actual) 67 | } 68 | }) 69 | 70 | t.Run("persistent blocks", func(t *testing.T) { 71 | expectedNumSamples := len(series) 72 | if actual := blockMeta.NumSamples; actual != uint64(expectedNumSamples) { 73 | t.Errorf("mismatch total samples. expected: %d, actual: %d", expectedNumSamples, actual) 74 | } 75 | 76 | expectedNumSeries := len(series) 77 | if actual := blockMeta.NumSeries; actual != uint64(expectedNumSeries) { 78 | t.Errorf("mismatch total series. expected: %d, actual: %d", expectedNumSeries, actual) 79 | } 80 | 81 | expectedMinTime, err := time.Parse(timeFormat, "2021-03-30 01:05:00 UTC") 82 | if err != nil { 83 | t.Fatal("unexpected error: ", err) 84 | } 85 | if actual := blockMeta.MinTime; !actual.Equal(expectedMinTime) { 86 | t.Errorf("mismatch min time. expected: %s, actual: %s", expectedMinTime, actual) 87 | } 88 | 89 | expectedMaxTime, err := time.Parse(timeFormat, "2021-04-01 20:52:31 UTC") 90 | if err != nil { 91 | t.Fatal("unexpected error: ", err) 92 | } 93 | if actual := blockMeta.MaxTime; !actual.Equal(expectedMaxTime) { 94 | t.Errorf("mismatch max time. expected: %s, actual: %s", expectedMaxTime, actual) 95 | } 96 | }) 97 | }) 98 | 99 | t.Run("blocks", func(t *testing.T) { 100 | var testCases = []struct { 101 | minTimeNano int64 102 | maxTimeNano int64 103 | expected []string 104 | }{ 105 | { 106 | minTimeNano: unix("2021-04-01 18:52:31 UTC", time.Nanosecond, t), 107 | maxTimeNano: unix("2021-04-01 20:52:31 UTC", time.Nanosecond, t), 108 | expected: []string{blockOne}, 109 | }, 110 | { 111 | minTimeNano: unix("2021-04-01 19:00:00 UTC", time.Nanosecond, t), 112 | maxTimeNano: unix("2021-04-01 20:00:00 UTC", time.Nanosecond, t), 113 | expected: []string{blockOne}, 114 | }, 115 | { 116 | minTimeNano: unix("2021-03-30 01:30:00 UTC", time.Nanosecond, t), 117 | maxTimeNano: unix("2021-03-30 02:00:00 UTC", time.Nanosecond, t), 118 | expected: []string{blockTwo}, 119 | }, 120 | { 121 | minTimeNano: unix("2021-03-30 01:05:00 UTC", time.Nanosecond, t), 122 | maxTimeNano: unix("2021-03-30 03:05:00 UTC", time.Nanosecond, t), 123 | expected: []string{blockTwo}, 124 | }, 125 | { 126 | minTimeNano: unix("2021-01-01 00:00:00 UTC", time.Nanosecond, t), 127 | maxTimeNano: unix("2021-01-01 02:16:00 UTC", time.Nanosecond, t), 128 | expected: []string{}, 129 | }, 130 | { 131 | minTimeNano: unix("2021-04-01 16:00:00 UTC", time.Nanosecond, t), 132 | maxTimeNano: unix("2021-04-01 19:12:10 UTC", time.Nanosecond, t), 133 | expected: []string{blockOne}, 134 | }, 135 | { 136 | minTimeNano: unix("2021-04-01 19:00:00 UTC", time.Nanosecond, t), 137 | maxTimeNano: unix("2021-04-01 22:48:09 UTC", time.Nanosecond, t), 138 | expected: []string{blockOne}, 139 | }, 140 | { 141 | minTimeNano: unix("2021-03-30 00:00:00 UTC", time.Nanosecond, t), 142 | maxTimeNano: unix("2021-03-30 02:58:09 UTC", time.Nanosecond, t), 143 | expected: []string{blockTwo}, 144 | }, 145 | { 146 | minTimeNano: unix("2021-03-30 02:00:00 UTC", time.Nanosecond, t), 147 | maxTimeNano: unix("2021-03-30 04:18:09 UTC", time.Nanosecond, t), 148 | expected: []string{blockTwo}, 149 | }, 150 | { 151 | minTimeNano: unix("2021-03-30 01:05:00 UTC", time.Nanosecond, t), 152 | maxTimeNano: unix("2021-04-01 20:52:31 UTC", time.Nanosecond, t), 153 | expected: []string{blockOne, blockTwo}, 154 | }, 155 | { 156 | minTimeNano: unix("2021-03-30 00:00:00 UTC", time.Nanosecond, t), 157 | maxTimeNano: unix("2021-04-02 00:00:00 UTC", time.Nanosecond, t), 158 | expected: []string{blockOne, blockTwo}, 159 | }, 160 | } 161 | 162 | for i, tc := range testCases { 163 | t.Run(fmt.Sprintf("test case #%d", i), func(t *testing.T) { 164 | actual, err := tsdb.Blocks(tc.minTimeNano, tc.maxTimeNano) 165 | if err != nil { 166 | t.Fatal("unexpected error: ", err) 167 | } 168 | 169 | if len(tc.expected) == 0 && len(actual) == 0 { 170 | return 171 | } 172 | 173 | var found bool 174 | LOOP: 175 | for _, path := range tc.expected { 176 | for _, block := range actual { 177 | if path == block.Dir() { 178 | found = true 179 | break LOOP 180 | } 181 | } 182 | } 183 | if !found { 184 | t.Errorf("missing expected blocks: %v\n actual: %v", tc.expected, actual) 185 | } 186 | }) 187 | } 188 | }) 189 | } 190 | 191 | func initHeadBlock(tempDir string) error { 192 | // copy checkpoint test data to tempDir 193 | walDir := filepath.Join(tempDir, "wal") 194 | checkpointDir := filepath.Join(walDir, "checkpoint.00000036") 195 | if err := os.MkdirAll(checkpointDir, 0755); err != nil { 196 | return err 197 | } 198 | 199 | checkpointTestFile := filepath.Join("testdata", "wal", "checkpoint.00000036", "00000000") 200 | checkpointTestdata, err := os.ReadFile(checkpointTestFile) 201 | if err != nil { 202 | return err 203 | } 204 | 205 | checkpointFile := filepath.Join(checkpointDir, "00000037") 206 | if err := os.WriteFile(checkpointFile, checkpointTestdata, 0600); err != nil { 207 | return err 208 | } 209 | 210 | // copy wal test data to tempDir 211 | for _, testFile := range []string{"00000037", "00000038", "00000039"} { 212 | wal, err := os.ReadFile(filepath.Join("testdata", "wal", testFile)) 213 | if err != nil { 214 | return err 215 | } 216 | 217 | walFile := filepath.Join(walDir, testFile) 218 | if err := os.WriteFile(walFile, wal, 0600); err != nil { 219 | return err 220 | } 221 | } 222 | 223 | return nil 224 | } 225 | 226 | func initPersistentBlocks(tempDir string, logger *log.Logger, t *testing.T) (string, string, error) { 227 | series = []*promtsdb.MetricSample{ 228 | { 229 | TimestampMs: unix("2021-04-01 20:30:31 UTC", time.Millisecond, t), 230 | Labels: []labels.Label{ 231 | {Name: "job", Value: "tsdb"}, 232 | {Name: "app", Value: "app-00"}, 233 | }, 234 | }, 235 | { 236 | TimestampMs: unix("2021-04-01 20:00:31 UTC", time.Millisecond, t), 237 | Labels: []labels.Label{ 238 | {Name: "job", Value: "tsdb"}, 239 | {Name: "app", Value: "app-01"}, 240 | }, 241 | }, 242 | { 243 | TimestampMs: unix("2021-04-01 19:33:00 UTC", time.Millisecond, t), 244 | Labels: []labels.Label{ 245 | {Name: "job", Value: "tsdb"}, 246 | {Name: "app", Value: "app-02"}, 247 | }, 248 | }, 249 | { 250 | TimestampMs: unix("2021-03-30 01:06:47 UTC", time.Millisecond, t), 251 | Labels: []labels.Label{ 252 | {Name: "job", Value: "tsdb"}, 253 | {Name: "app", Value: "app-00"}, 254 | }, 255 | }, 256 | { 257 | TimestampMs: unix("2021-03-30 02:36:00 UTC", time.Millisecond, t), 258 | Labels: []labels.Label{ 259 | {Name: "job", Value: "tsdb"}, 260 | {Name: "app", Value: "app-01"}, 261 | }, 262 | }, 263 | { 264 | TimestampMs: unix("2021-03-30 03:01:48 UTC", time.Millisecond, t), 265 | Labels: []labels.Label{ 266 | {Name: "job", Value: "tsdb"}, 267 | {Name: "app", Value: "app-02"}, 268 | }, 269 | }, 270 | } 271 | 272 | blockOne, err := promtsdb.CreateBlock(series[:3], tempDir, 273 | unix("2021-04-01 18:52:31 UTC", time.Millisecond, t), 274 | unix("2021-04-01 20:52:31 UTC", time.Millisecond, t), 275 | logger.Logger) 276 | if err != nil { 277 | return "", "", err 278 | } 279 | 280 | blockTwo, err := promtsdb.CreateBlock(series[3:], tempDir, 281 | unix("2021-03-30 01:05:00 UTC", time.Millisecond, t), 282 | unix("2021-03-30 03:05:00 UTC", time.Millisecond, t), 283 | logger.Logger) 284 | if err != nil { 285 | return "", "", err 286 | } 287 | 288 | return blockOne, blockTwo, err 289 | } 290 | 291 | const timeFormat = "2006-01-02 15:04:05 MST" 292 | 293 | func unix(date string, d time.Duration, t *testing.T) int64 { 294 | parsed, err := time.Parse(timeFormat, date) 295 | if err != nil { 296 | t.Fatal("unexpected error: ", err) 297 | } 298 | 299 | switch d { 300 | case time.Millisecond: 301 | return parsed.Unix() * 1000 302 | case time.Nanosecond: 303 | return parsed.UnixNano() 304 | default: 305 | t.Fatal("unsupported duration type: ", d) 306 | } 307 | 308 | return 0 309 | } 310 | -------------------------------------------------------------------------------- /plugins/promdump.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: krew.googlecontainertools.github.com/v1alpha2 2 | kind: Plugin 3 | metadata: 4 | name: promdump 5 | spec: 6 | version: {{ .TagName }} 7 | homepage: https://github.com/ihcsim/promdump 8 | shortDescription: Dumps the head and persistent blocks of Prometheus. 9 | description: | 10 | promdump dumps the head and persistent blocks of Prometheus. It supports 11 | filtering the persistent blocks by time range. 12 | 13 | promdump is a tool that can be used to dump Prometheus data blocks. It is 14 | different from the 'promtool tsdb dump' command in such a way that its output 15 | can be re-used in another Prometheus instance. And unlike the Promethues TSDB 16 | 'snapshot' API, promdump doesn't require Prometheus to be started with the 17 | '--web.enable-admin-api' option. Instead of dumping the entire TSDB, promdump 18 | offers the flexibility to filter persistent blocks by time range. 19 | 20 | To get started, follow the steps at https://github.com/ihcsim/promdump#getting-started 21 | platforms: 22 | - selector: 23 | matchLabels: 24 | os: darwin 25 | arch: amd64 26 | bin: kubectl-promdump 27 | {{ addURIAndSha "https://github.com/ihcsim/promdump/releases/download/{{ .TagName }}/kubectl-promdump-darwin-amd64-{{ .TagName }}.tar.gz" .TagName }} 28 | - selector: 29 | matchLabels: 30 | os: darwin 31 | arch: arm64 32 | bin: kubectl-promdump 33 | {{ addURIAndSha "https://github.com/ihcsim/promdump/releases/download/{{ .TagName }}/kubectl-promdump-darwin-arm64-{{ .TagName }}.tar.gz" .TagName }} 34 | - selector: 35 | matchLabels: 36 | os: linux 37 | arch: amd64 38 | bin: kubectl-promdump 39 | {{ addURIAndSha "https://github.com/ihcsim/promdump/releases/download/{{ .TagName }}/kubectl-promdump-linux-amd64-{{ .TagName }}.tar.gz" .TagName }} 40 | - selector: 41 | matchLabels: 42 | os: windows 43 | arch: amd64 44 | bin: kubectl-promdump.exe 45 | {{ addURIAndSha "https://github.com/ihcsim/promdump/releases/download/{{ .TagName }}/kubectl-promdump-windows-amd64-{{ .TagName }}.tar.gz" .TagName }} 46 | --------------------------------------------------------------------------------