├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ ├── golangci_lint.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── LICENSE.md ├── README.md ├── docs └── snapshots.png ├── example-config.yml ├── go.mod ├── go.sum ├── internal ├── cmd │ ├── fetch │ │ └── fetch.go │ ├── mirror │ │ └── mirror.go │ ├── root.go │ ├── sidecar │ │ └── sidecar.go │ └── tracker │ │ └── tracker.go ├── discovery │ ├── consul.go │ └── discovery.go ├── fetch │ ├── fetch.go │ ├── fetch_test.go │ ├── sidecar.go │ ├── sidecar_test.go │ └── tracker.go ├── index │ ├── index.go │ ├── index_test.go │ ├── schema.go │ └── types.go ├── integrationtest │ ├── sidecar_test.go │ └── tracker_test.go ├── ledger │ ├── snapshot.go │ └── snapshot_test.go ├── ledgertest │ └── ledgertest.go ├── logger │ ├── logger.go │ └── logger_test.go ├── mirror │ ├── mirror.go │ └── uploader.go ├── netx │ ├── netx.go │ └── netx_test.go ├── scraper │ ├── collector.go │ ├── manager.go │ ├── prober.go │ └── scraper.go ├── sidecar │ ├── consensus.go │ ├── snapshot.go │ └── snapshot_test.go └── tracker │ └── tracker.go ├── main.go └── types ├── auth.go ├── auth_test.go ├── config.go ├── config_test.go ├── snapshot.go └── snapshot_test.go /.gitattributes: -------------------------------------------------------------------------------- 1 | *.jpg filter=lfs diff=lfs merge=lfs -text 2 | *.webp filter=lfs diff=lfs merge=lfs -text 3 | *.png filter=lfs diff=lfs merge=lfs -text 4 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "daily" 11 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | tags: 6 | - 'v*' 7 | name: Build docker image 8 | jobs: 9 | build-docker: 10 | runs-on: ubuntu-latest 11 | name: Build Docker Image 12 | permissions: 13 | packages: write 14 | steps: 15 | - uses: actions/checkout@v4 16 | 17 | - uses: docker/setup-buildx-action@v3 18 | 19 | - id: meta 20 | uses: docker/metadata-action@v5 21 | with: 22 | images: ghcr.io/blockdaemon/solana-cluster-manager 23 | tags: | 24 | type=sha,event=branch,prefix= 25 | type=raw,value=latest 26 | type=ref,event=tag 27 | 28 | - uses: docker/login-action@v3 29 | with: 30 | registry: ghcr.io 31 | username: ${{ github.actor }} 32 | password: ${{ secrets.GITHUB_TOKEN }} 33 | 34 | - uses: docker/build-push-action@v6 35 | with: 36 | context: . 37 | platforms: linux/amd64,linux/arm64 38 | push: true 39 | build-args: GITHUB_SHA=${{ github.sha }} 40 | tags: ${{ steps.meta.outputs.tags }} 41 | labels: ${{ steps.meta.outputs.labels }} 42 | -------------------------------------------------------------------------------- /.github/workflows/golangci_lint.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: golangci-lint 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | branches: 8 | - main 9 | pull_request: 10 | permissions: 11 | contents: read 12 | jobs: 13 | golangci: 14 | name: lint 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: actions/setup-go@v5 19 | with: 20 | go-version: 'stable' 21 | 22 | - name: golangci-lint 23 | uses: golangci/golangci-lint-action@v8 24 | with: 25 | args: --timeout 5m0s 26 | only-new-issues: true 27 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | on: 2 | - push 3 | - pull_request 4 | name: unit tests 5 | jobs: 6 | unit_tests: 7 | runs-on: ubuntu-22.04 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: actions/setup-go@v5 11 | with: 12 | go-version: 'stable' 13 | 14 | - uses: actions/cache@v4 15 | with: 16 | path: | 17 | ~/go/pkg/mod # Module download cache 18 | ~/.cache/go-build # Build cache (Linux) 19 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 20 | restore-keys: | 21 | ${{ runner.os }}-go- 22 | - name: Test 23 | run: go test ./... -v 24 | - name: Check Format 25 | run: '[ "$(gofmt -l ./ | wc -l)" -eq 0 ]' 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .DS_Store 3 | /cluster-manager 4 | config.yml 5 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: mixed-line-ending 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1.8 2 | 3 | FROM golang:1.24-alpine AS builder 4 | RUN apk add --no-cache build-base git 5 | WORKDIR /app 6 | COPY . . 7 | RUN \ 8 | --mount=type=cache,target=/go/pkg \ 9 | --mount=type=cache,target=/root/.cache/go-build \ 10 | go build -o solcluster . 11 | 12 | # Copy final binary into light stage. 13 | FROM alpine:3 14 | 15 | ARG GITHUB_SHA=local 16 | ENV GITHUB_SHA=${GITHUB_SHA} 17 | 18 | COPY --from=builder /app/solcluster /usr/local/bin/ 19 | 20 | ENV USER=solcluster 21 | ENV UID=13852 22 | ENV GID=13852 23 | RUN addgroup -g "$GID" "$USER" 24 | RUN adduser \ 25 | --disabled-password \ 26 | --gecos "solana" \ 27 | --home "/opt/$USER" \ 28 | --ingroup "$USER" \ 29 | --no-create-home \ 30 | --uid "$UID" \ 31 | "$USER" 32 | RUN chown solcluster /usr/local/bin/solcluster 33 | RUN chmod u+x /usr/local/bin/solcluster 34 | 35 | WORKDIR "/opt/$USER" 36 | USER solcluster 37 | ENTRYPOINT ["/usr/local/bin/solcluster"] 38 | CMD ["sidecar"] 39 | 40 | LABEL org.opencontainers.image.source="https://github.com/Blockdaemon/solana-cluster" 41 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Solana Cluster Manager 2 | 3 | Tooling to manage private clusters of Solana nodes. 4 | 5 | Go Reference 6 | 7 | 8 | 9 | [Issue Tracker](https://github.com/orgs/Blockdaemon/projects/1/views/1) 10 | 11 | 12 | ## Deployment 13 | 14 | ### Building from source 15 | 16 | Requirements: Go 1.18 & build essentials. 17 | 18 | ```shell 19 | go mod download 20 | go build -o ./solana-cluster . 21 | ``` 22 | 23 | ### Docker Image 24 | 25 | Find amd64 and arm64 Docker images for all releases on [GitHub Container Registry](https://github.com/Blockdaemon/solana-cluster/pkgs/container/solana-cluster-manager). 26 | 27 | ```shell 28 | docker pull ghcr.io/blockdaemon/solana-cluster-manager 29 | docker run \ 30 | --network-mode=host \ 31 | -v /solana/ledger:/ledger:ro \ 32 | ghcr.io/blockdaemon/solana-cluster-manager \ 33 | sidecar --ledger /ledger 34 | ``` 35 | 36 | ## Usage 37 | 38 | ``` 39 | $ solana-cluster sidecar --help 40 | 41 | Runs on a Solana node and serves available snapshot archives. 42 | Do not expose this API publicly. 43 | 44 | Usage: 45 | solana-snapshots sidecar [flags] 46 | 47 | Flags: 48 | --interface string Only accept connections from this interface 49 | --ledger string Path to ledger dir 50 | --port uint16 Listen port (default 13080) 51 | ``` 52 | 53 | ``` 54 | $ solana-cluster tracker --help 55 | 56 | Connects to sidecars on nodes and scrapes the available snapshot versions. 57 | Provides an API allowing fetch jobs to find the latest snapshots. 58 | Do not expose this API publicly. 59 | 60 | Usage: 61 | solana-snapshots tracker [flags] 62 | 63 | Flags: 64 | --config string Path to config file 65 | --internal-listen string Internal listen URL (default ":8457") 66 | --listen string Listen URL (default ":8458") 67 | ``` 68 | 69 | ``` 70 | $ solana-cluster fetch --help 71 | 72 | Fetches a snapshot from another node using the tracker API. 73 | 74 | Usage: 75 | solana-snapshots fetch [flags] 76 | 77 | Flags: 78 | --download-timeout duration Max time to try downloading in total (default 10m0s) 79 | --ledger string Path to ledger dir 80 | --max-slots uint Refuse to download slots older than the newest (default 10000) 81 | --min-slots uint Download only snapshots slots newer than local (default 500) 82 | --request-timeout duration Max time to wait for headers (excluding download) (default 3s) 83 | --tracker string Download as instructed by given tracker URL 84 | ``` 85 | 86 | ``` 87 | $ solana-cluster mirror --help 88 | 89 | Periodically mirrors snapshots from nodes to an S3-compatible data store. 90 | Specify credentials via env $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY 91 | 92 | Usage: 93 | solana-snapshots mirror [flags] 94 | 95 | Flags: 96 | --refresh duration Refresh interval to discover new snapshots (default 30s) 97 | --s3-bucket string Bucket name 98 | --s3-prefix string Prefix for S3 object names (optional) 99 | --s3-region string S3 region (optional) 100 | --s3-secure Use secure S3 transport (default true) 101 | --s3-url string URL to S3 API 102 | --tracker string URL to tracker API 103 | ``` 104 | 105 | ## Architecture 106 | 107 | ### Snapshot management 108 | 109 | **[Twitter 🧵](https://twitter.com/terorie_dev/status/1520289936611725312)** 110 | 111 | Snapshot management tooling enables efficient peer-to-peer transfers of accounts database archives. 112 | 113 | ![Snapshot Fetch](./docs/snapshots.png) 114 | 115 | **Scraping** (Flow A) 116 | 117 | Snapshot metadata collection runs periodically similarly to Prometheus scraping. 118 | 119 | Each cluster-aware node runs a lightweight `solana-cluster sidecar` agent providing telemetry about its snapshots. 120 | 121 | The `solana-cluster tracker` then connects to all sidecars to assemble a complete list of snapshot metadata. 122 | The tracker is stateless so it can be replicated. 123 | Service discovery is available through HTTP and JSON files. Consul SD support is planned. 124 | 125 | Side note: Snapshot sources are configurable in stock Solana software but only via static lists. 126 | This does not scale well with large fleets because each cluster change requires updating the lists of all nodes. 127 | 128 | **Downloading** (Flow B) 129 | 130 | When a Solana node needs to fetch a snapshot remotely, the tracker helps it find the best snapshot source. 131 | Nodes will download snapshots directly from the sidecars of other nodes. 132 | 133 | ### TPU & TVU 134 | 135 | Not yet public. 🚜 Subscribe to releases! ✨ 136 | 137 | ## Motivation 138 | 139 | Blockdaemon manages one of the largest Solana validator and RPC infrastructure deployments to date, backed by a custom peer-to-peer backbone. 140 | This repository shares our performance and sustainability optimizations. 141 | 142 | When Solana validators first start, they have to retrieve and validate hundreds of gigabytes of state data from a remote node. 143 | During normal operation, validators stream at least 500 Mbps of traffic in either direction. 144 | 145 | For Solana infra operators that manage more than node (not to mention hundreds), this cost currently scales linearly as well. 146 | Unmodified Solana deployments treat their cluster peers the same as any other. 147 | This can end in a large number of streams between globally dispersed validators. 148 | 149 | This is obviously inefficient. 10 Gbps connectivity is cheap and abundant locally within data centers. 150 | In contrast, major public clouds (who shall not be named) charge egregious premiums on Internet traffic. 151 | 152 | The solution: Co-located Solana validators that are controlled by the same entity should also behave as one entity. 153 | 154 | Leveraging internal connectivity to distribute blockchain data can 155 | reduce public network _leeching_ and increase total cluster bandwidth. 156 | 157 | Authenticated internal connectivity allows delegation of expensive calculations and re-use of results thereof. 158 | Concretely, the amount write-heavy snapshot creation & verification procedures per node can decrease as the cluster scales out. 159 | -------------------------------------------------------------------------------- /docs/snapshots.png: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:67e353f32da97489c9d5638214c5e10315d1c5756bc0ace6def1ce2ccf61e5dd 3 | size 373504 4 | -------------------------------------------------------------------------------- /example-config.yml: -------------------------------------------------------------------------------- 1 | scrape_interval: 15s 2 | 3 | target_groups: 4 | # A group of nodes on the same Solana network. 5 | - group: mainnet 6 | 7 | # ------------------------------------------------ 8 | # Snapshot retrieval 9 | # ------------------------------------------------ 10 | 11 | # URL scheme, use "http" or "https". 12 | scheme: http 13 | 14 | # ------------------------------------------------ 15 | # Discovery 16 | # ------------------------------------------------ 17 | 18 | # Discover targets from a hardcoded set of nodes. 19 | static_targets: 20 | targets: 21 | - solana-mainnet-1.example.org:8899 22 | - solana-mainnet-2.example.org:8899 23 | - solana-mainnet-3.example.org:8899 24 | 25 | # Discover targets from a JSON file. 26 | # 27 | # file_targets: 28 | # path: 29 | 30 | # Discover targets from a HTTP server. 31 | # 32 | # http_targets: 33 | # url: 34 | # basic_auth: <...> 35 | # bearer_auth: <...> 36 | # tls_config: <...> 37 | 38 | # ------------------------------------------------ 39 | # Authentication 40 | # ------------------------------------------------ 41 | 42 | # Set up RFC 7617 Basic Authentication on requests. 43 | # 44 | # basic_auth: 45 | # username: 46 | # password: 47 | 48 | # Set up a Bearer Auth token. 49 | # 50 | # bearer_auth: 51 | # token: 52 | 53 | # Set up TLS config (requires https scheme). 54 | # 55 | # tls_config: 56 | # ca_file: 57 | # cert_file: 58 | # key_file: 59 | # insecure_skip_verify: 60 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.blockdaemon.com/solana/cluster-manager 2 | 3 | go 1.24.0 4 | 5 | require ( 6 | github.com/gagliardetto/solana-go v1.12.0 7 | github.com/gin-contrib/zap v1.1.5 8 | github.com/gin-gonic/gin v1.10.1 9 | github.com/go-resty/resty/v2 v2.16.5 10 | github.com/hashicorp/consul/api v1.32.1 11 | github.com/hashicorp/go-memdb v1.3.5 12 | github.com/minio/minio-go/v7 v7.0.92 13 | github.com/prometheus/client_golang v1.22.0 14 | github.com/spf13/afero v1.14.0 15 | github.com/spf13/cobra v1.9.1 16 | github.com/spf13/pflag v1.0.6 17 | github.com/stretchr/testify v1.10.0 18 | github.com/vbauerster/mpb/v8 v8.10.1 19 | go.uber.org/atomic v1.11.0 20 | go.uber.org/zap v1.27.0 21 | golang.org/x/sync v0.14.0 22 | gopkg.in/yaml.v3 v3.0.1 23 | ) 24 | 25 | require ( 26 | filippo.io/edwards25519 v1.0.0-rc.1 // indirect 27 | github.com/VividCortex/ewma v1.2.0 // indirect 28 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect 29 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect 30 | github.com/armon/go-metrics v0.4.1 // indirect 31 | github.com/beorn7/perks v1.0.1 // indirect 32 | github.com/blendle/zapdriver v1.3.1 // indirect 33 | github.com/buger/jsonparser v1.1.1 // indirect 34 | github.com/bytedance/sonic v1.13.2 // indirect 35 | github.com/bytedance/sonic/loader v0.2.4 // indirect 36 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 37 | github.com/cloudwego/base64x v0.1.5 // indirect 38 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 39 | github.com/dustin/go-humanize v1.0.1 // indirect 40 | github.com/fatih/color v1.16.0 // indirect 41 | github.com/gabriel-vasile/mimetype v1.4.8 // indirect 42 | github.com/gagliardetto/binary v0.8.0 // indirect 43 | github.com/gagliardetto/treeout v0.1.4 // indirect 44 | github.com/gin-contrib/sse v1.0.0 // indirect 45 | github.com/go-ini/ini v1.67.0 // indirect 46 | github.com/go-playground/locales v0.14.1 // indirect 47 | github.com/go-playground/universal-translator v0.18.1 // indirect 48 | github.com/go-playground/validator/v10 v10.26.0 // indirect 49 | github.com/goccy/go-json v0.10.5 // indirect 50 | github.com/google/uuid v1.6.0 // indirect 51 | github.com/gorilla/rpc v1.2.0 // indirect 52 | github.com/gorilla/websocket v1.4.2 // indirect 53 | github.com/hashicorp/errwrap v1.1.0 // indirect 54 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 55 | github.com/hashicorp/go-hclog v1.5.0 // indirect 56 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 57 | github.com/hashicorp/go-multierror v1.1.1 // indirect 58 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 59 | github.com/hashicorp/golang-lru v0.5.4 // indirect 60 | github.com/hashicorp/serf v0.10.1 // indirect 61 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 62 | github.com/json-iterator/go v1.1.12 // indirect 63 | github.com/klauspost/compress v1.18.0 // indirect 64 | github.com/klauspost/cpuid/v2 v2.2.10 // indirect 65 | github.com/leodido/go-urn v1.4.0 // indirect 66 | github.com/logrusorgru/aurora v2.0.3+incompatible // indirect 67 | github.com/mattn/go-colorable v0.1.13 // indirect 68 | github.com/mattn/go-isatty v0.0.20 // indirect 69 | github.com/mattn/go-runewidth v0.0.16 // indirect 70 | github.com/minio/crc64nvme v1.0.1 // indirect 71 | github.com/minio/md5-simd v1.1.2 // indirect 72 | github.com/mitchellh/go-homedir v1.1.0 // indirect 73 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 74 | github.com/mitchellh/mapstructure v1.5.0 // indirect 75 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 76 | github.com/modern-go/reflect2 v1.0.2 // indirect 77 | github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 // indirect 78 | github.com/mr-tron/base58 v1.2.0 // indirect 79 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 80 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 81 | github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect 82 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 83 | github.com/prometheus/client_model v0.6.1 // indirect 84 | github.com/prometheus/common v0.62.0 // indirect 85 | github.com/prometheus/procfs v0.15.1 // indirect 86 | github.com/rivo/uniseg v0.4.7 // indirect 87 | github.com/rs/xid v1.6.0 // indirect 88 | github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect 89 | github.com/tinylib/msgp v1.3.0 // indirect 90 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 91 | github.com/ugorji/go/codec v1.2.12 // indirect 92 | go.mongodb.org/mongo-driver v1.12.2 // indirect 93 | go.uber.org/multierr v1.11.0 // indirect 94 | go.uber.org/ratelimit v0.2.0 // indirect 95 | golang.org/x/arch v0.15.0 // indirect 96 | golang.org/x/crypto v0.36.0 // indirect 97 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect 98 | golang.org/x/net v0.38.0 // indirect 99 | golang.org/x/sys v0.33.0 // indirect 100 | golang.org/x/term v0.30.0 // indirect 101 | golang.org/x/text v0.23.0 // indirect 102 | golang.org/x/time v0.8.0 // indirect 103 | google.golang.org/protobuf v1.36.6 // indirect 104 | ) 105 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= 2 | filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= 3 | github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= 4 | github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= 5 | github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 6 | github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= 7 | github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= 8 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= 9 | github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= 10 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 11 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 12 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 13 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 14 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= 15 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= 16 | github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 17 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 18 | github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= 19 | github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= 20 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 21 | github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 22 | github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 23 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 24 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 25 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 26 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 27 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 28 | github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= 29 | github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= 30 | github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= 31 | github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= 32 | github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= 33 | github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= 34 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 35 | github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= 36 | github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= 37 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 38 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 39 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 40 | github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= 41 | github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= 42 | github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= 43 | github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 44 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 45 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 46 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 47 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 48 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 49 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 50 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 51 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 52 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 53 | github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= 54 | github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= 55 | github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= 56 | github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= 57 | github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= 58 | github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= 59 | github.com/gagliardetto/binary v0.8.0 h1:U9ahc45v9HW0d15LoN++vIXSJyqR/pWw8DDlhd7zvxg= 60 | github.com/gagliardetto/binary v0.8.0/go.mod h1:2tfj51g5o9dnvsc+fL3Jxr22MuWzYXwx9wEoN0XQ7/c= 61 | github.com/gagliardetto/solana-go v1.12.0 h1:rzsbilDPj6p+/DOPXBMLhwMZeBgeRuXjm5zQFCoXgsg= 62 | github.com/gagliardetto/solana-go v1.12.0/go.mod h1:l/qqqIN6qJJPtxW/G1PF4JtcE3Zg2vD2EliZrr9Gn5k= 63 | github.com/gagliardetto/treeout v0.1.4 h1:ozeYerrLCmCubo1TcIjFiOWTTGteOOHND1twdFpgwaw= 64 | github.com/gagliardetto/treeout v0.1.4/go.mod h1:loUefvXTrlRG5rYmJmExNryyBRh8f89VZhmMOyCyqok= 65 | github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= 66 | github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= 67 | github.com/gin-contrib/zap v1.1.5 h1:qKwhWb4DQgPriCl1AHLLob6hav/KUIctKXIjTmWIN3I= 68 | github.com/gin-contrib/zap v1.1.5/go.mod h1:lAchUtGz9M2K6xDr1rwtczyDrThmSx6c9F384T45iOE= 69 | github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= 70 | github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= 71 | github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= 72 | github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 73 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 74 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 75 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 76 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 77 | github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= 78 | github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 79 | github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= 80 | github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= 81 | github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= 82 | github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= 83 | github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= 84 | github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= 85 | github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= 86 | github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= 87 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 88 | github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= 89 | github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 90 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 91 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 92 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 93 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 94 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 95 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 96 | github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= 97 | github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= 98 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 99 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 100 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 101 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 102 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 103 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 104 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 105 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 106 | github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk= 107 | github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ= 108 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 109 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 110 | github.com/hashicorp/consul/api v1.32.1 h1:0+osr/3t/aZNAdJX558crU3PEjVrG4x6715aZHRgceE= 111 | github.com/hashicorp/consul/api v1.32.1/go.mod h1:mXUWLnxftwTmDv4W3lzxYCPD199iNLLUyLfLGFJbtl4= 112 | github.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg= 113 | github.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s= 114 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 115 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 116 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 117 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 118 | github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= 119 | github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= 120 | github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= 121 | github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= 122 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 123 | github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= 124 | github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 125 | github.com/hashicorp/go-memdb v1.3.5 h1:b3taDMxCBCBVgyRrS1AZVHO14ubMYZB++QpNhBg+Nyo= 126 | github.com/hashicorp/go-memdb v1.3.5/go.mod h1:8IVKKBkVe+fxFgdFOYxzQQNjz+sWCyHCdIC/+5+Vy1Y= 127 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 128 | github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= 129 | github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 130 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 131 | github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= 132 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 133 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 134 | github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 135 | github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= 136 | github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= 137 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 138 | github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= 139 | github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= 140 | github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= 141 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 142 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 143 | github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= 144 | github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 145 | github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= 146 | github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 147 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 148 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 149 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 150 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 151 | github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= 152 | github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= 153 | github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= 154 | github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= 155 | github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= 156 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 157 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 158 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 159 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 160 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 161 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 162 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 163 | github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= 164 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 165 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 166 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 167 | github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 168 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 169 | github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= 170 | github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 171 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 172 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 173 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 174 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 175 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 176 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 177 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 178 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 179 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 180 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 181 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 182 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 183 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 184 | github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= 185 | github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= 186 | github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= 187 | github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= 188 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 189 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 190 | github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 191 | github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 192 | github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= 193 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 194 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 195 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 196 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 197 | github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= 198 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 199 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 200 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 201 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 202 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 203 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 204 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 205 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 206 | github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= 207 | github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= 208 | github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= 209 | github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= 210 | github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= 211 | github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= 212 | github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= 213 | github.com/minio/minio-go/v7 v7.0.92 h1:jpBFWyRS3p8P/9tsRc+NuvqoFi7qAmTCFPoRFmobbVw= 214 | github.com/minio/minio-go/v7 v7.0.92/go.mod h1:vTIc8DNcnAZIhyFsk8EB90AbPjj3j68aWIEQCiPj7d0= 215 | github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= 216 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 217 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 218 | github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= 219 | github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= 220 | github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 221 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 222 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 223 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 224 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 225 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 226 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 227 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 228 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 229 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 230 | github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= 231 | github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1 h1:mPMvm6X6tf4w8y7j9YIt6V9jfWhL6QlbEc7CCmeQlWk= 232 | github.com/mostynb/zstdpool-freelist v0.0.0-20201229113212-927304c0c3b1/go.mod h1:ye2e/VUEtE2BHE+G/QcKkcLQVAEJoYRFj5VUOQatCRE= 233 | github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= 234 | github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= 235 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 236 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 237 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 238 | github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= 239 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 240 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 241 | github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= 242 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 243 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 244 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 245 | github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= 246 | github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= 247 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 248 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 249 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 250 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 251 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 252 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 253 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 254 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 255 | github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= 256 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 257 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 258 | github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 259 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 260 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 261 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 262 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 263 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 264 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 265 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 266 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 267 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 268 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 269 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 270 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 271 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 272 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 273 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 274 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 275 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 276 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 277 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 278 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 279 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 280 | github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= 281 | github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 282 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 283 | github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 284 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= 285 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 286 | github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= 287 | github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= 288 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 289 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 290 | github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= 291 | github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= 292 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 293 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 294 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 295 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 296 | github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo= 297 | github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU= 298 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 299 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 300 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 301 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 302 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 303 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 304 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 305 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 306 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 307 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 308 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 309 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 310 | github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= 311 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 312 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 313 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 314 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 315 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 316 | github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= 317 | github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= 318 | github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= 319 | github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= 320 | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= 321 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 322 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 323 | github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= 324 | github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= 325 | github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= 326 | github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= 327 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 328 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 329 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 330 | github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= 331 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 332 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 333 | go.mongodb.org/mongo-driver v1.12.2 h1:gbWY1bJkkmUB9jjZzcdhOL8O85N9H+Vvsf2yFN0RDws= 334 | go.mongodb.org/mongo-driver v1.12.2/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= 335 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 336 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 337 | go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= 338 | go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= 339 | go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= 340 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 341 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 342 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 343 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 344 | go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= 345 | go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= 346 | go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= 347 | go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= 348 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 349 | go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= 350 | go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= 351 | go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= 352 | golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= 353 | golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= 354 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 355 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 356 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= 357 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 358 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 359 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 360 | golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 361 | golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= 362 | golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= 363 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= 364 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= 365 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 366 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 367 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 368 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 369 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 370 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 371 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 372 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 373 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 374 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 375 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 376 | golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= 377 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 378 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 379 | golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= 380 | golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 381 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 382 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 383 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 384 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 385 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 386 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 387 | golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= 388 | golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 389 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 390 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 391 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 392 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 393 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 394 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 395 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 396 | golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 397 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 398 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 399 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 400 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 401 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 402 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 403 | golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 404 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 405 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 406 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 407 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 408 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 409 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 410 | golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 411 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 412 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 413 | golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 414 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 415 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 416 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 417 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 418 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 419 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 420 | golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= 421 | golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= 422 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 423 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 424 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 425 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 426 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 427 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 428 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 429 | golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= 430 | golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= 431 | golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= 432 | golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 433 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 434 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 435 | golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 436 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 437 | golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 438 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 439 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 440 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 441 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 442 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 443 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 444 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 445 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 446 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 447 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 448 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 449 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 450 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 451 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 452 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 453 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 454 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 455 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 456 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 457 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 458 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 459 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 460 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 461 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 462 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 463 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 464 | -------------------------------------------------------------------------------- /internal/cmd/fetch/fetch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fetch provides the `fetch` command. 16 | package fetch 17 | 18 | import ( 19 | "context" 20 | "encoding/json" 21 | "io" 22 | "net/http" 23 | "os" 24 | "os/signal" 25 | "time" 26 | 27 | "github.com/go-resty/resty/v2" 28 | "github.com/spf13/cobra" 29 | "github.com/vbauerster/mpb/v8" 30 | "github.com/vbauerster/mpb/v8/decor" 31 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 32 | "go.blockdaemon.com/solana/cluster-manager/internal/ledger" 33 | "go.blockdaemon.com/solana/cluster-manager/internal/logger" 34 | "go.uber.org/zap" 35 | "golang.org/x/sync/errgroup" 36 | ) 37 | 38 | var Cmd = cobra.Command{ 39 | Use: "fetch", 40 | Short: "Snapshot downloader", 41 | Long: "Fetches a snapshot from another node using the tracker API.", 42 | Run: func(_ *cobra.Command, _ []string) { 43 | run() 44 | }, 45 | } 46 | 47 | var ( 48 | ledgerDir string 49 | trackerURL string 50 | minSnapAge uint64 51 | maxSnapAge uint64 52 | requestTimeout time.Duration 53 | downloadTimeout time.Duration 54 | ) 55 | 56 | func init() { 57 | flags := Cmd.Flags() 58 | flags.StringVar(&ledgerDir, "ledger", "", "Path to ledger dir") 59 | flags.StringVar(&trackerURL, "tracker", "", "Download as instructed by given tracker URL") 60 | flags.Uint64Var(&minSnapAge, "min-slots", 500, "Download only snapshots slots newer than local") 61 | flags.Uint64Var(&maxSnapAge, "max-slots", 10000, "Refuse to download slots older than the newest") 62 | flags.DurationVar(&requestTimeout, "request-timeout", 3*time.Second, "Max time to wait for headers (excluding download)") 63 | flags.DurationVar(&downloadTimeout, "download-timeout", 10*time.Minute, "Max time to try downloading in total") 64 | } 65 | 66 | func run() { 67 | log := logger.GetConsoleLogger() 68 | 69 | // Regardless which API we talk to, we want to cap time from request to response header. 70 | // This defends against black holes and really slow servers. 71 | // Download time (reading response body) is not affected. 72 | http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = requestTimeout 73 | 74 | // Run until interrupted or time out occurs. 75 | ctx := context.Background() 76 | ctx, cancel := signal.NotifyContext(ctx, os.Interrupt) 77 | defer cancel() 78 | ctx, cancel2 := context.WithTimeout(ctx, downloadTimeout) 79 | defer cancel2() 80 | 81 | // Check what snapshots we have locally. 82 | localSnaps, err := ledger.ListSnapshots(os.DirFS(ledgerDir)) 83 | if err != nil { 84 | log.Fatal("Failed to check existing snapshots", zap.Error(err)) 85 | } 86 | 87 | // Ask tracker for best snapshots. 88 | trackerClient := fetch.NewTrackerClientWithResty( 89 | resty.New(). 90 | SetHostURL(trackerURL). 91 | SetTimeout(requestTimeout), 92 | ) 93 | remoteSnaps, err := trackerClient.GetBestSnapshots(ctx, -1) 94 | if err != nil { 95 | log.Fatal("Failed to request snapshot info", zap.Error(err)) 96 | } 97 | 98 | // Decide what we want to do. 99 | _, advice := fetch.ShouldFetchSnapshot(localSnaps, remoteSnaps, minSnapAge, maxSnapAge) 100 | switch advice { 101 | case fetch.AdviceNothingFound: 102 | log.Error("No snapshots available remotely") 103 | return 104 | case fetch.AdviceUpToDate: 105 | log.Info("Existing snapshot is recent enough, no download needed", 106 | zap.Uint64("existing_slot", localSnaps[0].Slot)) 107 | return 108 | case fetch.AdviceFetch: 109 | } 110 | 111 | // Print snapshot to user. 112 | snap := &remoteSnaps[0] 113 | buf, _ := json.MarshalIndent(snap, "", "\t") 114 | log.Info("Downloading a snapshot", zap.ByteString("snap", buf)) 115 | 116 | // Setup progress bars for download. 117 | bars := mpb.New() 118 | sidecarClient := fetch.NewSidecarClientWithOpts(snap.Target, fetch.SidecarClientOpts{ 119 | ProxyReaderFunc: func(name string, size int64, rd io.Reader) io.ReadCloser { 120 | bar := bars.New( 121 | size, 122 | mpb.BarStyle(), 123 | mpb.PrependDecorators(decor.Name(name)), 124 | mpb.AppendDecorators( 125 | decor.AverageSpeed(decor.SizeB1024(0), "% .1f"), 126 | decor.Percentage(), 127 | ), 128 | ) 129 | return bar.ProxyReader(rd) 130 | }, 131 | }) 132 | 133 | // Download. 134 | beforeDownload := time.Now() 135 | group, ctx := errgroup.WithContext(ctx) 136 | for _, file := range snap.Files { 137 | file_ := file 138 | group.Go(func() error { 139 | err := sidecarClient.DownloadSnapshotFile(ctx, ".", file_.FileName) 140 | if err != nil { 141 | log.Error("Download failed", 142 | zap.String("snapshot", file_.FileName)) 143 | } 144 | return err 145 | }) 146 | } 147 | downloadErr := group.Wait() 148 | downloadDuration := time.Since(beforeDownload) 149 | 150 | if downloadErr == nil { 151 | log.Info("Download completed", zap.Duration("download_time", downloadDuration)) 152 | } else { 153 | log.Info("Aborting download", zap.Duration("download_time", downloadDuration)) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /internal/cmd/mirror/mirror.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package mirror provides the `mirror` command. 16 | package mirror 17 | 18 | import ( 19 | "context" 20 | "net/url" 21 | "time" 22 | 23 | "github.com/minio/minio-go/v7" 24 | "github.com/minio/minio-go/v7/pkg/credentials" 25 | "github.com/spf13/cobra" 26 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 27 | "go.blockdaemon.com/solana/cluster-manager/internal/logger" 28 | "go.blockdaemon.com/solana/cluster-manager/internal/mirror" 29 | "go.uber.org/zap" 30 | ) 31 | 32 | var Cmd = cobra.Command{ 33 | Use: "mirror", 34 | Short: "Daemon to periodically upload snapshots to S3", 35 | Long: "Periodically mirrors snapshots from nodes to an S3-compatible data store.\n" + 36 | "Specify credentials via env $AWS_ACCESS_KEY_ID and $AWS_SECRET_ACCESS_KEY", 37 | Run: func(cmd *cobra.Command, _ []string) { 38 | run(cmd) 39 | }, 40 | } 41 | 42 | var ( 43 | refreshInterval time.Duration 44 | trackerURL string 45 | s3URL string 46 | s3Bucket string 47 | objectPrefix string 48 | s3Region string 49 | ) 50 | 51 | func init() { 52 | flags := Cmd.Flags() 53 | flags.DurationVar(&refreshInterval, "refresh", 30*time.Second, "Refresh interval to discover new snapshots") 54 | flags.StringVar(&trackerURL, "tracker", "http://localhost:8458", "URL to tracker API") 55 | flags.StringVar(&s3URL, "s3-url", "", "URL to S3 API") 56 | flags.StringVar(&s3Region, "s3-region", "", "S3 region (optional)") 57 | flags.StringVar(&s3Bucket, "s3-bucket", "", "Bucket name") 58 | flags.StringVar(&objectPrefix, "s3-prefix", "", "Prefix for S3 object names (optional)") 59 | flags.AddFlagSet(logger.Flags) 60 | } 61 | 62 | func run(cmd *cobra.Command) { 63 | log := logger.GetLogger() 64 | _ = log 65 | 66 | if trackerURL == "" || s3URL == "" || s3Bucket == "" { 67 | cobra.CheckErr(cmd.Usage()) 68 | cobra.CheckErr("required argument missing") 69 | } 70 | 71 | trackerClient := fetch.NewTrackerClient(trackerURL) 72 | 73 | parsedS3URL, err := url.Parse(s3URL) 74 | cobra.CheckErr(err) 75 | s3Secure := parsedS3URL.Scheme != "http" 76 | 77 | s3Client, err := minio.New(parsedS3URL.Host, &minio.Options{ 78 | Creds: credentials.NewEnvAWS(), 79 | Secure: s3Secure, 80 | Region: s3Region, 81 | }) 82 | if err != nil { 83 | log.Fatal("Failed to connect to S3", zap.Error(err)) 84 | } 85 | 86 | uploader := mirror.Uploader{ 87 | S3Client: s3Client, 88 | Bucket: s3Bucket, 89 | ObjectPrefix: objectPrefix, 90 | } 91 | 92 | worker := mirror.Worker{ 93 | Tracker: trackerClient, 94 | Uploader: &uploader, 95 | Log: log.Named("uploader"), 96 | Refresh: refreshInterval, 97 | SyncCount: 10, // TODO 98 | } 99 | worker.Run(context.TODO()) 100 | } 101 | -------------------------------------------------------------------------------- /internal/cmd/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package cmd 16 | 17 | import ( 18 | "github.com/spf13/cobra" 19 | "go.blockdaemon.com/solana/cluster-manager/internal/cmd/fetch" 20 | "go.blockdaemon.com/solana/cluster-manager/internal/cmd/mirror" 21 | "go.blockdaemon.com/solana/cluster-manager/internal/cmd/sidecar" 22 | "go.blockdaemon.com/solana/cluster-manager/internal/cmd/tracker" 23 | ) 24 | 25 | var Cmd = cobra.Command{ 26 | Use: "solana-snapshots", 27 | CompletionOptions: cobra.CompletionOptions{ 28 | DisableDefaultCmd: true, 29 | }, 30 | } 31 | 32 | func init() { 33 | Cmd.AddCommand( 34 | &fetch.Cmd, 35 | &sidecar.Cmd, 36 | &tracker.Cmd, 37 | &mirror.Cmd, 38 | ) 39 | } 40 | -------------------------------------------------------------------------------- /internal/cmd/sidecar/sidecar.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package sidecar provides the `sidecar` command. 16 | package sidecar 17 | 18 | import ( 19 | "time" 20 | 21 | ginzap "github.com/gin-contrib/zap" 22 | "github.com/gin-gonic/gin" 23 | "github.com/spf13/cobra" 24 | "go.blockdaemon.com/solana/cluster-manager/internal/logger" 25 | "go.blockdaemon.com/solana/cluster-manager/internal/netx" 26 | "go.blockdaemon.com/solana/cluster-manager/internal/sidecar" 27 | "go.uber.org/zap" 28 | ) 29 | 30 | var Cmd = cobra.Command{ 31 | Use: "sidecar", 32 | Short: "Snapshot node sidecar", 33 | Long: "Runs on a Solana node and serves available snapshot archives.\n" + 34 | "Do not expose this API publicly.", 35 | Run: func(_ *cobra.Command, _ []string) { 36 | run() 37 | }, 38 | } 39 | 40 | var ( 41 | netInterface string 42 | listenPort uint16 43 | ledgerDir string 44 | rpcWsUrl string 45 | ) 46 | 47 | func init() { 48 | flags := Cmd.Flags() 49 | flags.StringVar(&netInterface, "interface", "", "Only accept connections from this interface") 50 | flags.Uint16Var(&listenPort, "port", 13080, "Listen port") 51 | flags.StringVar(&ledgerDir, "ledger", "", "Path to ledger dir") 52 | flags.StringVar(&rpcWsUrl, "ws", "ws://localhost:8900", "Solana RPC PubSub WebSocket endpoint") 53 | flags.AddFlagSet(logger.Flags) 54 | } 55 | 56 | func run() { 57 | log := logger.GetLogger() 58 | listener, listenAddrs, err := netx.ListenTCPInterface("tcp", netInterface, listenPort) 59 | if err != nil { 60 | cobra.CheckErr(err) 61 | } 62 | for _, addr := range listenAddrs { 63 | log.Info("Listening for conns", zap.Stringer("addr", &addr)) 64 | } 65 | 66 | gin.SetMode(gin.ReleaseMode) 67 | server := gin.New() 68 | httpLog := log.Named("http") 69 | server.Use(ginzap.Ginzap(httpLog, time.RFC3339, true)) 70 | server.Use(ginzap.RecoveryWithZap(httpLog, false)) 71 | 72 | groupV1 := server.Group("/v1") 73 | 74 | snapshotHandler := sidecar.NewSnapshotHandler(ledgerDir, httpLog) 75 | snapshotHandler.RegisterHandlers(groupV1) 76 | 77 | consensusHandler := sidecar.NewConsensusHandler(rpcWsUrl, httpLog) 78 | consensusHandler.RegisterHandlers(groupV1) 79 | 80 | err = server.RunListener(listener) 81 | log.Error("Server stopped", zap.Error(err)) 82 | } 83 | -------------------------------------------------------------------------------- /internal/cmd/tracker/tracker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package tracker provides the `tracker` command. 16 | package tracker 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "net/http" 22 | "os" 23 | "os/signal" 24 | "syscall" 25 | "time" 26 | 27 | ginzap "github.com/gin-contrib/zap" 28 | "github.com/gin-gonic/gin" 29 | "github.com/prometheus/client_golang/prometheus" 30 | "github.com/prometheus/client_golang/prometheus/promhttp" 31 | "github.com/spf13/cobra" 32 | "go.blockdaemon.com/solana/cluster-manager/internal/index" 33 | "go.blockdaemon.com/solana/cluster-manager/internal/logger" 34 | "go.blockdaemon.com/solana/cluster-manager/internal/scraper" 35 | "go.blockdaemon.com/solana/cluster-manager/internal/tracker" 36 | "go.blockdaemon.com/solana/cluster-manager/types" 37 | "go.uber.org/zap" 38 | "golang.org/x/sync/errgroup" 39 | ) 40 | 41 | var Cmd = cobra.Command{ 42 | Use: "tracker", 43 | Short: "Snapshot tracker server", 44 | Long: "Connects to sidecars on nodes and scrapes the available snapshot versions.\n" + 45 | "Provides an API allowing fetch jobs to find the latest snapshots.\n" + 46 | "Do not expose this API publicly.", 47 | Run: func(_ *cobra.Command, _ []string) { 48 | run() 49 | }, 50 | } 51 | 52 | var ( 53 | configPath string 54 | internalListen string 55 | listen string 56 | ) 57 | 58 | func init() { 59 | flags := Cmd.Flags() 60 | flags.StringVar(&configPath, "config", "", "Path to config file") 61 | flags.StringVar(&internalListen, "internal-listen", ":8457", "Internal listen URL") 62 | flags.StringVar(&listen, "listen", ":8458", "Listen URL") 63 | flags.AddFlagSet(logger.Flags) 64 | } 65 | 66 | func run() { 67 | log := logger.GetLogger() 68 | 69 | // Install signal handlers. 70 | onReload := make(chan os.Signal, 1) 71 | signal.Notify(onReload, syscall.SIGHUP) 72 | ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) 73 | defer cancel() 74 | 75 | // Install HTTP handlers. 76 | http.HandleFunc("/reload", func(wr http.ResponseWriter, req *http.Request) { 77 | if req.Method != http.MethodPost { 78 | http.Error(wr, "method not allowed", http.StatusMethodNotAllowed) 79 | return 80 | } 81 | onReload <- syscall.SIGHUP 82 | http.Error(wr, "reloaded", http.StatusOK) 83 | }) 84 | httpErrLog, err := zap.NewStdLogAt(log.Named("prometheus"), zap.ErrorLevel) 85 | if err != nil { 86 | panic(err.Error()) 87 | } 88 | http.Handle("/metrics", promhttp.HandlerFor( 89 | prometheus.DefaultGatherer, 90 | promhttp.HandlerOpts{ 91 | ErrorLog: httpErrLog, 92 | }, 93 | )) 94 | 95 | // Create result collector. 96 | db := index.NewDB() 97 | collector := scraper.NewCollector(db) 98 | collector.Log = log.Named("collector") 99 | collector.Start() 100 | defer collector.Close() 101 | 102 | gin.SetMode(gin.ReleaseMode) 103 | server := gin.New() 104 | httpLog := log.Named("http") 105 | server.Use(ginzap.Ginzap(httpLog, time.RFC3339, true)) 106 | server.Use(ginzap.RecoveryWithZap(httpLog, false)) 107 | 108 | handler := tracker.NewHandler(db) 109 | handler.RegisterHandlers(server.Group("/v1")) 110 | 111 | // Start services. 112 | group, ctx := errgroup.WithContext(ctx) 113 | if internalListen != "" { 114 | httpLog.Info("Starting internal server", zap.String("listen", internalListen)) 115 | } 116 | runGroupServer(ctx, group, internalListen, nil) // default handler 117 | httpLog.Info("Starting server", zap.String("listen", listen)) 118 | runGroupServer(ctx, group, listen, server) // public handler 119 | 120 | // Create config reloader. 121 | config, err := types.LoadConfig(configPath) 122 | if err != nil { 123 | log.Fatal("Failed to load config", zap.Error(err)) 124 | } 125 | 126 | // Create scrape managers. 127 | manager := scraper.NewManager(collector.Probes()) 128 | manager.Log = log.Named("scraper") 129 | manager.Update(config) 130 | 131 | // TODO Config reloading 132 | 133 | // Wait until crash or graceful exit. 134 | if err := group.Wait(); err != nil { 135 | log.Error("Crashed", zap.Error(err)) 136 | } else { 137 | log.Info("Shutting down") 138 | } 139 | } 140 | 141 | func runGroupServer(ctx context.Context, group *errgroup.Group, listen string, handler http.Handler) { 142 | group.Go(func() error { 143 | server := http.Server{ 144 | Addr: listen, 145 | Handler: handler, 146 | } 147 | go func() { 148 | <-ctx.Done() 149 | _ = server.Close() 150 | }() 151 | if err := server.ListenAndServe(); errors.Is(err, http.ErrServerClosed) { 152 | return nil 153 | } else { 154 | return err 155 | } 156 | }) 157 | } 158 | -------------------------------------------------------------------------------- /internal/discovery/consul.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package discovery 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | 21 | "github.com/hashicorp/consul/api" 22 | "go.blockdaemon.com/solana/cluster-manager/types" 23 | ) 24 | 25 | // Consul service discovery backend 26 | type Consul struct { 27 | Client *api.Client 28 | Service string 29 | 30 | Datacenter string // Consul datacenter (dc param) 31 | Filter string // Consul filter expression (filter param) 32 | } 33 | 34 | // NewConsulFromConfig invokes NewConsul using typed config. 35 | func NewConsulFromConfig(config *types.ConsulSDConfig) (*Consul, error) { 36 | client, err := api.NewClient(&api.Config{ 37 | Address: config.Server, 38 | Token: config.Token, 39 | TokenFile: config.TokenFile, 40 | }) 41 | if err != nil { 42 | return nil, err 43 | } 44 | sd := NewConsul(client, config.Service) 45 | sd.Datacenter = config.Datacenter 46 | sd.Filter = config.Filter 47 | return sd, nil 48 | } 49 | 50 | // NewConsul creates a new service discovery provider for Solana cluster 51 | func NewConsul(client *api.Client, service string) *Consul { 52 | return &Consul{ 53 | Client: client, 54 | Service: service, 55 | } 56 | } 57 | 58 | // DiscoverTargets queries Consul Catalog API to find nodes. 59 | // Returns a list of targets referred to by IP addresses. 60 | func (c *Consul) DiscoverTargets(ctx context.Context) ([]string, error) { 61 | services, _, err := c.Client.Catalog().Service(c.Service, "", (&api.QueryOptions{ 62 | Datacenter: c.Datacenter, 63 | Filter: c.Filter, 64 | }).WithContext(ctx)) 65 | if err != nil { 66 | return nil, err 67 | } 68 | targets := make([]string, 0, len(services)) 69 | for _, service := range services { 70 | targets = append(targets, fmt.Sprintf("%s:%d", service.Address, service.ServicePort)) 71 | } 72 | return targets, nil 73 | } 74 | -------------------------------------------------------------------------------- /internal/discovery/discovery.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package discovery provides service discovery basics. 16 | package discovery 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | 22 | "go.blockdaemon.com/solana/cluster-manager/types" 23 | ) 24 | 25 | // Discoverer returns a list of host:port combinations for all targets. 26 | type Discoverer interface { 27 | DiscoverTargets(ctx context.Context) ([]string, error) 28 | } 29 | 30 | // Simple backends can be found in ../types/config.go 31 | 32 | // NewFromConfig attempts to create a discoverer from config. 33 | func NewFromConfig(t *types.TargetGroup) (Discoverer, error) { 34 | if t.StaticTargets != nil { 35 | return t.StaticTargets, nil 36 | } 37 | if t.FileTargets != nil { 38 | return t.FileTargets, nil 39 | } 40 | if t.ConsulSDConfig != nil { 41 | return NewConsulFromConfig(t.ConsulSDConfig) 42 | } 43 | return nil, fmt.Errorf("missing config") 44 | } 45 | -------------------------------------------------------------------------------- /internal/fetch/fetch.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fetch 16 | 17 | import "go.blockdaemon.com/solana/cluster-manager/types" 18 | 19 | // ShouldFetchSnapshot returns whether a new snapshot should be fetched. 20 | // 21 | // If advice is AdviceFetch, `minSlot` indicates the lowest slot number at which fetch is useful. 22 | func ShouldFetchSnapshot( 23 | local []*types.SnapshotInfo, 24 | remote []types.SnapshotSource, 25 | minAge uint64, // if diff between remote and local is smaller than minAge, use local 26 | maxAge uint64, // if diff between latest remote and any other remote is larger than maxAge, abort 27 | ) (minSlot uint64, advice Advice) { 28 | // Check if remote reports to snapshots. 29 | if len(remote) == 0 { 30 | advice = AdviceNothingFound 31 | return 32 | } 33 | 34 | // Compare local and remote slot numbers. 35 | remoteSlot := remote[0].Slot 36 | var localSlot uint64 37 | if len(local) > 0 { 38 | localSlot = local[0].Slot 39 | } 40 | 41 | // Check if local is newer or remote is not new enough to be interesting. 42 | if int64(remoteSlot)-int64(localSlot) < int64(minAge) { 43 | advice = AdviceUpToDate 44 | return 45 | } 46 | 47 | // Remote is new enough. 48 | if maxAge < remoteSlot { 49 | minSlot = remoteSlot - maxAge 50 | } 51 | advice = AdviceFetch 52 | return 53 | } 54 | 55 | // Advice indicates the recommended next action. 56 | type Advice int 57 | 58 | const ( 59 | AdviceFetch = Advice(iota) // download a snapshot 60 | AdviceNothingFound // no snapshot available 61 | AdviceUpToDate // local snapshot is up-to-date or newer, don't download 62 | ) 63 | -------------------------------------------------------------------------------- /internal/fetch/fetch_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package fetch 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/stretchr/testify/assert" 21 | "go.blockdaemon.com/solana/cluster-manager/types" 22 | ) 23 | 24 | func TestShouldFetchSnapshot(t *testing.T) { 25 | cases := []struct { 26 | name string 27 | 28 | local []uint64 29 | remote []uint64 30 | minAge uint64 31 | maxAge uint64 32 | 33 | minSlot uint64 34 | advice Advice 35 | }{ 36 | { 37 | name: "NothingRemote", 38 | local: []uint64{}, 39 | remote: []uint64{}, 40 | minAge: 500, 41 | maxAge: 10000, 42 | minSlot: 0, 43 | advice: AdviceNothingFound, 44 | }, 45 | { 46 | name: "NothingLocal", 47 | local: []uint64{}, 48 | remote: []uint64{123456}, 49 | minAge: 500, 50 | maxAge: 10000, 51 | minSlot: 113456, 52 | advice: AdviceFetch, 53 | }, 54 | { 55 | name: "LowSlotNumber", 56 | local: []uint64{}, 57 | remote: []uint64{100}, 58 | minAge: 50, 59 | maxAge: 10000, 60 | minSlot: 0, 61 | advice: AdviceFetch, 62 | }, 63 | { 64 | name: "Refresh", 65 | local: []uint64{100000}, 66 | remote: []uint64{123456}, 67 | minAge: 500, 68 | maxAge: 10000, 69 | minSlot: 113456, 70 | advice: AdviceFetch, 71 | }, 72 | { 73 | name: "NotNewEnough", 74 | local: []uint64{100000}, 75 | remote: []uint64{100002}, 76 | minAge: 500, 77 | maxAge: 10000, 78 | minSlot: 0, 79 | advice: AdviceUpToDate, 80 | }, 81 | { 82 | name: "UpToDate", 83 | local: []uint64{223456}, 84 | remote: []uint64{123456}, 85 | minAge: 500, 86 | maxAge: 10000, 87 | minSlot: 0, 88 | advice: AdviceUpToDate, 89 | }, 90 | } 91 | 92 | for _, tc := range cases { 93 | t.Run(tc.name, func(t *testing.T) { 94 | minSlot, advice := ShouldFetchSnapshot( 95 | fakeSnapshotInfo(tc.local), 96 | fakeSnapshotSources(tc.remote), 97 | tc.minAge, 98 | tc.maxAge, 99 | ) 100 | assert.Equal(t, tc.minSlot, minSlot, "different minSlot") 101 | assert.Equal(t, tc.advice, advice, "different advice") 102 | }) 103 | } 104 | } 105 | 106 | func fakeSnapshotInfo(slots []uint64) []*types.SnapshotInfo { 107 | infos := make([]*types.SnapshotInfo, len(slots)) 108 | for i, slot := range slots { 109 | infos[i] = &types.SnapshotInfo{ 110 | Slot: slot, 111 | } 112 | } 113 | return infos 114 | } 115 | 116 | func fakeSnapshotSources(slots []uint64) []types.SnapshotSource { 117 | infos := make([]types.SnapshotSource, len(slots)) 118 | for i, slot := range slots { 119 | infos[i] = types.SnapshotSource{ 120 | SnapshotInfo: types.SnapshotInfo{ 121 | Slot: slot, 122 | }, 123 | } 124 | } 125 | return infos 126 | } 127 | -------------------------------------------------------------------------------- /internal/fetch/sidecar.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fetch contains a sidecar API client. 16 | package fetch 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "io" 22 | "net/http" 23 | "net/url" 24 | "os" 25 | "path/filepath" 26 | "time" 27 | 28 | "github.com/go-resty/resty/v2" 29 | "go.blockdaemon.com/solana/cluster-manager/types" 30 | "go.uber.org/zap" 31 | ) 32 | 33 | // TODO rewrite this package to use OpenAPI code gen 34 | 35 | // SidecarClient accesses the sidecar API. 36 | type SidecarClient struct { 37 | resty *resty.Client 38 | log *zap.Logger 39 | proxyReaderFunc ProxyReaderFunc 40 | } 41 | 42 | type SidecarClientOpts struct { 43 | Resty *resty.Client 44 | Log *zap.Logger 45 | ProxyReaderFunc ProxyReaderFunc 46 | } 47 | 48 | type ProxyReaderFunc func(name string, size int64, rd io.Reader) io.ReadCloser 49 | 50 | func NewSidecarClient(sidecarURL string) *SidecarClient { 51 | return NewSidecarClientWithOpts(sidecarURL, SidecarClientOpts{}) 52 | } 53 | 54 | func NewSidecarClientWithOpts(sidecarURL string, opts SidecarClientOpts) *SidecarClient { 55 | if opts.Resty == nil { 56 | opts.Resty = resty.New() 57 | } 58 | opts.Resty.SetHostURL(sidecarURL) 59 | if opts.ProxyReaderFunc == nil { 60 | opts.ProxyReaderFunc = func(_ string, _ int64, rd io.Reader) io.ReadCloser { 61 | return io.NopCloser(rd) 62 | } 63 | } 64 | if opts.Log == nil { 65 | opts.Log = zap.NewNop() 66 | } 67 | return &SidecarClient{ 68 | resty: opts.Resty, 69 | log: opts.Log, 70 | proxyReaderFunc: opts.ProxyReaderFunc, 71 | } 72 | } 73 | 74 | func (c *SidecarClient) ListSnapshots(ctx context.Context) (infos []*types.SnapshotInfo, err error) { 75 | res, err := c.resty.R(). 76 | SetContext(ctx). 77 | SetHeader("accept", "application/json"). 78 | SetResult(&infos). 79 | Get("/v1/snapshots") 80 | if err != nil { 81 | return nil, err 82 | } 83 | if err := expectOK(res.RawResponse, "list snapshots"); err != nil { 84 | return nil, err 85 | } 86 | return 87 | } 88 | 89 | // StreamSnapshot starts a download of a snapshot file. 90 | // The returned response is guaranteed to have a valid ContentLength. 91 | // The caller has the responsibility to close the response body even if the error is not nil. 92 | func (c *SidecarClient) StreamSnapshot(ctx context.Context, name string) (res *http.Response, err error) { 93 | snapURL := c.resty.HostURL + "/v1/snapshot/" + url.PathEscape(name) 94 | c.log.Debug("Downloading snapshot", zap.String("snapshot_url", snapURL)) 95 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, snapURL, nil) 96 | if err != nil { 97 | return nil, err 98 | } 99 | res, err = c.resty.GetClient().Do(req) 100 | if err != nil { 101 | return 102 | } 103 | if err = expectOK(res, "download snapshot"); err != nil { 104 | return 105 | } 106 | if res.ContentLength < 0 { 107 | err = fmt.Errorf("content length unknown") 108 | } 109 | return 110 | } 111 | 112 | // DownloadSnapshotFile downloads a snapshot to a file in the local file system. 113 | func (c *SidecarClient) DownloadSnapshotFile(ctx context.Context, destDir string, name string) error { 114 | res, err := c.StreamSnapshot(ctx, name) 115 | if res != nil { 116 | defer res.Body.Close() 117 | } 118 | if err != nil { 119 | return err 120 | } 121 | 122 | // Open temporary file. (Consider using O_TMPFILE) 123 | f, err := os.Create(filepath.Join(destDir, ".tmp."+name)) 124 | if err != nil { 125 | return err 126 | } 127 | defer f.Close() 128 | 129 | // Download 130 | proxyRd := c.proxyReaderFunc(name, res.ContentLength, res.Body) 131 | _, err = io.Copy(f, proxyRd) 132 | if err != nil { 133 | _ = proxyRd.Close() 134 | return fmt.Errorf("download failed: %w", err) 135 | } 136 | _ = proxyRd.Close() 137 | 138 | // Promote temporary file. 139 | destPath := filepath.Join(destDir, name) 140 | err = os.Rename(f.Name(), destPath) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | // Change modification time to what server said. 146 | modTime, err := time.Parse(http.TimeFormat, res.Header.Get("last-modified")) 147 | if err == nil && !modTime.IsZero() { 148 | _ = os.Chtimes(destPath, time.Now(), modTime) 149 | } 150 | 151 | return nil 152 | } 153 | 154 | func expectOK(res *http.Response, op string) error { 155 | if res.StatusCode != http.StatusOK { 156 | return fmt.Errorf("%s: %s", op, res.Status) 157 | } 158 | return nil 159 | } 160 | -------------------------------------------------------------------------------- /internal/fetch/sidecar_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "math" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "path/filepath" 12 | "testing" 13 | "time" 14 | 15 | "github.com/go-resty/resty/v2" 16 | "github.com/stretchr/testify/assert" 17 | "github.com/stretchr/testify/require" 18 | "go.uber.org/atomic" 19 | ) 20 | 21 | func TestConnectError(t *testing.T) { 22 | client := NewSidecarClient("invalid://e") 23 | 24 | _, err := client.ListSnapshots(context.TODO()) 25 | assert.Error(t, err) 26 | 27 | err = client.DownloadSnapshotFile(context.TODO(), "/nonexistent9", "snap") 28 | assert.EqualError(t, err, "Get \"invalid://e/v1/snapshot/snap\": unsupported protocol scheme \"invalid\"") 29 | } 30 | 31 | func TestInternalServerError(t *testing.T) { 32 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 33 | w.WriteHeader(http.StatusInternalServerError) 34 | })) 35 | defer server.Close() 36 | 37 | client := NewSidecarClientWithOpts(server.URL, SidecarClientOpts{Resty: resty.NewWithClient(server.Client())}) 38 | 39 | _, err := client.ListSnapshots(context.TODO()) 40 | assert.EqualError(t, err, "list snapshots: 500 Internal Server Error") 41 | 42 | err = client.DownloadSnapshotFile(context.TODO(), "/nonexistent3", "bla") 43 | assert.EqualError(t, err, "download snapshot: 500 Internal Server Error") 44 | } 45 | 46 | func TestSidecarClient_DownloadSnapshotFile(t *testing.T) { 47 | const snapshotName = "bla.tar.zst" 48 | const size = 100 49 | 50 | // Start server 51 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 52 | assert.Equal(t, "/v1/snapshot/bla.tar.zst", r.URL.Path) 53 | w.Header().Set("content-length", "100") 54 | w.Header().Set("last-modified", "Wed, 01 Jan 2020 01:01:01 GMT") 55 | w.WriteHeader(http.StatusOK) 56 | _, _ = w.Write(bytes.Repeat([]byte{'A'}, size)) 57 | })) 58 | defer server.Close() 59 | 60 | // Create client 61 | var proxyReader atomic.Value 62 | client := NewSidecarClientWithOpts(server.URL, SidecarClientOpts{ 63 | Resty: resty.NewWithClient(server.Client()), 64 | ProxyReaderFunc: func(name string, size_ int64, rd io.Reader) io.ReadCloser { 65 | assert.Equal(t, snapshotName, name) 66 | assert.Equal(t, int64(size), size_) 67 | proxy := &mockReadCloser{rd: rd} 68 | assert.True(t, proxyReader.CompareAndSwap(nil, proxy)) 69 | return proxy 70 | }, 71 | }) 72 | 73 | // Temp dir 74 | tmpDir, err := os.MkdirTemp("", "download_test") 75 | require.NoError(t, err) 76 | defer os.RemoveAll(tmpDir) 77 | 78 | // Download snapshot to temp dir 79 | err = client.DownloadSnapshotFile(context.TODO(), tmpDir, snapshotName) 80 | require.NoError(t, err) 81 | 82 | // Ensure proxy was closed exactly once. 83 | assert.Equal(t, proxyReader.Load().(*mockReadCloser).closes.Load(), int32(1)) 84 | 85 | // Make sure file details check out 86 | stat, err := os.Stat(filepath.Join(tmpDir, snapshotName)) 87 | require.NoError(t, err) 88 | assert.Equal(t, int64(size), stat.Size()) 89 | var modTime = time.Date(2020, 1, 1, 1, 1, 1, 0, time.UTC) 90 | assert.Less(t, math.Abs(modTime.Sub(stat.ModTime()).Seconds()), float64(2), "different mod times") 91 | } 92 | 93 | type mockReadCloser struct { 94 | rd io.Reader 95 | closes atomic.Int32 96 | } 97 | 98 | func (m *mockReadCloser) Read(p []byte) (int, error) { 99 | return m.rd.Read(p) 100 | } 101 | 102 | func (m *mockReadCloser) Close() error { 103 | m.closes.Add(1) 104 | return nil 105 | } 106 | -------------------------------------------------------------------------------- /internal/fetch/tracker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package fetch contains a tracker API client. 16 | package fetch 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "net/http" 22 | "strconv" 23 | 24 | "github.com/go-resty/resty/v2" 25 | "go.blockdaemon.com/solana/cluster-manager/types" 26 | ) 27 | 28 | // TrackerClient accesses the tracker API. 29 | type TrackerClient struct { 30 | resty *resty.Client 31 | } 32 | 33 | func NewTrackerClient(trackerURL string) *TrackerClient { 34 | return NewTrackerClientWithResty(resty.New().SetHostURL(trackerURL)) 35 | } 36 | 37 | func NewTrackerClientWithResty(client *resty.Client) *TrackerClient { 38 | return &TrackerClient{resty: client} 39 | } 40 | 41 | func (c *TrackerClient) GetBestSnapshots(ctx context.Context, count int) (sources []types.SnapshotSource, err error) { 42 | res, err := c.resty.R(). 43 | SetContext(ctx). 44 | SetHeader("accept", "application/json"). 45 | SetQueryParam("max", strconv.Itoa(count)). 46 | SetResult(&sources). 47 | Get("/v1/best_snapshots") 48 | if err != nil { 49 | return nil, err 50 | } 51 | if res.StatusCode() != http.StatusOK { 52 | return nil, fmt.Errorf("get best snapshots: %s", res.Status()) 53 | } 54 | return 55 | } 56 | -------------------------------------------------------------------------------- /internal/index/index.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package index is an in-memory index for snapshots. 16 | package index 17 | 18 | import ( 19 | "time" 20 | 21 | "github.com/hashicorp/go-memdb" 22 | ) 23 | 24 | type DB struct { 25 | DB *memdb.MemDB 26 | } 27 | 28 | // NewDB creates a new, empty in-memory database. 29 | func NewDB() *DB { 30 | db, err := memdb.NewMemDB(&schema) 31 | if err != nil { 32 | panic("failed to create memDB: " + err.Error()) // unreachable 33 | } 34 | return &DB{ 35 | DB: db, 36 | } 37 | } 38 | 39 | // UpsertSnapshots inserts the given snapshot entries. 40 | // All entries must come from the same given target. 41 | // 42 | // Snapshots that have the same (target, slot) combination get replaced. 43 | // Returns the number of snapshots that have been replaced (excluding new inserts). 44 | func (d *DB) UpsertSnapshots(entries ...*SnapshotEntry) { 45 | txn := d.DB.Txn(true) 46 | defer txn.Abort() 47 | for _, entry := range entries { 48 | insertSnapshotEntry(txn, entry) 49 | } 50 | txn.Commit() 51 | } 52 | 53 | // GetSnapshotsByTarget returns all snapshots served by a host 54 | // ordered by newest to oldest. 55 | func (d *DB) GetSnapshotsByTarget(target string) (entries []*SnapshotEntry) { 56 | res, err := d.DB.Txn(false).Get(tableSnapshotEntry, "id_prefix", target) 57 | if err != nil { 58 | panic("getting snapshots by target failed: " + err.Error()) 59 | } 60 | for { 61 | entry := res.Next() 62 | if entry == nil { 63 | break 64 | } 65 | entries = append(entries, entry.(*SnapshotEntry)) 66 | } 67 | return 68 | } 69 | 70 | // GetAllSnapshots returns a list of all snapshots. 71 | func (d *DB) GetAllSnapshots() (entries []*SnapshotEntry) { 72 | iter, err := d.DB.Txn(false).LowerBound(tableSnapshotEntry, "id", "", uint64(0)) 73 | if err != nil { 74 | panic("getting best snapshots failed: " + err.Error()) 75 | } 76 | for { 77 | el := iter.Next() 78 | if el == nil { 79 | break 80 | } 81 | entries = append(entries, el.(*SnapshotEntry)) 82 | } 83 | return 84 | } 85 | 86 | // GetBestSnapshots returns newest-to-oldest snapshots. 87 | // The `max` argument controls the max number of snapshots to return. 88 | // If max is negative, it returns all snapshots. 89 | func (d *DB) GetBestSnapshots(max int) (entries []*SnapshotEntry) { 90 | res, err := d.DB.Txn(false).Get(tableSnapshotEntry, "slot") 91 | if err != nil { 92 | panic("getting best snapshots failed: " + err.Error()) 93 | } 94 | for max < 0 || len(entries) <= max { 95 | entry := res.Next() 96 | if entry == nil { 97 | break 98 | } 99 | entries = append(entries, entry.(*SnapshotEntry)) 100 | } 101 | return 102 | } 103 | 104 | // DeleteOldSnapshots delete snapshot entry older than the given timestamp. 105 | func (d *DB) DeleteOldSnapshots(minTime time.Time) (n int) { 106 | txn := d.DB.Txn(true) 107 | defer txn.Abort() 108 | res, err := txn.Get(tableSnapshotEntry, "id_prefix") 109 | if err != nil { 110 | panic("failed to range over all snapshots: " + err.Error()) 111 | } 112 | for { 113 | entry := res.Next() 114 | if entry == nil { 115 | break 116 | } 117 | if entry.(*SnapshotEntry).UpdatedAt.Before(minTime) { 118 | if err := txn.Delete(tableSnapshotEntry, entry); err != nil { 119 | panic("failed to delete expired snapshot: " + err.Error()) 120 | } 121 | n++ 122 | } 123 | } 124 | txn.Commit() 125 | return 126 | } 127 | 128 | // DeleteSnapshotsByTarget deletes all snapshots owned by a given target. 129 | // Returns the number of deletions made. 130 | func (d *DB) DeleteSnapshotsByTarget(target string) int { 131 | txn := d.DB.Txn(true) 132 | defer txn.Abort() 133 | n, err := txn.DeleteAll(tableSnapshotEntry, "id_prefix", target) 134 | if err != nil { 135 | panic("failed to delete snapshots by target: " + err.Error()) 136 | } 137 | txn.Commit() 138 | return n 139 | } 140 | 141 | func insertSnapshotEntry(txn *memdb.Txn, snap *SnapshotEntry) { 142 | if err := txn.Insert(tableSnapshotEntry, snap); err != nil { 143 | panic("failed to insert snapshot entry: " + err.Error()) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /internal/index/index_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package index 16 | 17 | import ( 18 | "testing" 19 | "time" 20 | 21 | "github.com/gagliardetto/solana-go" 22 | "github.com/stretchr/testify/assert" 23 | "go.blockdaemon.com/solana/cluster-manager/types" 24 | ) 25 | 26 | var dummyTime1 = time.Date(2022, 4, 27, 15, 33, 20, 0, time.UTC) 27 | 28 | var snapshotEntry1 = &SnapshotEntry{ 29 | SnapshotKey: NewSnapshotKey("host1", 100), 30 | UpdatedAt: dummyTime1, 31 | Info: &types.SnapshotInfo{ 32 | Slot: 100, 33 | Hash: solana.Hash{0x03}, 34 | Files: []*types.SnapshotFile{}, 35 | TotalSize: 0, 36 | }, 37 | } 38 | 39 | var snapshotEntry2 = &SnapshotEntry{ 40 | SnapshotKey: NewSnapshotKey("host1", 99), 41 | UpdatedAt: dummyTime1.Add(-20 * time.Second), 42 | Info: &types.SnapshotInfo{ 43 | Slot: 99, 44 | Hash: solana.Hash{0x04}, 45 | Files: []*types.SnapshotFile{}, 46 | TotalSize: 0, 47 | }, 48 | } 49 | 50 | var snapshotEntry3 = &SnapshotEntry{ 51 | SnapshotKey: NewSnapshotKey("host2", 100), 52 | UpdatedAt: dummyTime1, 53 | Info: &types.SnapshotInfo{ 54 | Slot: 100, 55 | Hash: solana.Hash{0x03}, 56 | Files: []*types.SnapshotFile{}, 57 | TotalSize: 0, 58 | }, 59 | } 60 | 61 | func TestDB(t *testing.T) { 62 | db := NewDB() 63 | 64 | assert.Equal(t, 0, db.DeleteSnapshotsByTarget("host1")) 65 | assert.Equal(t, 0, db.DeleteSnapshotsByTarget("host2")) 66 | 67 | assert.Len(t, db.GetSnapshotsByTarget("host1"), 0) 68 | assert.Len(t, db.GetSnapshotsByTarget("host2"), 0) 69 | assert.Len(t, db.GetBestSnapshots(-1), 0) 70 | 71 | db.UpsertSnapshots(snapshotEntry1) 72 | assert.Len(t, db.GetSnapshotsByTarget("host1"), 1) 73 | assert.Len(t, db.GetSnapshotsByTarget("host2"), 0) 74 | assert.Len(t, db.GetBestSnapshots(-1), 1) 75 | 76 | db.UpsertSnapshots(snapshotEntry1, snapshotEntry2) 77 | assert.Len(t, db.GetSnapshotsByTarget("host1"), 2) 78 | assert.Len(t, db.GetSnapshotsByTarget("host2"), 0) 79 | assert.Equal(t, 80 | []*SnapshotEntry{ 81 | snapshotEntry1, 82 | snapshotEntry2, 83 | }, 84 | db.GetBestSnapshots(-1)) 85 | 86 | db.UpsertSnapshots(snapshotEntry2, snapshotEntry3) 87 | assert.Len(t, db.GetSnapshotsByTarget("host1"), 2) 88 | assert.Len(t, db.GetSnapshotsByTarget("host2"), 1) 89 | assert.Equal(t, 90 | []*SnapshotEntry{ 91 | snapshotEntry1, 92 | snapshotEntry3, 93 | snapshotEntry2, 94 | }, 95 | db.GetBestSnapshots(-1)) 96 | 97 | assert.Equal(t, 2, db.DeleteSnapshotsByTarget("host1")) 98 | assert.Len(t, db.GetSnapshotsByTarget("host1"), 0) 99 | assert.Len(t, db.GetSnapshotsByTarget("host2"), 1) 100 | assert.Equal(t, 101 | []*SnapshotEntry{ 102 | snapshotEntry3, 103 | }, 104 | db.GetBestSnapshots(-1)) 105 | 106 | db.UpsertSnapshots(snapshotEntry1, snapshotEntry2) 107 | 108 | assert.Equal(t, 1, db.DeleteOldSnapshots(snapshotEntry2.UpdatedAt.Add(time.Second))) 109 | assert.Len(t, db.GetSnapshotsByTarget("host1"), 1) 110 | assert.Len(t, db.GetSnapshotsByTarget("host2"), 1) 111 | assert.Equal(t, 112 | []*SnapshotEntry{ 113 | snapshotEntry1, 114 | snapshotEntry3, 115 | }, 116 | db.GetBestSnapshots(-1)) 117 | } 118 | -------------------------------------------------------------------------------- /internal/index/schema.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package index 16 | 17 | import "github.com/hashicorp/go-memdb" 18 | 19 | const tableSnapshotEntry = "snapshot_entry" 20 | 21 | var schema = memdb.DBSchema{ 22 | Tables: map[string]*memdb.TableSchema{ 23 | tableSnapshotEntry: { 24 | Name: tableSnapshotEntry, 25 | Indexes: map[string]*memdb.IndexSchema{ 26 | "id": { 27 | Name: "id", 28 | Unique: true, 29 | Indexer: &memdb.CompoundIndex{ 30 | Indexes: []memdb.Indexer{ 31 | &memdb.StringFieldIndex{Field: "Target"}, 32 | &memdb.UintFieldIndex{Field: "InverseSlot"}, 33 | }, 34 | AllowMissing: false, 35 | }, 36 | }, 37 | "slot": { 38 | Name: "slot", 39 | Unique: false, 40 | Indexer: &memdb.UintFieldIndex{Field: "InverseSlot"}, 41 | }, 42 | }, 43 | }, 44 | }, 45 | } 46 | -------------------------------------------------------------------------------- /internal/index/types.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package index 16 | 17 | import ( 18 | "time" 19 | 20 | "go.blockdaemon.com/solana/cluster-manager/types" 21 | ) 22 | 23 | type SnapshotEntry struct { 24 | SnapshotKey 25 | Info *types.SnapshotInfo `json:"info"` 26 | UpdatedAt time.Time `json:"updated_at"` 27 | } 28 | 29 | type SnapshotKey struct { 30 | Target string `json:"target"` 31 | InverseSlot uint64 `json:"inverse_slot"` // newest-to-oldest sort 32 | } 33 | 34 | func NewSnapshotKey(target string, slot uint64) SnapshotKey { 35 | return SnapshotKey{ 36 | Target: target, 37 | InverseSlot: ^slot, 38 | } 39 | } 40 | 41 | func (k SnapshotKey) Slot() uint64 { 42 | return ^k.InverseSlot 43 | } 44 | -------------------------------------------------------------------------------- /internal/integrationtest/sidecar_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integrationtest 16 | 17 | import ( 18 | "context" 19 | "encoding/binary" 20 | "fmt" 21 | "net/http/httptest" 22 | "testing" 23 | 24 | "github.com/gagliardetto/solana-go" 25 | "github.com/gin-gonic/gin" 26 | "github.com/go-resty/resty/v2" 27 | "github.com/stretchr/testify/assert" 28 | "github.com/stretchr/testify/require" 29 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 30 | "go.blockdaemon.com/solana/cluster-manager/internal/ledgertest" 31 | "go.blockdaemon.com/solana/cluster-manager/internal/sidecar" 32 | "go.blockdaemon.com/solana/cluster-manager/types" 33 | "go.uber.org/zap/zaptest" 34 | ) 35 | 36 | // TestSidecar creates 37 | // a sidecar server with a fake ledger dir, 38 | // and a sidecar client 39 | func TestSidecar(t *testing.T) { 40 | server, root := newSidecar(t, 100) 41 | defer server.Close() 42 | client := fetch.NewSidecarClientWithOpts(server.URL, 43 | fetch.SidecarClientOpts{Resty: resty.NewWithClient(server.Client())}) 44 | 45 | ctx := context.TODO() 46 | 47 | t.Run("ListSnapshots", func(t *testing.T) { 48 | infos, err := client.ListSnapshots(ctx) 49 | require.NoError(t, err) 50 | assert.Equal(t, 51 | []*types.SnapshotInfo{ 52 | { 53 | Slot: 100, 54 | Hash: solana.MustHashFromBase58("7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V"), 55 | TotalSize: 1, 56 | Files: []*types.SnapshotFile{ 57 | { 58 | FileName: "snapshot-100-7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V.tar.bz2", 59 | Slot: 100, 60 | Hash: solana.MustHashFromBase58("7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V"), 61 | Ext: ".tar.bz2", 62 | Size: 1, 63 | ModTime: &root.DummyTime, 64 | }, 65 | }, 66 | }, 67 | }, 68 | infos, 69 | ) 70 | }) 71 | 72 | t.Run("DownloadSnapshot", func(t *testing.T) { 73 | res, err := client.StreamSnapshot(ctx, "snapshot-100-7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V.tar.bz2") 74 | require.NoError(t, err) 75 | assert.Equal(t, int64(1), res.ContentLength) 76 | require.NoError(t, res.Body.Close()) 77 | }) 78 | } 79 | 80 | func newSidecar(t *testing.T, slots ...uint64) (server *httptest.Server, root *ledgertest.FS) { 81 | root = ledgertest.NewFS(t) 82 | for _, slot := range slots { 83 | var fakeBin [32]byte 84 | binary.LittleEndian.PutUint64(fakeBin[:], slot) 85 | root.AddFakeFile(t, fmt.Sprintf("snapshot-%d-%s.tar.bz2", slot, solana.HashFromBytes(fakeBin[:]))) 86 | } 87 | ledgerDir := root.GetLedgerDir(t) 88 | 89 | handler := &sidecar.SnapshotHandler{ 90 | LedgerDir: ledgerDir, 91 | Log: zaptest.NewLogger(t), 92 | } 93 | 94 | gin.SetMode(gin.ReleaseMode) 95 | engine := gin.New() 96 | handler.RegisterHandlers(engine.Group("/v1")) 97 | server = httptest.NewServer(engine) 98 | return 99 | } 100 | -------------------------------------------------------------------------------- /internal/integrationtest/tracker_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package integrationtest 16 | 17 | import ( 18 | "context" 19 | "net/http/httptest" 20 | "net/url" 21 | "testing" 22 | "time" 23 | 24 | "github.com/gagliardetto/solana-go" 25 | "github.com/gin-gonic/gin" 26 | "github.com/go-resty/resty/v2" 27 | "github.com/stretchr/testify/assert" 28 | "github.com/stretchr/testify/require" 29 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 30 | "go.blockdaemon.com/solana/cluster-manager/internal/index" 31 | "go.blockdaemon.com/solana/cluster-manager/internal/scraper" 32 | "go.blockdaemon.com/solana/cluster-manager/internal/tracker" 33 | "go.blockdaemon.com/solana/cluster-manager/types" 34 | "go.uber.org/zap/zaptest" 35 | ) 36 | 37 | // TestTracker creates 38 | // a bunch of sidecars with a fake ledger dir, 39 | // a tracker running scraping infrastructure, 40 | // a tracker client 41 | func TestTracker(t *testing.T) { 42 | const sidecarCount = 4 43 | var targets []string 44 | for i := uint64(100); i < 100+sidecarCount; i++ { 45 | server, _ := newSidecar(t, i) 46 | defer server.Close() 47 | u, err := url.Parse(server.URL) 48 | require.NoError(t, err) 49 | targets = append(targets, u.Host) 50 | } 51 | group := &types.TargetGroup{ 52 | Group: "test", 53 | Scheme: "http", 54 | APIPath: "", 55 | StaticTargets: &types.StaticTargets{Targets: targets}, 56 | } 57 | 58 | // Create scraper infra. 59 | db := index.NewDB() 60 | collector := scraper.NewCollector(db) 61 | collector.Log = zaptest.NewLogger(t).Named("scraper") 62 | collector.Start() 63 | defer collector.Close() 64 | prober, err := scraper.NewProber(group) 65 | require.NoError(t, err) 66 | scraper_ := scraper.NewScraper(prober, group.StaticTargets) 67 | defer scraper_.Close() 68 | 69 | // Create tracker server. 70 | server := newTracker(db) 71 | defer server.Close() 72 | 73 | // Scrape for a while. 74 | scraper_.Start(collector.Probes(), 50*time.Millisecond) 75 | for i := 0; ; i++ { 76 | time.Sleep(25 * time.Millisecond) 77 | if i >= 50 { 78 | t.Fatal("Scrape timeout") 79 | } 80 | found := len(db.GetBestSnapshots(-1)) 81 | if found == sidecarCount { 82 | break 83 | } 84 | t.Logf("Found %d snapshots, waiting for %d", found, sidecarCount) 85 | } 86 | scraper_.Close() // We can stop scraping 87 | 88 | // Create tracker client. 89 | client := fetch.NewTrackerClientWithResty(resty.NewWithClient(server.Client()).SetHostURL(server.URL)) 90 | snaps, err := client.GetBestSnapshots(context.TODO(), -1) 91 | require.NoError(t, err) 92 | // Remove timestamps and port numbers. 93 | for i := range snaps { 94 | snap := &snaps[i] 95 | assert.False(t, snap.UpdatedAt.IsZero()) 96 | snap.UpdatedAt = time.Time{} 97 | for _, file := range snap.Files { 98 | assert.NotNil(t, file.ModTime) 99 | file.ModTime = nil 100 | } 101 | assert.NotEmpty(t, snap.Target) 102 | snap.Target = "" 103 | } 104 | assert.Equal(t, 105 | []types.SnapshotSource{ 106 | { 107 | SnapshotInfo: types.SnapshotInfo{ 108 | Slot: 103, 109 | Hash: solana.MustHashFromBase58("7w4zb1jh47zY5FPMPyRzDSmYf1CPirVP9LmTr5xWEs6X"), 110 | Files: []*types.SnapshotFile{ 111 | { 112 | FileName: "snapshot-103-7w4zb1jh47zY5FPMPyRzDSmYf1CPirVP9LmTr5xWEs6X.tar.bz2", 113 | Slot: 103, 114 | Hash: solana.MustHashFromBase58("7w4zb1jh47zY5FPMPyRzDSmYf1CPirVP9LmTr5xWEs6X"), 115 | Size: 1, 116 | Ext: ".tar.bz2", 117 | }, 118 | }, 119 | TotalSize: 1, 120 | }, 121 | }, 122 | { 123 | SnapshotInfo: types.SnapshotInfo{ 124 | Slot: 102, 125 | Hash: solana.MustHashFromBase58("7sAawX1cAHVpfZGNtUAYKX2KPzdd1uPUZUTaLteWX4SB"), 126 | Files: []*types.SnapshotFile{ 127 | { 128 | FileName: "snapshot-102-7sAawX1cAHVpfZGNtUAYKX2KPzdd1uPUZUTaLteWX4SB.tar.bz2", 129 | Slot: 102, 130 | Hash: solana.MustHashFromBase58("7sAawX1cAHVpfZGNtUAYKX2KPzdd1uPUZUTaLteWX4SB"), 131 | Size: 1, 132 | Ext: ".tar.bz2", 133 | }, 134 | }, 135 | TotalSize: 1, 136 | }, 137 | }, 138 | { 139 | SnapshotInfo: types.SnapshotInfo{ 140 | Slot: 101, 141 | Hash: solana.MustHashFromBase58("7oGBJ2HXGT17Fs9QNxu6RbH68z4rJxHZyc9gqhLWoFmq"), 142 | Files: []*types.SnapshotFile{ 143 | { 144 | FileName: "snapshot-101-7oGBJ2HXGT17Fs9QNxu6RbH68z4rJxHZyc9gqhLWoFmq.tar.bz2", 145 | Slot: 101, 146 | Hash: solana.MustHashFromBase58("7oGBJ2HXGT17Fs9QNxu6RbH68z4rJxHZyc9gqhLWoFmq"), 147 | Size: 1, 148 | Ext: ".tar.bz2", 149 | }, 150 | }, 151 | TotalSize: 1, 152 | }, 153 | }, 154 | { 155 | SnapshotInfo: types.SnapshotInfo{ 156 | Slot: 100, 157 | Hash: solana.MustHashFromBase58("7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V"), 158 | Files: []*types.SnapshotFile{ 159 | { 160 | FileName: "snapshot-100-7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V.tar.bz2", 161 | Slot: 100, 162 | Hash: solana.MustHashFromBase58("7jMmeXZSNcWPrB2RsTdeXfXrsyW5c1BfPjqoLW2X5T7V"), 163 | Size: 1, 164 | Ext: ".tar.bz2", 165 | }, 166 | }, 167 | TotalSize: 1, 168 | }, 169 | }, 170 | }, 171 | snaps) 172 | } 173 | 174 | func newTracker(db *index.DB) *httptest.Server { 175 | handler := tracker.NewHandler(db) 176 | gin.SetMode(gin.ReleaseMode) 177 | engine := gin.New() 178 | handler.RegisterHandlers(engine.Group("/v1")) 179 | return httptest.NewServer(engine) 180 | } 181 | -------------------------------------------------------------------------------- /internal/ledger/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package ledger interacts with the ledger dir. 16 | package ledger 17 | 18 | import ( 19 | "fmt" 20 | "io/fs" 21 | "path/filepath" 22 | "sort" 23 | "strings" 24 | 25 | "github.com/gagliardetto/solana-go" 26 | "go.blockdaemon.com/solana/cluster-manager/types" 27 | ) 28 | 29 | // ListSnapshotFiles returns all snapshot files in a ledger dir. 30 | func ListSnapshotFiles(ledgerDir fs.FS) ([]*types.SnapshotFile, error) { 31 | dirEntries, err := fs.ReadDir(ledgerDir, ".") 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to list ledger dir: %w", err) 34 | } 35 | var files []*types.SnapshotFile 36 | for _, dirEntry := range dirEntries { 37 | if !dirEntry.Type().IsRegular() { 38 | continue 39 | } 40 | info := ParseSnapshotFileName(dirEntry.Name()) 41 | if info == nil { 42 | continue 43 | } 44 | if err := SnapshotStat(ledgerDir, info); err != nil { 45 | continue 46 | } 47 | files = append(files, info) 48 | } 49 | sort.Slice(files, func(i, j int) bool { 50 | return files[i].Compare(files[j]) > 0 51 | }) 52 | return files, nil 53 | } 54 | 55 | // ListSnapshots shows all available snapshots of a ledger dir in the specified FS. 56 | // Result is sorted by best-to-worst. 57 | func ListSnapshots(ledgerDir fs.FS) ([]*types.SnapshotInfo, error) { 58 | // List and stat snapshot files. 59 | files, err := ListSnapshotFiles(ledgerDir) 60 | if err != nil { 61 | return nil, err 62 | } 63 | // Reconstruct snapshot chains for all available snapshots. 64 | infos := make([]*types.SnapshotInfo, 0, len(files)) 65 | for _, file := range files { 66 | if info := buildSnapshotInfo(files, file); info != nil { 67 | infos = append(infos, info) 68 | } 69 | } 70 | return infos, nil 71 | } 72 | 73 | // buildSnapshotInfo builds a snapshot info object against the target snapshot file. 74 | // The files array must be sorted. 75 | func buildSnapshotInfo(files []*types.SnapshotFile, target *types.SnapshotFile) *types.SnapshotInfo { 76 | // Start at target snapshot and reconstruct chain of snapshots. 77 | chain := []*types.SnapshotFile{target} 78 | totalSize := target.Size 79 | for { 80 | base := chain[len(chain)-1].BaseSlot 81 | if base == 0 { 82 | break // complete chain 83 | } 84 | // Find snapshot matching base slot number. 85 | index := sort.Search(len(files), func(i int) bool { 86 | return files[i].Slot <= base 87 | }) 88 | if index >= len(files) || files[index].Slot != base { 89 | return nil // incomplete chain 90 | } 91 | // Extend snapshot chain. 92 | chain = append(chain, files[index]) 93 | totalSize += files[index].Size 94 | } 95 | return &types.SnapshotInfo{ 96 | Slot: target.Slot, 97 | Hash: target.Hash, 98 | Files: chain, 99 | TotalSize: totalSize, 100 | } 101 | } 102 | 103 | // ParseSnapshotFileName parses a snapshot's name. 104 | func ParseSnapshotFileName(name string) *types.SnapshotFile { 105 | // Split file name into base and stem. 106 | stem := name 107 | var ext string 108 | for i := 0; i < 2; i++ { 109 | extPart := filepath.Ext(stem) 110 | stem = strings.TrimSuffix(stem, extPart) 111 | ext = extPart + ext 112 | } 113 | if strings.ContainsAny(stem, "\\/ \t\n") { 114 | return nil 115 | } 116 | // Parse file name fields. 117 | if strings.HasPrefix(stem, "snapshot-") { 118 | var slot uint64 119 | var hashStr string 120 | n, err := fmt.Sscanf(stem, "snapshot-%d-%s", &slot, &hashStr) 121 | if n != 2 || err != nil { 122 | return nil 123 | } 124 | hash, err := solana.HashFromBase58(hashStr) 125 | if err != nil { 126 | return nil 127 | } 128 | return &types.SnapshotFile{ 129 | FileName: name, 130 | Slot: slot, 131 | Hash: hash, 132 | Ext: ext, 133 | } 134 | } 135 | if strings.HasPrefix(stem, "incremental-snapshot-") { 136 | var baseSlot, incrementalSlot uint64 137 | var hashStr string 138 | n, err := fmt.Sscanf(stem, "incremental-snapshot-%d-%d-%s", &baseSlot, &incrementalSlot, &hashStr) 139 | if n != 3 || err != nil { 140 | return nil 141 | } 142 | hash, err := solana.HashFromBase58(hashStr) 143 | if err != nil { 144 | return nil 145 | } 146 | if incrementalSlot <= baseSlot { 147 | return nil 148 | } 149 | return &types.SnapshotFile{ 150 | FileName: name, 151 | Slot: incrementalSlot, 152 | BaseSlot: baseSlot, 153 | Hash: hash, 154 | Ext: ext, 155 | } 156 | } 157 | return nil 158 | } 159 | 160 | // SnapshotStat fills stat info into the snapshot file. 161 | func SnapshotStat(fs_ fs.FS, snap *types.SnapshotFile) error { 162 | stat, err := fs.Stat(fs_, snap.FileName) 163 | if err != nil { 164 | return err 165 | } 166 | snap.Size = uint64(stat.Size()) 167 | if modTime := stat.ModTime(); !modTime.IsZero() { 168 | snap.ModTime = &modTime 169 | } 170 | return nil 171 | } 172 | -------------------------------------------------------------------------------- /internal/ledger/snapshot_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package ledger 16 | 17 | import ( 18 | "encoding/json" 19 | "testing" 20 | 21 | "github.com/gagliardetto/solana-go" 22 | "github.com/stretchr/testify/assert" 23 | "github.com/stretchr/testify/require" 24 | "go.blockdaemon.com/solana/cluster-manager/internal/ledgertest" 25 | "go.blockdaemon.com/solana/cluster-manager/types" 26 | ) 27 | 28 | func TestListSnapshots(t *testing.T) { 29 | // Construct a new ledger dir with a bunch of snapshots. 30 | root := ledgertest.NewFS(t) 31 | fakeFiles := []string{ 32 | "snapshot-50-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 33 | "incremental-snapshot-50-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 34 | "snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 35 | "incremental-snapshot-100-200-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 36 | "incremental-snapshot-200-300-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 37 | "incremental-snapshot-99999-1010101-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 38 | } 39 | for _, name := range fakeFiles { 40 | root.AddFakeFile(t, name) 41 | } 42 | ledgerDir := root.GetLedgerDir(t) 43 | 44 | snapshots, err := ListSnapshots(ledgerDir) 45 | require.NoError(t, err) 46 | 47 | j, _ := json.MarshalIndent(snapshots, "", "\t") 48 | t.Log(string(j)) 49 | 50 | assert.Equal(t, 51 | []*types.SnapshotInfo{ 52 | { 53 | Slot: 300, 54 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 55 | TotalSize: 3, 56 | Files: []*types.SnapshotFile{ 57 | { 58 | FileName: "incremental-snapshot-200-300-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 59 | BaseSlot: 200, 60 | Slot: 300, 61 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 62 | Ext: ".tar.zst", 63 | Size: 1, 64 | ModTime: &root.DummyTime, 65 | }, 66 | { 67 | FileName: "incremental-snapshot-100-200-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 68 | BaseSlot: 100, 69 | Slot: 200, 70 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 71 | Ext: ".tar.zst", 72 | Size: 1, 73 | ModTime: &root.DummyTime, 74 | }, 75 | { 76 | FileName: "snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 77 | Slot: 100, 78 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 79 | Ext: ".tar.bz2", 80 | Size: 1, 81 | ModTime: &root.DummyTime, 82 | }, 83 | }, 84 | }, 85 | { 86 | Slot: 200, 87 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 88 | TotalSize: 2, 89 | Files: []*types.SnapshotFile{ 90 | { 91 | FileName: "incremental-snapshot-100-200-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 92 | BaseSlot: 100, 93 | Slot: 200, 94 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 95 | Ext: ".tar.zst", 96 | Size: 1, 97 | ModTime: &root.DummyTime, 98 | }, 99 | { 100 | FileName: "snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 101 | Slot: 100, 102 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 103 | Ext: ".tar.bz2", 104 | Size: 1, 105 | ModTime: &root.DummyTime, 106 | }, 107 | }, 108 | }, 109 | { 110 | Slot: 100, 111 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 112 | TotalSize: 1, 113 | Files: []*types.SnapshotFile{ 114 | { 115 | FileName: "snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 116 | Slot: 100, 117 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 118 | Ext: ".tar.bz2", 119 | Size: 1, 120 | ModTime: &root.DummyTime, 121 | }, 122 | }, 123 | }, 124 | { 125 | Slot: 100, 126 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 127 | TotalSize: 2, 128 | Files: []*types.SnapshotFile{ 129 | { 130 | FileName: "incremental-snapshot-50-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 131 | BaseSlot: 50, 132 | Slot: 100, 133 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 134 | Ext: ".tar.zst", 135 | Size: 1, 136 | ModTime: &root.DummyTime, 137 | }, 138 | { 139 | FileName: "snapshot-50-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 140 | Slot: 50, 141 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 142 | Ext: ".tar.bz2", 143 | Size: 1, 144 | ModTime: &root.DummyTime, 145 | }, 146 | }, 147 | }, 148 | { 149 | Slot: 50, 150 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 151 | TotalSize: 1, 152 | Files: []*types.SnapshotFile{ 153 | { 154 | FileName: "snapshot-50-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 155 | Slot: 50, 156 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 157 | Ext: ".tar.bz2", 158 | Size: 1, 159 | ModTime: &root.DummyTime, 160 | }, 161 | }, 162 | }, 163 | }, 164 | snapshots, 165 | ) 166 | } 167 | 168 | func TestParseSnapshotFileName(t *testing.T) { 169 | cases := []struct { 170 | name string 171 | path string 172 | info *types.SnapshotFile 173 | }{ 174 | { 175 | name: "Empty", 176 | path: "", 177 | info: nil, 178 | }, 179 | { 180 | name: "MissingParts", 181 | path: "snapshot-121646378.tar.zst", 182 | info: nil, 183 | }, 184 | { 185 | name: "InvalidSlotNumber", 186 | path: "snapshot-notaslotnumber-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 187 | info: nil, 188 | }, 189 | { 190 | name: "InvalidHash", 191 | path: "snapshot-12345678-bad!hash.tar", 192 | info: nil, 193 | }, 194 | { 195 | name: "IncrementalSnapshot", 196 | path: "incremental-snapshot-100-200-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 197 | info: &types.SnapshotFile{ 198 | FileName: "incremental-snapshot-100-200-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 199 | BaseSlot: 100, 200 | Slot: 200, 201 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 202 | Ext: ".tar.zst", 203 | }, 204 | }, 205 | { 206 | name: "IncrementalSnapshotInvalidHash", 207 | path: "incremental-snapshot-100-12345678-bad!hash.tar", 208 | info: nil, 209 | }, 210 | { 211 | name: "IncrementalSnapshotWeird", 212 | path: "incremental-snapshot-100.tar", 213 | info: nil, 214 | }, 215 | { 216 | name: "IncrementalSnapshotWhitespace", 217 | path: "incremental-snapshot- e.tar", 218 | info: nil, 219 | }, 220 | { 221 | name: "IncrementalSnapshotImpossible", 222 | path: "incremental-snapshot-300-200-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.zst", 223 | info: nil, 224 | }, 225 | { 226 | name: "NormalSnapshot", 227 | path: "snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 228 | info: &types.SnapshotFile{ 229 | FileName: "snapshot-100-AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr.tar.bz2", 230 | Slot: 100, 231 | Hash: solana.MustHashFromBase58("AvFf9oS8A8U78HdjT9YG2sTTThLHJZmhaMn2g8vkWYnr"), 232 | Ext: ".tar.bz2", 233 | }, 234 | }, 235 | } 236 | 237 | for _, tc := range cases { 238 | t.Run(tc.name, func(t *testing.T) { 239 | info := ParseSnapshotFileName(tc.path) 240 | assert.Equal(t, tc.info, info) 241 | }) 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /internal/ledgertest/ledgertest.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package ledgertest mocks the ledger dir for testing. 16 | package ledgertest 17 | 18 | import ( 19 | "io/fs" 20 | "path/filepath" 21 | "testing" 22 | "time" 23 | 24 | "github.com/spf13/afero" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | type FS struct { 29 | DummyTime time.Time // arbitrary but consistent timestamp 30 | Root afero.Fs 31 | } 32 | 33 | // ledgerPath is the relative path of the fake ledger dir to file system root. 34 | const ledgerPath = "data/ledger" 35 | 36 | // NewFS creates an in-memory file system resembling the ledger dir. 37 | func NewFS(t *testing.T) *FS { 38 | t.Helper() 39 | root := afero.NewMemMapFs() 40 | require.NoError(t, root.MkdirAll(ledgerPath, 0755)) 41 | return &FS{ 42 | DummyTime: time.Date(2022, 4, 27, 15, 33, 20, 0, time.UTC), 43 | Root: root, 44 | } 45 | } 46 | 47 | // AddFakeFile adds a file sized one byte to the fake ledger dir. 48 | func (f *FS) AddFakeFile(t *testing.T, name string) { 49 | t.Helper() 50 | filePath := filepath.Join(ledgerPath, name) 51 | file, err := f.Root.Create(filePath) 52 | require.NoError(t, err) 53 | require.NoError(t, file.Truncate(1)) 54 | require.NoError(t, file.Close()) 55 | require.NoError(t, f.Root.Chtimes(filePath, f.DummyTime, f.DummyTime)) 56 | } 57 | 58 | // GetLedgerDir returns the ledger dir as a standard library fs.FS. 59 | func (f *FS) GetLedgerDir(t *testing.T) fs.FS { 60 | t.Helper() 61 | ledgerDir, err := fs.Sub(afero.NewIOFS(f.Root), ledgerPath) 62 | require.NoError(t, err) 63 | return ledgerDir 64 | } 65 | -------------------------------------------------------------------------------- /internal/logger/logger.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package logger 16 | 17 | import ( 18 | "github.com/spf13/pflag" 19 | "go.uber.org/zap" 20 | "go.uber.org/zap/zapcore" 21 | ) 22 | 23 | var ( 24 | Flags = pflag.NewFlagSet("logger", pflag.ExitOnError) 25 | 26 | logLevel = LogLevel{zap.InfoLevel} 27 | logFormat string 28 | ) 29 | 30 | func init() { 31 | Flags.Var(&logLevel, "log-level", "Log level") 32 | Flags.StringVar(&logFormat, "log-format", "console", "Log format (console, json)") 33 | } 34 | 35 | func GetLogger() *zap.Logger { 36 | var config zap.Config 37 | if logFormat == "json" { 38 | config = zap.NewProductionConfig() 39 | } else { 40 | config = zap.NewDevelopmentConfig() 41 | config.DisableStacktrace = true 42 | } 43 | config.DisableCaller = true 44 | config.Level.SetLevel(logLevel.Level) 45 | logger, err := config.Build() 46 | if err != nil { 47 | panic(err.Error()) 48 | } 49 | return logger 50 | } 51 | 52 | // LogLevel is required to use zap level as a pflag. 53 | type LogLevel struct{ zapcore.Level } 54 | 55 | func (LogLevel) Type() string { 56 | return "string" 57 | } 58 | 59 | func GetConsoleLogger() *zap.Logger { 60 | logConfig := zap.Config{ 61 | Level: zap.NewAtomicLevel(), 62 | DisableCaller: true, 63 | DisableStacktrace: true, 64 | Encoding: "console", 65 | EncoderConfig: zap.NewDevelopmentEncoderConfig(), 66 | } 67 | logConfig.Level.SetLevel(zap.InfoLevel) 68 | log, err := logConfig.Build() 69 | if err != nil { 70 | panic(err.Error()) 71 | } 72 | return log 73 | } 74 | -------------------------------------------------------------------------------- /internal/logger/logger_test.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGetLogger(t *testing.T) { 10 | testCases := []struct { 11 | name string 12 | format string 13 | }{ 14 | {name: "JSON", format: "json"}, 15 | {name: "Console", format: "console"}, 16 | {name: "Default", format: ""}, 17 | {name: "Unknown", format: "bla"}, 18 | } 19 | for _, tc := range testCases { 20 | logFormat = tc.format // global 21 | log := GetLogger() 22 | require.NotNil(t, log) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /internal/mirror/mirror.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package mirror maintains a copy of snapshots in S3. 16 | package mirror 17 | 18 | import ( 19 | "context" 20 | "net/http" 21 | "sync" 22 | "time" 23 | 24 | "github.com/minio/minio-go/v7" 25 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 26 | "go.blockdaemon.com/solana/cluster-manager/types" 27 | "go.uber.org/zap" 28 | ) 29 | 30 | // Worker mirrors snapshots from nodes to S3. 31 | type Worker struct { 32 | Tracker *fetch.TrackerClient 33 | Uploader *Uploader 34 | Log *zap.Logger 35 | 36 | Refresh time.Duration 37 | SyncCount int 38 | } 39 | 40 | func NewWorker(tracker *fetch.TrackerClient, uploader *Uploader) *Worker { 41 | return &Worker{ 42 | Tracker: tracker, 43 | Uploader: uploader, 44 | Log: zap.NewNop(), 45 | Refresh: 15 * time.Second, 46 | SyncCount: 10, 47 | } 48 | } 49 | 50 | func (w *Worker) Run(ctx context.Context) { 51 | w.Log.Info("Worker starting") 52 | defer w.Log.Info("Worker stopped") 53 | 54 | ticker := time.NewTicker(w.Refresh) 55 | defer ticker.Stop() 56 | for { 57 | select { 58 | case <-ctx.Done(): 59 | return 60 | case <-ticker.C: 61 | w.tick(ctx) 62 | } 63 | } 64 | } 65 | 66 | func (w *Worker) tick(ctx context.Context) { 67 | w.Log.Debug("Tick") 68 | sources, err := w.Tracker.GetBestSnapshots(ctx, w.SyncCount) 69 | if err != nil { 70 | w.Log.Error("Failed to find new snapshots", zap.Error(err)) 71 | return 72 | } 73 | 74 | type fileSource struct { 75 | target string 76 | file *types.SnapshotFile 77 | } 78 | files := make(map[uint64]fileSource) 79 | for _, src := range sources { 80 | for _, file := range src.Files { 81 | if _, ok := files[file.Slot]; !ok { 82 | files[file.Slot] = fileSource{ 83 | target: "http://" + src.Target, // TODO don't hardcode protocol scheme 84 | file: file, 85 | } 86 | } 87 | } 88 | } 89 | 90 | var wg sync.WaitGroup 91 | for _, src := range files { 92 | // TODO Consider using a semaphore 93 | job := UploadJob{ 94 | Provider: src.target, 95 | File: src.file, 96 | Uploader: w.Uploader, 97 | Log: w.Log.With(zap.String("snapshot", src.file.FileName)), 98 | } 99 | wg.Add(1) 100 | go func() { 101 | defer wg.Done() 102 | job.Run(ctx) 103 | }() 104 | } 105 | wg.Wait() 106 | } 107 | 108 | type UploadJob struct { 109 | Provider string 110 | File *types.SnapshotFile 111 | Uploader *Uploader 112 | Log *zap.Logger 113 | } 114 | 115 | func (j *UploadJob) Run(ctx context.Context) { 116 | fileName := j.File.FileName 117 | // Check if snapshot exists. 118 | stat, statErr := j.Uploader.StatSnapshot(ctx, fileName) 119 | if statErr == nil { 120 | j.Log.Debug("Already uploaded", 121 | zap.Time("last_modified", stat.LastModified)) 122 | return 123 | } 124 | statResp := minio.ToErrorResponse(statErr) 125 | if statResp.StatusCode != http.StatusNotFound { 126 | j.Log.Error("Unexpected error", zap.Error(statErr)) 127 | return 128 | } 129 | 130 | // TODO use client factory 131 | sidecarClient := fetch.NewSidecarClientWithOpts(j.Provider, fetch.SidecarClientOpts{ 132 | Log: j.Log.Named("fetch"), 133 | }) 134 | 135 | j.Log.Info("Starting upload") 136 | beforeUpload := time.Now() 137 | uploadInfo, err := j.Uploader.UploadSnapshot(ctx, sidecarClient, fileName) 138 | uploadDuration := time.Since(beforeUpload) 139 | if err != nil { 140 | j.Log.Error("Upload failed", zap.Error(err), 141 | zap.Duration("upload_duration", uploadDuration)) 142 | return 143 | } 144 | j.Log.Info("Upload succeeded", 145 | zap.String("bucket", uploadInfo.Bucket), 146 | zap.String("object", uploadInfo.Key), 147 | zap.Duration("upload_duration", uploadDuration)) 148 | } 149 | -------------------------------------------------------------------------------- /internal/mirror/uploader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package mirror 16 | 17 | import ( 18 | "context" 19 | 20 | "github.com/minio/minio-go/v7" 21 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 22 | ) 23 | 24 | // Uploader streams snapshots from a node to an S3 mirror. 25 | type Uploader struct { 26 | S3Client *minio.Client 27 | Bucket string 28 | ObjectPrefix string 29 | } 30 | 31 | // StatSnapshot checks whether a snapshot has been uploaded already. 32 | func (u *Uploader) StatSnapshot(ctx context.Context, fileName string) (minio.ObjectInfo, error) { 33 | objectName := u.getSnapshotObjectName(fileName) 34 | return u.S3Client.StatObject(ctx, u.Bucket, objectName, minio.StatObjectOptions{}) 35 | } 36 | 37 | // UploadSnapshot streams a snapshot from the given sidecar client to S3. 38 | func (u *Uploader) UploadSnapshot(ctx context.Context, sourceClient *fetch.SidecarClient, fileName string) (minio.UploadInfo, error) { 39 | res, err := sourceClient.StreamSnapshot(ctx, fileName) 40 | if res != nil { 41 | defer res.Body.Close() 42 | } 43 | if err != nil { 44 | return minio.UploadInfo{}, err 45 | } 46 | objectName := u.getSnapshotObjectName(fileName) 47 | return u.S3Client.PutObject(ctx, u.Bucket, objectName, res.Body, res.ContentLength, minio.PutObjectOptions{}) 48 | } 49 | 50 | func (u *Uploader) getSnapshotObjectName(fileName string) string { 51 | return u.ObjectPrefix + fileName 52 | } 53 | -------------------------------------------------------------------------------- /internal/netx/netx.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package netx contains network hacks 16 | package netx 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "net" 22 | 23 | "go.uber.org/atomic" 24 | "golang.org/x/sync/errgroup" 25 | ) 26 | 27 | // MergeListeners merges multiple net.Listeners into one. Trips on the first error seen. 28 | func MergeListeners(listeners ...net.Listener) net.Listener { 29 | merged := &mergedListener{listeners: listeners} 30 | merged.start() 31 | return merged 32 | } 33 | 34 | type mergedListener struct { 35 | listeners []net.Listener 36 | cancel context.CancelFunc 37 | group *errgroup.Group 38 | ctx context.Context 39 | conns chan net.Conn 40 | err atomic.Error 41 | } 42 | 43 | func (m *mergedListener) start() { 44 | m.ctx, m.cancel = context.WithCancel(context.Background()) 45 | m.group, m.ctx = errgroup.WithContext(m.ctx) 46 | m.conns = make(chan net.Conn) 47 | for _, listener := range m.listeners { 48 | listener_ := listener 49 | go func() { 50 | <-m.ctx.Done() 51 | if err := listener_.Close(); err != nil { 52 | m.err.Store(err) 53 | } 54 | }() 55 | m.group.Go(func() error { 56 | for { 57 | accept, err := listener_.Accept() 58 | if err != nil { 59 | if m.ctx.Err() != nil { 60 | return nil // context cancel requested, exit gracefully 61 | } else { 62 | return err 63 | } 64 | } 65 | select { 66 | case <-m.ctx.Done(): 67 | return nil 68 | case m.conns <- accept: 69 | } 70 | } 71 | }) 72 | } 73 | } 74 | 75 | func (m *mergedListener) Accept() (net.Conn, error) { 76 | select { 77 | case <-m.ctx.Done(): 78 | err := net.ErrClosed 79 | if storedErr := m.err.Load(); storedErr != nil { 80 | err = storedErr 81 | } 82 | return nil, err 83 | case conn := <-m.conns: 84 | return conn, nil 85 | } 86 | } 87 | 88 | func (m *mergedListener) Close() error { 89 | m.cancel() 90 | return m.group.Wait() 91 | } 92 | 93 | func (m *mergedListener) Addr() net.Addr { 94 | for _, listener := range m.listeners { 95 | if ta, ok := listener.Addr().(*net.TCPAddr); ok { 96 | if ta.IP.IsGlobalUnicast() { 97 | return ta 98 | } 99 | } 100 | } 101 | return m.listeners[0].Addr() 102 | } 103 | 104 | // ListenTCPInterface is like net.ListenTCP but can bind to one interface only. 105 | // 106 | // Internally, it listens to all host IP addresses of the given interface. 107 | // If the interface name is empty, it listens on all addresses. 108 | func ListenTCPInterface(network string, ifaceName string, port uint16) (net.Listener, []net.TCPAddr, error) { 109 | var listenAddrs []net.TCPAddr 110 | if ifaceName != "" { 111 | iface, err := net.InterfaceByName(ifaceName) 112 | if err != nil { 113 | return nil, nil, err 114 | } 115 | ifaceAddrs, err := iface.Addrs() 116 | if err != nil { 117 | return nil, nil, err 118 | } 119 | for _, addr := range ifaceAddrs { 120 | ip, _, err := net.ParseCIDR(addr.String()) 121 | if err != nil { 122 | continue 123 | } 124 | listenAddrs = append(listenAddrs, net.TCPAddr{ 125 | IP: ip, 126 | Port: int(port), 127 | Zone: iface.Name, 128 | }) 129 | } 130 | } else { 131 | listenAddrs = []net.TCPAddr{ 132 | { 133 | IP: net.IPv6zero, // dual-stack 134 | Port: int(port), 135 | }, 136 | } 137 | } 138 | 139 | listeners := make([]net.Listener, len(listenAddrs)) 140 | tcpAddrs := make([]net.TCPAddr, len(listenAddrs)) 141 | for i, tcp := range listenAddrs { 142 | listen, err := net.ListenTCP(network, &tcp) 143 | if err != nil { 144 | return nil, nil, err 145 | } 146 | listeners[i] = listen 147 | tcpAddrs[i] = tcp 148 | } 149 | if len(listeners) == 0 { 150 | return nil, nil, fmt.Errorf("listen on %s: interface has no addresses", ifaceName) 151 | } 152 | return MergeListeners(listeners...), tcpAddrs, nil 153 | } 154 | -------------------------------------------------------------------------------- /internal/netx/netx_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package netx 16 | 17 | import ( 18 | "context" 19 | "errors" 20 | "net" 21 | "net/http" 22 | "sync" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/assert" 27 | "github.com/stretchr/testify/require" 28 | ) 29 | 30 | func TestListenTCPInterface(t *testing.T) { 31 | // Find loopback interface. 32 | ifaces, err := net.Interfaces() 33 | require.NoError(t, err) 34 | var lo string 35 | for _, iface := range ifaces { 36 | if iface.Flags&net.FlagLoopback != 0 { 37 | lo = iface.Name 38 | break 39 | } 40 | } 41 | if lo == "" { 42 | t.Skip("Could not find loopback interface") 43 | } 44 | 45 | // Bind to loopback. 46 | t.Logf("Binding to %v", lo) 47 | listener, addrs, err := ListenTCPInterface("tcp", lo, 0) 48 | require.NoError(t, err) 49 | assert.NotEmpty(t, addrs) 50 | t.Log("Addresses", addrs) 51 | require.NoError(t, listener.Close()) 52 | } 53 | 54 | // TestMergeListeners ensures an HTTP server can accept conns from multiple listeners. 55 | func TestMergeListeners(t *testing.T) { 56 | l1, err := net.Listen("tcp", "127.0.0.1:0") 57 | require.NoError(t, err) 58 | l2, err := net.Listen("tcp", "127.0.0.1:0") 59 | require.NoError(t, err) 60 | 61 | merged := MergeListeners(l1, l2) 62 | assert.Equal(t, l1.Addr(), merged.Addr()) 63 | 64 | server := http.Server{ 65 | Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 66 | w.WriteHeader(http.StatusNoContent) 67 | }), 68 | } 69 | go func() { 70 | _ = server.Serve(merged) 71 | }() 72 | 73 | pingUntilAwake(t, "http://"+l1.Addr().String()) 74 | 75 | _, err = http.DefaultClient.Get("http://" + l1.Addr().String()) 76 | require.NoError(t, err) 77 | _, err = http.DefaultClient.Get("http://" + l2.Addr().String()) 78 | require.NoError(t, err) 79 | 80 | require.NoError(t, merged.Close()) 81 | time.Sleep(100 * time.Millisecond) 82 | assert.True(t, errors.Is(l1.Close(), net.ErrClosed)) 83 | assert.True(t, errors.Is(l2.Close(), net.ErrClosed)) 84 | } 85 | 86 | // TestMergeListeners_ExternalClose ensures a merged listener shuts down when any of its listeners fail. 87 | func TestMergeListeners_ExternalClose(t *testing.T) { 88 | l1, err := net.Listen("tcp", "127.0.0.1:0") 89 | require.NoError(t, err) 90 | l2, err := net.Listen("tcp", "127.0.0.1:0") 91 | require.NoError(t, err) 92 | 93 | merged := MergeListeners(l1, l2) 94 | assert.Equal(t, l1.Addr(), merged.Addr()) 95 | 96 | server := http.Server{ 97 | Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { 98 | w.WriteHeader(http.StatusNoContent) 99 | }), 100 | } 101 | go func() { 102 | _ = server.Serve(merged) 103 | }() 104 | 105 | pingUntilAwake(t, "http://"+l1.Addr().String()) 106 | 107 | // Externally close one of the listeners. 108 | require.NoError(t, l1.Close()) 109 | 110 | time.Sleep(100 * time.Millisecond) 111 | 112 | assert.True(t, errors.Is(l1.Close(), net.ErrClosed)) 113 | assert.True(t, errors.Is(l2.Close(), net.ErrClosed)) 114 | require.True(t, errors.Is(merged.Close(), net.ErrClosed)) 115 | } 116 | 117 | // TestMergeListeners_CloseSlowAcceptor ensures a merged listener shuts down when net.Listener::Accept() has never returned. 118 | func TestMergeListeners_CloseSlowAcceptor(t *testing.T) { 119 | l1, err := net.Listen("tcp", "127.0.0.1:0") 120 | require.NoError(t, err) 121 | l2, err := net.Listen("tcp", "127.0.0.1:0") 122 | require.NoError(t, err) 123 | 124 | merged := MergeListeners(l1, l2) 125 | assert.Equal(t, l1.Addr(), merged.Addr()) 126 | 127 | time.Sleep(200 * time.Millisecond) 128 | 129 | var wg sync.WaitGroup 130 | wg.Add(1) 131 | go func() { 132 | defer wg.Done() 133 | _, err := net.Dial("tcp", l1.Addr().String()) 134 | assert.NoError(t, err) 135 | }() 136 | 137 | time.Sleep(200 * time.Millisecond) 138 | 139 | require.NoError(t, merged.Close()) 140 | wg.Wait() 141 | time.Sleep(200 * time.Millisecond) 142 | assert.True(t, errors.Is(l1.Close(), net.ErrClosed)) 143 | assert.True(t, errors.Is(l2.Close(), net.ErrClosed)) 144 | } 145 | 146 | func pingUntilAwake(t *testing.T, url string) { 147 | const timeout = 3 * time.Second 148 | const interval = 20 * time.Millisecond 149 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 150 | defer cancel() 151 | 152 | for { 153 | time.Sleep(interval) 154 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 155 | require.NoError(t, err) 156 | _, err = http.DefaultClient.Do(req) 157 | if errors.Is(err, context.DeadlineExceeded) { 158 | t.Fatal(err) 159 | } else if err != nil { 160 | continue 161 | } 162 | return 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /internal/scraper/collector.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scraper 16 | 17 | import ( 18 | "sync/atomic" 19 | "time" 20 | 21 | "go.blockdaemon.com/solana/cluster-manager/internal/index" 22 | "go.blockdaemon.com/solana/cluster-manager/types" 23 | "go.uber.org/zap" 24 | ) 25 | 26 | // Collector streams probe results into the database. 27 | type Collector struct { 28 | resChan chan ProbeResult 29 | DB *index.DB 30 | Log *zap.Logger 31 | 32 | closed uint32 33 | } 34 | 35 | func NewCollector(db *index.DB) *Collector { 36 | this := &Collector{ 37 | resChan: make(chan ProbeResult), 38 | DB: db, 39 | Log: zap.NewNop(), 40 | } 41 | return this 42 | } 43 | 44 | func (c *Collector) Start() { 45 | go c.run() 46 | } 47 | 48 | // Probes returns a send-channel that collects and indexes probe results. 49 | func (c *Collector) Probes() chan<- ProbeResult { 50 | return c.resChan 51 | } 52 | 53 | // Close stops the collector and closes the send-channel. 54 | func (c *Collector) Close() { 55 | if atomic.CompareAndSwapUint32(&c.closed, 0, 1) { 56 | close(c.resChan) 57 | } 58 | } 59 | 60 | func (c *Collector) run() { 61 | for res := range c.resChan { 62 | if res.Err != nil { 63 | c.Log.Warn("Scrape failed", 64 | zap.String("target", res.Target), 65 | zap.Error(res.Err)) 66 | continue 67 | } 68 | c.Log.Debug("Scrape success", 69 | zap.String("target", res.Target), 70 | zap.Int("num_snapshots", len(res.Infos))) 71 | c.DB.DeleteSnapshotsByTarget(res.Target) 72 | entries := make([]*index.SnapshotEntry, len(res.Infos)) 73 | for i, info := range res.Infos { 74 | entries[i] = &index.SnapshotEntry{ 75 | SnapshotKey: index.NewSnapshotKey(res.Target, info.Slot), 76 | Info: info, 77 | UpdatedAt: res.Time, 78 | } 79 | } 80 | c.DB.UpsertSnapshots(entries...) 81 | } 82 | } 83 | 84 | type ProbeResult struct { 85 | Time time.Time 86 | Target string 87 | Infos []*types.SnapshotInfo 88 | Err error 89 | } 90 | -------------------------------------------------------------------------------- /internal/scraper/manager.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scraper 16 | 17 | import ( 18 | "sync" 19 | 20 | "go.blockdaemon.com/solana/cluster-manager/internal/discovery" 21 | "go.blockdaemon.com/solana/cluster-manager/types" 22 | "go.uber.org/zap" 23 | ) 24 | 25 | // Manager maintains a group of scrapers. 26 | type Manager struct { 27 | res chan<- ProbeResult 28 | scrapers []*Scraper 29 | 30 | Log *zap.Logger 31 | } 32 | 33 | func NewManager(results chan<- ProbeResult) *Manager { 34 | return &Manager{ 35 | res: results, 36 | 37 | Log: zap.NewNop(), 38 | } 39 | } 40 | 41 | // Reset shuts down all scrapers. 42 | func (m *Manager) Reset() { 43 | var wg sync.WaitGroup 44 | wg.Add(len(m.scrapers)) 45 | for _, scraper := range m.scrapers { 46 | go func(scraper *Scraper) { 47 | defer wg.Done() 48 | scraper.Close() 49 | }(scraper) 50 | } 51 | m.scrapers = nil 52 | } 53 | 54 | // Update shuts down and reloads all scrapers from config. 55 | func (m *Manager) Update(conf *types.Config) { 56 | m.Reset() 57 | for _, group := range conf.TargetGroups { 58 | log := m.Log.With(zap.String("group", group.Group)) 59 | if err := m.loadGroup(group, log); err != nil { 60 | log.Error("Failed to load group", zap.Error(err)) 61 | } 62 | } 63 | for _, scraper := range m.scrapers { 64 | scraper.Start(m.res, conf.ScrapeInterval) 65 | } 66 | } 67 | 68 | func (m *Manager) loadGroup(group *types.TargetGroup, log *zap.Logger) error { 69 | disc, err := discovery.NewFromConfig(group) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | prober, err := NewProber(group) 75 | if err != nil { 76 | return err 77 | } 78 | 79 | scraper := NewScraper(prober, disc) 80 | scraper.Log = log 81 | m.scrapers = append(m.scrapers, scraper) 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /internal/scraper/prober.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scraper 16 | 17 | import ( 18 | "context" 19 | "crypto/tls" 20 | "net" 21 | "net/http" 22 | "net/url" 23 | "time" 24 | 25 | "go.blockdaemon.com/solana/cluster-manager/internal/fetch" 26 | "go.blockdaemon.com/solana/cluster-manager/types" 27 | ) 28 | 29 | // Prober checks snapshot info from Solana nodes. 30 | type Prober struct { 31 | client *http.Client 32 | scheme string 33 | apiPath string 34 | header http.Header 35 | } 36 | 37 | func NewProber(group *types.TargetGroup) (*Prober, error) { 38 | var tlsConfig *tls.Config 39 | if group.TLSConfig != nil { 40 | var err error 41 | tlsConfig, err = group.TLSConfig.Build() 42 | if err != nil { 43 | return nil, err 44 | } 45 | } 46 | 47 | header := make(http.Header) 48 | if group.BasicAuth != nil { 49 | group.BasicAuth.Apply(header) 50 | } 51 | if group.BearerAuth != nil { 52 | group.BearerAuth.Apply(header) 53 | } 54 | 55 | client := &http.Client{ 56 | Transport: &http.Transport{ 57 | Proxy: http.ProxyFromEnvironment, 58 | DialContext: (&net.Dialer{ 59 | Timeout: 5 * time.Second, 60 | KeepAlive: 5 * time.Second, 61 | }).DialContext, 62 | TLSClientConfig: tlsConfig, 63 | MaxIdleConnsPerHost: 1, 64 | MaxConnsPerHost: 3, 65 | IdleConnTimeout: 90 * time.Second, 66 | TLSHandshakeTimeout: 5 * time.Second, 67 | ExpectContinueTimeout: 1 * time.Second, 68 | ForceAttemptHTTP2: true, 69 | }, 70 | Timeout: 10 * time.Second, 71 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 72 | if len(via) == 1 { 73 | return nil 74 | } 75 | return http.ErrUseLastResponse 76 | }, 77 | } 78 | 79 | return &Prober{ 80 | client: client, 81 | scheme: group.Scheme, 82 | apiPath: group.APIPath, 83 | header: header, 84 | }, nil 85 | } 86 | 87 | // Probe fetches the snapshots of a single target. 88 | func (p *Prober) Probe(ctx context.Context, target string) ([]*types.SnapshotInfo, error) { 89 | u := url.URL{ 90 | Scheme: p.scheme, 91 | Host: target, 92 | Path: p.apiPath, 93 | } 94 | return fetch.NewSidecarClient(u.String()).ListSnapshots(ctx) 95 | } 96 | -------------------------------------------------------------------------------- /internal/scraper/scraper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package scraper 16 | 17 | import ( 18 | "context" 19 | "sync" 20 | "time" 21 | 22 | "go.blockdaemon.com/solana/cluster-manager/internal/discovery" 23 | "go.uber.org/zap" 24 | ) 25 | 26 | type Scraper struct { 27 | prober *Prober 28 | discoverer discovery.Discoverer 29 | rootCtx context.Context 30 | cancel context.CancelFunc 31 | wg sync.WaitGroup 32 | 33 | Log *zap.Logger 34 | } 35 | 36 | func NewScraper(prober *Prober, discoverer discovery.Discoverer) *Scraper { 37 | ctx, cancel := context.WithCancel(context.Background()) 38 | return &Scraper{ 39 | prober: prober, 40 | discoverer: discoverer, 41 | rootCtx: ctx, 42 | cancel: cancel, 43 | Log: zap.NewNop(), 44 | } 45 | } 46 | 47 | func (s *Scraper) Start(results chan<- ProbeResult, interval time.Duration) { 48 | s.wg.Add(1) 49 | go s.run(results, interval) 50 | } 51 | 52 | func (s *Scraper) Close() { 53 | s.cancel() 54 | s.wg.Wait() 55 | } 56 | 57 | func (s *Scraper) run(results chan<- ProbeResult, interval time.Duration) { 58 | s.Log.Info("Starting scraper") 59 | defer s.Log.Info("Stopping scraper") 60 | 61 | defer s.wg.Done() 62 | ticker := time.NewTicker(interval) 63 | defer ticker.Stop() 64 | for { 65 | ctx, cancel := context.WithCancel(s.rootCtx) 66 | go s.scrape(ctx, results) 67 | 68 | select { 69 | case <-s.rootCtx.Done(): 70 | cancel() 71 | return 72 | case <-ticker.C: 73 | cancel() 74 | } 75 | } 76 | } 77 | 78 | func (s *Scraper) scrape(ctx context.Context, results chan<- ProbeResult) { 79 | discoveryStart := time.Now() 80 | targets, err := s.discoverer.DiscoverTargets(ctx) 81 | if err != nil { 82 | s.Log.Error("Service discovery failed", zap.Error(err)) 83 | return 84 | } 85 | 86 | scrapeStart := time.Now() 87 | s.Log.Debug("Scrape starting", 88 | zap.Duration("discovery_duration", time.Since(discoveryStart)), 89 | zap.Int("num_targets", len(targets))) 90 | 91 | var wg sync.WaitGroup 92 | wg.Add(len(targets)) 93 | for _, target := range targets { 94 | go func(target string) { 95 | defer wg.Done() 96 | infos, err := s.prober.Probe(ctx, target) 97 | results <- ProbeResult{ 98 | Time: time.Now(), 99 | Target: target, 100 | Infos: infos, 101 | Err: err, 102 | } 103 | }(target) 104 | } 105 | wg.Wait() 106 | 107 | s.Log.Debug("Scrape finished", 108 | zap.Duration("scrape_duration", time.Since(scrapeStart))) 109 | } 110 | -------------------------------------------------------------------------------- /internal/sidecar/consensus.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sidecar 16 | 17 | import ( 18 | "io" 19 | "net/http" 20 | 21 | "github.com/gagliardetto/solana-go/rpc/ws" 22 | "github.com/gin-gonic/gin" 23 | "go.uber.org/zap" 24 | ) 25 | 26 | // ConsensusHandler implements the consensus-related sidecar API methods. 27 | type ConsensusHandler struct { 28 | RpcWsUrl string 29 | Log *zap.Logger 30 | } 31 | 32 | // NewConsensusHandler creates a new sidecar consensus API handler using the provided WS RPC and logger. 33 | func NewConsensusHandler(rpcWsUrl string, log *zap.Logger) *ConsensusHandler { 34 | return &ConsensusHandler{ 35 | RpcWsUrl: rpcWsUrl, 36 | Log: log, 37 | } 38 | } 39 | 40 | // RegisterHandlers registers this API with Gin web framework. 41 | func (h *ConsensusHandler) RegisterHandlers(group gin.IRoutes) { 42 | group.GET("/slot_updates", h.GetSlotUpdates) 43 | } 44 | 45 | // GetSlotUpdates streams RPC "slotsUpdatesSubscribe" events via SSE. 46 | func (h *ConsensusHandler) GetSlotUpdates(c *gin.Context) { 47 | ctx := c.Request.Context() 48 | 49 | conn, err := ws.Connect(ctx, h.RpcWsUrl) 50 | if err != nil { 51 | h.Log.Error("Failed to connect to Solana RPC WebSocket", zap.Error(err)) 52 | c.AbortWithStatus(http.StatusBadGateway) 53 | return 54 | } 55 | defer conn.Close() 56 | 57 | go func() { 58 | <-ctx.Done() 59 | conn.Close() 60 | }() 61 | 62 | slotUpdates, err := conn.SlotsUpdatesSubscribe() 63 | if err != nil { 64 | h.Log.Error("Failed to connect to subscribe to slot updates", zap.Error(err)) 65 | c.AbortWithStatus(http.StatusServiceUnavailable) 66 | return 67 | } 68 | defer slotUpdates.Unsubscribe() 69 | 70 | c.Stream(func(w io.Writer) bool { 71 | update, err := slotUpdates.Recv(c) 72 | if err != nil { 73 | h.Log.Error("Failed to receive slot update event", zap.Error(err)) 74 | return false 75 | } 76 | c.SSEvent("slot_update", update) 77 | return true 78 | }) 79 | } 80 | -------------------------------------------------------------------------------- /internal/sidecar/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package sidecar contains the Solana cluster sidecar logic. 16 | package sidecar 17 | 18 | import ( 19 | "errors" 20 | "io" 21 | "io/fs" 22 | "net/http" 23 | "os" 24 | 25 | "github.com/gin-gonic/gin" 26 | "go.blockdaemon.com/solana/cluster-manager/internal/ledger" 27 | "go.blockdaemon.com/solana/cluster-manager/types" 28 | "go.uber.org/zap" 29 | ) 30 | 31 | // SnapshotHandler implements the snapshot-related sidecar API methods. 32 | type SnapshotHandler struct { 33 | LedgerDir fs.FS 34 | Log *zap.Logger 35 | } 36 | 37 | // NewSnapshotHandler creates a new sidecar snapshot API handler using the provided ledger dir and logger. 38 | func NewSnapshotHandler(ledgerDir string, log *zap.Logger) *SnapshotHandler { 39 | return &SnapshotHandler{ 40 | LedgerDir: os.DirFS(ledgerDir), 41 | Log: log, 42 | } 43 | } 44 | 45 | // RegisterHandlers registers this API with Gin web framework. 46 | func (s *SnapshotHandler) RegisterHandlers(group gin.IRoutes) { 47 | group.GET("/snapshots", s.ListSnapshots) 48 | group.HEAD("/snapshot.tar.bz2", s.DownloadBestSnapshot) 49 | group.GET("/snapshot.tar.bz2", s.DownloadBestSnapshot) 50 | group.HEAD("/snapshot.tar.zst", s.DownloadBestSnapshot) 51 | group.GET("/snapshot.tar.zst", s.DownloadBestSnapshot) 52 | group.HEAD("/snapshot/:name", s.DownloadSnapshot) 53 | group.GET("/snapshot/:name", s.DownloadSnapshot) 54 | } 55 | 56 | // ListSnapshots is an API handler listing available snapshots on the node. 57 | func (s *SnapshotHandler) ListSnapshots(c *gin.Context) { 58 | infos, err := ledger.ListSnapshots(s.LedgerDir) 59 | if err != nil { 60 | s.Log.Error("Failed to list snapshots", zap.Error(err)) 61 | c.AbortWithStatus(http.StatusInternalServerError) 62 | return 63 | } 64 | if infos == nil { 65 | infos = make([]*types.SnapshotInfo, 0) 66 | } 67 | c.JSON(http.StatusOK, infos) 68 | } 69 | 70 | // DownloadBestSnapshot selects the best full snapshot and sends it to the client. 71 | func (s *SnapshotHandler) DownloadBestSnapshot(c *gin.Context) { 72 | files, err := ledger.ListSnapshotFiles(s.LedgerDir) 73 | if err != nil { 74 | s.Log.Error("Failed to list snapshot files", zap.Error(err)) 75 | c.AbortWithStatus(http.StatusInternalServerError) 76 | return 77 | } 78 | for _, file := range files { 79 | if file.IsFull() { 80 | s.serveSnapshot(c, file.FileName) 81 | return 82 | } 83 | } 84 | c.String(http.StatusAccepted, "no snapshot available") 85 | } 86 | 87 | // DownloadSnapshot sends a snapshot to the client. 88 | func (s *SnapshotHandler) DownloadSnapshot(c *gin.Context) { 89 | // Parse name and reject odd requests. 90 | name := c.Param("name") 91 | snapshot := ledger.ParseSnapshotFileName(name) 92 | if snapshot == nil { 93 | s.Log.Info("Ignoring snapshot download request due to odd name", zap.String("snapshot", name)) 94 | returnSnapshotNotFound(c) 95 | return 96 | } 97 | switch snapshot.Ext { 98 | case ".tar.bz2", ".tar.gz", ".tar.zst", ".tar.xz", ".tar": 99 | // ok 100 | default: 101 | s.Log.Info("Ignoring snapshot download request due to odd extension", zap.String("snapshot", name)) 102 | returnSnapshotNotFound(c) 103 | return 104 | } 105 | 106 | s.serveSnapshot(c, name) 107 | } 108 | 109 | func (s *SnapshotHandler) serveSnapshot(c *gin.Context, name string) { 110 | log := s.Log.With(zap.String("snapshot", name)) 111 | 112 | // Open file. 113 | baseFile, err := s.LedgerDir.Open(name) 114 | if errors.Is(err, fs.ErrNotExist) { 115 | log.Info("Requested snapshot not found") 116 | returnSnapshotNotFound(c) 117 | return 118 | } else if err != nil { 119 | log.Error("Failed to open file", zap.Error(err)) 120 | c.AbortWithStatus(http.StatusInternalServerError) 121 | return 122 | } 123 | defer baseFile.Close() 124 | snapFile, ok := baseFile.(io.ReadSeeker) 125 | if !ok { 126 | log.Error("Snapshot file is not an io.ReedSeeker") 127 | c.AbortWithStatus(http.StatusInternalServerError) 128 | return 129 | } 130 | 131 | info, err := baseFile.Stat() 132 | if err != nil { 133 | log.Warn("Stat failed on snapshot", zap.String("snapshot", name), zap.Error(err)) 134 | returnSnapshotNotFound(c) 135 | return 136 | } 137 | 138 | http.ServeContent(c.Writer, c.Request, name, info.ModTime(), snapFile) 139 | } 140 | 141 | func returnSnapshotNotFound(c *gin.Context) { 142 | c.String(http.StatusNotFound, "snapshot not found") 143 | } 144 | -------------------------------------------------------------------------------- /internal/sidecar/snapshot_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package sidecar 16 | 17 | import ( 18 | "net/http" 19 | "net/http/httptest" 20 | "testing" 21 | 22 | "github.com/gin-gonic/gin" 23 | "github.com/stretchr/testify/assert" 24 | "github.com/stretchr/testify/require" 25 | "go.uber.org/zap/zaptest" 26 | ) 27 | 28 | func newRouter(h *SnapshotHandler) http.Handler { 29 | router := gin.Default() 30 | h.RegisterHandlers(router) 31 | return router 32 | } 33 | 34 | func testRequest(h *SnapshotHandler, req *http.Request) *httptest.ResponseRecorder { 35 | router := newRouter(h) 36 | w := httptest.NewRecorder() 37 | router.ServeHTTP(w, req) 38 | return w 39 | } 40 | 41 | func TestHandler_ListSnapshots_Error(t *testing.T) { 42 | h := NewSnapshotHandler("???/some/nonexistent/path", zaptest.NewLogger(t)) 43 | 44 | req, err := http.NewRequest(http.MethodGet, "/snapshots", nil) 45 | require.NoError(t, err) 46 | 47 | res := testRequest(h, req) 48 | assert.Equal(t, http.StatusInternalServerError, res.Code) 49 | } 50 | -------------------------------------------------------------------------------- /internal/tracker/tracker.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package tracker contains the Solana cluster tracker logic. 16 | package tracker 17 | 18 | import ( 19 | "net/http" 20 | 21 | "github.com/gin-gonic/gin" 22 | "go.blockdaemon.com/solana/cluster-manager/internal/index" 23 | "go.blockdaemon.com/solana/cluster-manager/types" 24 | ) 25 | 26 | // Handler implements the tracker API methods. 27 | type Handler struct { 28 | DB *index.DB 29 | } 30 | 31 | // NewHandler creates a new tracker API using the provided database. 32 | func NewHandler(db *index.DB) *Handler { 33 | return &Handler{DB: db} 34 | } 35 | 36 | // RegisterHandlers registers this API with Gin web framework. 37 | func (h *Handler) RegisterHandlers(group gin.IRoutes) { 38 | group.GET("/snapshots", h.GetSnapshots) 39 | group.GET("/best_snapshots", h.GetBestSnapshots) 40 | } 41 | 42 | func (h *Handler) GetSnapshots(c *gin.Context) { 43 | c.JSON(http.StatusOK, h.DB.GetAllSnapshots()) 44 | } 45 | 46 | // GetBestSnapshots returns the currently available best snapshots. 47 | func (h *Handler) GetBestSnapshots(c *gin.Context) { 48 | var query struct { 49 | Max int `form:"max"` 50 | } 51 | if err := c.BindQuery(&query); err != nil { 52 | return 53 | } 54 | const maxItems = 25 55 | if query.Max < 0 || query.Max > 25 { 56 | query.Max = maxItems 57 | } 58 | entries := h.DB.GetBestSnapshots(query.Max) 59 | sources := make([]types.SnapshotSource, len(entries)) 60 | for i, entry := range entries { 61 | sources[i] = types.SnapshotSource{ 62 | SnapshotInfo: *entry.Info, 63 | Target: entry.Target, 64 | UpdatedAt: entry.UpdatedAt, 65 | } 66 | } 67 | c.JSON(http.StatusOK, sources) 68 | } 69 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "github.com/spf13/cobra" 19 | "go.blockdaemon.com/solana/cluster-manager/internal/cmd" 20 | ) 21 | 22 | func main() { 23 | cobra.CheckErr(cmd.Cmd.Execute()) 24 | } 25 | -------------------------------------------------------------------------------- /types/auth.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "crypto/tls" 19 | "crypto/x509" 20 | "encoding/base64" 21 | "encoding/gob" 22 | "fmt" 23 | "net/http" 24 | "os" 25 | ) 26 | 27 | type BasicAuth struct { 28 | Username string `json:"username" yaml:"username"` 29 | Password string `json:"password" yaml:"password"` 30 | } 31 | 32 | func (b *BasicAuth) Apply(header http.Header) { 33 | auth := b.Username + ":" + b.Password 34 | header.Add("authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth))) 35 | } 36 | 37 | type BearerAuth struct { 38 | Token string `json:"token" yaml:"token"` 39 | } 40 | 41 | func (b *BearerAuth) Apply(header http.Header) { 42 | header.Set("authorization", "Bearer "+b.Token) 43 | } 44 | 45 | type TLSConfig struct { 46 | CAFile string `json:"ca_file" yaml:"ca_file"` 47 | CertFile string `json:"cert_file" yaml:"cert_file"` 48 | KeyFile string `json:"key_file" yaml:"key_file"` 49 | InsecureSkipVerify bool `json:"insecure_skip_verify" yaml:"insecure_skip_verify"` 50 | } 51 | 52 | func init() { 53 | gob.Register(&TLSConfig{}) 54 | } 55 | 56 | func (t *TLSConfig) Build() (*tls.Config, error) { 57 | config := &tls.Config{ 58 | InsecureSkipVerify: t.InsecureSkipVerify, 59 | } 60 | if len(t.CAFile) > 0 { 61 | caBytes, err := os.ReadFile(t.CAFile) 62 | if err != nil { 63 | return nil, fmt.Errorf("failed to read CA file: %w", err) 64 | } 65 | caPool := x509.NewCertPool() 66 | if !caPool.AppendCertsFromPEM(caBytes) { 67 | return nil, fmt.Errorf("unable to load CA cert") 68 | } 69 | config.RootCAs = caPool 70 | } 71 | if len(t.CertFile) > 0 && len(t.KeyFile) == 0 { 72 | return nil, fmt.Errorf("TLS cert file given but key file missing") 73 | } else if len(t.CertFile) == 0 && len(t.KeyFile) > 0 { 74 | return nil, fmt.Errorf("TLS key file given but cert file missing") 75 | } else if len(t.CertFile) > 0 && len(t.KeyFile) > 0 { 76 | _, err := t.getClientCertificate(nil) 77 | if err != nil { 78 | return nil, fmt.Errorf("failed to load client cert and key: %w", err) 79 | } 80 | config.GetClientCertificate = t.getClientCertificate 81 | } 82 | return config, nil 83 | } 84 | 85 | func (t *TLSConfig) getClientCertificate(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) { 86 | cert, err := tls.LoadX509KeyPair(t.CertFile, t.KeyFile) 87 | if err != nil { 88 | return nil, err 89 | } 90 | return &cert, nil 91 | } 92 | -------------------------------------------------------------------------------- /types/auth_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "net/http" 19 | "testing" 20 | 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestBasicAuth_Apply(t *testing.T) { 25 | ba := BasicAuth{ 26 | Username: "Aladdin", 27 | Password: "open sesame", 28 | } 29 | header := http.Header{} 30 | ba.Apply(header) 31 | assert.Equal(t, "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==", header.Get("Authorization")) 32 | } 33 | 34 | func TestBearerAuth_Apply(t *testing.T) { 35 | ba := BearerAuth{Token: "123"} 36 | header := http.Header{} 37 | ba.Apply(header) 38 | assert.Equal(t, "Bearer 123", header.Get("Authorization")) 39 | } 40 | -------------------------------------------------------------------------------- /types/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "bufio" 19 | "context" 20 | "os" 21 | "strings" 22 | "time" 23 | 24 | "gopkg.in/yaml.v3" 25 | ) 26 | 27 | // Config describes the root-level config file. 28 | type Config struct { 29 | ScrapeInterval time.Duration `json:"scrape_interval" yaml:"scrape_interval"` 30 | TargetGroups []*TargetGroup `json:"target_groups" yaml:"target_groups"` 31 | } 32 | 33 | // LoadConfig reads the config object from the file system. 34 | func LoadConfig(filePath string) (*Config, error) { 35 | f, err := os.Open(filePath) 36 | if err != nil { 37 | return nil, err 38 | } 39 | defer f.Close() 40 | 41 | conf := new(Config) 42 | decoder := yaml.NewDecoder(f) 43 | decoder.KnownFields(true) 44 | confErr := decoder.Decode(conf) 45 | return conf, confErr 46 | } 47 | 48 | // TargetGroup explains how to retrieve snapshots from a group of Solana nodes. 49 | type TargetGroup struct { 50 | Group string `json:"group" yaml:"group"` 51 | Scheme string `json:"scheme" yaml:"scheme"` 52 | APIPath string `json:"api_path" yaml:"api_path"` 53 | BasicAuth *BasicAuth `json:"basic_auth" yaml:"basic_auth"` 54 | BearerAuth *BearerAuth `json:"bearer_auth" yaml:"bearer_auth"` 55 | TLSConfig *TLSConfig `json:"tls_config" yaml:"tls_config"` 56 | 57 | StaticTargets *StaticTargets `json:"static_targets" yaml:"static_targets"` 58 | FileTargets *FileTargets `json:"file_targets" yaml:"file_targets"` 59 | ConsulSDConfig *ConsulSDConfig `json:"consul_sd_config" yaml:"consul_sd_config"` 60 | } 61 | 62 | // StaticTargets is a hardcoded list of Solana nodes. 63 | type StaticTargets struct { 64 | Targets []string `json:"targets" yaml:"targets"` 65 | } 66 | 67 | func (s *StaticTargets) DiscoverTargets(_ context.Context) ([]string, error) { 68 | return s.Targets, nil 69 | } 70 | 71 | // FileTargets reads targets from a JSON file. 72 | type FileTargets struct { 73 | Path string `json:"path" yaml:"path"` 74 | } 75 | 76 | func (d *FileTargets) DiscoverTargets(_ context.Context) ([]string, error) { 77 | f, err := os.Open(d.Path) 78 | if err != nil { 79 | return nil, err 80 | } 81 | defer f.Close() 82 | 83 | var lines []string 84 | scn := bufio.NewScanner(f) 85 | for scn.Scan() { 86 | lines = append(lines, strings.TrimSpace(scn.Text())) 87 | } 88 | 89 | return lines, scn.Err() 90 | } 91 | 92 | // ConsulSDConfig configures Consul service discovery. 93 | type ConsulSDConfig struct { 94 | Server string `json:"host" yaml:"host"` 95 | Token string `json:"token" yaml:"token"` 96 | TokenFile string `json:"token_file" yaml:"token_file"` 97 | Datacenter string `json:"datacenter" yaml:"datacenter"` 98 | Service string `json:"service" yaml:"service"` 99 | Filter string `json:"filter" yaml:"filter"` 100 | } 101 | -------------------------------------------------------------------------------- /types/config_test.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "path/filepath" 5 | "runtime" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestLoadConfig(t *testing.T) { 14 | _, testFile, _, ok := runtime.Caller(0) 15 | assert.True(t, ok) 16 | exampleConfig := filepath.Join(filepath.Dir(testFile), "../example-config.yml") 17 | 18 | actual, err := LoadConfig(exampleConfig) 19 | require.NoError(t, err) 20 | 21 | expected := &Config{ 22 | ScrapeInterval: 15 * time.Second, 23 | TargetGroups: []*TargetGroup{ 24 | { 25 | Group: "mainnet", 26 | Scheme: "http", 27 | StaticTargets: &StaticTargets{ 28 | Targets: []string{ 29 | "solana-mainnet-1.example.org:8899", 30 | "solana-mainnet-2.example.org:8899", 31 | "solana-mainnet-3.example.org:8899", 32 | }, 33 | }, 34 | }, 35 | }, 36 | } 37 | 38 | assert.Equal(t, expected, actual) 39 | } 40 | -------------------------------------------------------------------------------- /types/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "bytes" 19 | "time" 20 | 21 | "github.com/gagliardetto/solana-go" 22 | ) 23 | 24 | // SnapshotSource describes a snapshot, and where to get it from. 25 | type SnapshotSource struct { 26 | SnapshotInfo 27 | Target string `json:"target"` 28 | UpdatedAt time.Time `json:"updated_at"` 29 | } 30 | 31 | // SnapshotInfo describes a snapshot. 32 | type SnapshotInfo struct { 33 | Slot uint64 `json:"slot"` 34 | Hash solana.Hash `json:"hash"` 35 | Files []*SnapshotFile `json:"files"` 36 | TotalSize uint64 `json:"size"` 37 | } 38 | 39 | // SnapshotFile is a file that makes up a snapshot (either full or incremental). 40 | type SnapshotFile struct { 41 | FileName string `json:"file_name"` 42 | Slot uint64 `json:"slot"` 43 | BaseSlot uint64 `json:"base_slot,omitempty"` 44 | Hash solana.Hash `json:"hash"` 45 | Ext string `json:"ext"` 46 | 47 | ModTime *time.Time `json:"mod_time,omitempty"` 48 | Size uint64 `json:"size,omitempty"` 49 | } 50 | 51 | // IsFull returns whether the snapshot is a full snapshot. 52 | func (s *SnapshotFile) IsFull() bool { 53 | return s.BaseSlot == 0 54 | } 55 | 56 | // Compare implements lexicographic ordering by (slot, base_slot, hash). 57 | func (s *SnapshotFile) Compare(o *SnapshotFile) int { 58 | if s.Slot < o.Slot { 59 | return -1 60 | } else if s.Slot > o.Slot { 61 | return +1 62 | } else if s.BaseSlot != 0 && o.BaseSlot == 0 { 63 | return -1 64 | } else if s.BaseSlot == 0 && o.BaseSlot != 0 { 65 | return +1 66 | } else if s.BaseSlot < o.BaseSlot { 67 | return -1 68 | } else if s.BaseSlot > o.BaseSlot { 69 | return +1 70 | } else { 71 | return bytes.Compare(s.Hash[:], o.Hash[:]) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /types/snapshot_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Blockdaemon Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/gagliardetto/solana-go" 21 | "github.com/stretchr/testify/assert" 22 | ) 23 | 24 | func TestSnapshotFile_Compare(t *testing.T) { 25 | // Dead left is worse/better than right 26 | const worsee = -1 27 | const sameee = 0 28 | const better = +1 29 | t.Run("DifferentSlot", func(t *testing.T) { 30 | assert.Equal(t, worsee, (&SnapshotFile{Slot: 10}).Compare(&SnapshotFile{Slot: 12})) 31 | assert.Equal(t, better, (&SnapshotFile{Slot: 10}).Compare(&SnapshotFile{Slot: 8})) 32 | }) 33 | t.Run("DifferentBaseSlot", func(t *testing.T) { 34 | assert.Equal(t, worsee, (&SnapshotFile{Slot: 10, BaseSlot: 10}).Compare(&SnapshotFile{Slot: 10, BaseSlot: 12})) 35 | assert.Equal(t, better, (&SnapshotFile{Slot: 10, BaseSlot: 10}).Compare(&SnapshotFile{Slot: 10, BaseSlot: 8})) 36 | }) 37 | t.Run("FullVsIncrementalSnap", func(t *testing.T) { 38 | assert.Equal(t, better, (&SnapshotFile{Slot: 10}).Compare(&SnapshotFile{Slot: 10, BaseSlot: 12})) 39 | assert.Equal(t, worsee, (&SnapshotFile{Slot: 10, BaseSlot: 12}).Compare(&SnapshotFile{Slot: 10})) 40 | }) 41 | t.Run("HashMismatch", func(t *testing.T) { 42 | assert.Equal(t, better, (&SnapshotFile{Slot: 10, Hash: solana.Hash{0x69}}).Compare(&SnapshotFile{Slot: 10, Hash: solana.Hash{0x68}})) 43 | assert.Equal(t, better, (&SnapshotFile{Slot: 10, BaseSlot: 12, Hash: solana.Hash{0x69}}).Compare(&SnapshotFile{Slot: 10, BaseSlot: 12, Hash: solana.Hash{0x68}})) 44 | assert.Equal(t, worsee, (&SnapshotFile{Slot: 10, Hash: solana.Hash{0x69}}).Compare(&SnapshotFile{Slot: 10, Hash: solana.Hash{0x70}})) 45 | assert.Equal(t, worsee, (&SnapshotFile{Slot: 10, BaseSlot: 12, Hash: solana.Hash{0x69}}).Compare(&SnapshotFile{Slot: 10, BaseSlot: 12, Hash: solana.Hash{0x70}})) 46 | }) 47 | t.Run("Same", func(t *testing.T) { 48 | assert.Equal(t, sameee, (&SnapshotFile{Slot: 10}).Compare(&SnapshotFile{Slot: 10})) 49 | }) 50 | } 51 | --------------------------------------------------------------------------------