├── .copywrite.hcl ├── .github ├── CODEOWNERS └── workflows │ ├── build.yaml │ ├── ci.yaml │ └── jira-sync.yml ├── .gitignore ├── .go-version ├── .release ├── ci.hcl ├── nomad-device-nvidia-artifacts.hcl ├── release-metadata.hcl └── security-scan.hcl ├── CHANGELOG.md ├── LICENSE ├── Makefile ├── README.md ├── cmd └── main.go ├── device.go ├── device_test.go ├── fingerprint.go ├── fingerprint_test.go ├── go.mod ├── go.sum ├── nvml ├── client.go ├── client_test.go ├── driver_default.go ├── driver_linux.go └── shared.go ├── scripts └── version.sh ├── stats.go ├── stats_test.go └── version ├── generate.sh └── version.go /.copywrite.hcl: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | schema_version = 1 5 | 6 | project { 7 | license = "MPL-2.0" 8 | copyright_year = 2024 9 | 10 | header_ignore = [ 11 | ".golangci.yaml", 12 | ".copywrite.hcl", 13 | ".github/**", 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @hashicorp/nomad-eng 2 | 3 | # release configuration 4 | /.release/ @hashicorp/nomad-eng 5 | /.github/workflows/build.yml @hashicorp/nomad-eng 6 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: [workflow_dispatch, push] 4 | 5 | env: 6 | PKG_NAME: "nomad-device-nvidia" 7 | 8 | jobs: 9 | get-go-version: 10 | name: "Determine Go toolchain version" 11 | runs-on: ubuntu-24.04 12 | outputs: 13 | go-version: ${{ steps.get-go-version.outputs.go-version }} 14 | steps: 15 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 16 | - name: Determine Go version 17 | id: get-go-version 18 | run: | 19 | echo "Building with Go $(cat .go-version)" 20 | echo "go-version=$(cat .go-version)" >> "$GITHUB_OUTPUT" 21 | 22 | get-product-version: 23 | runs-on: ubuntu-24.04 24 | outputs: 25 | product-version: ${{ steps.get-product-version.outputs.product-version }} 26 | steps: 27 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 28 | - name: get product version 29 | id: get-product-version 30 | run: | 31 | make version 32 | echo "product-version=$(make version)" >> "$GITHUB_OUTPUT" 33 | 34 | generate-metadata-file: 35 | needs: get-product-version 36 | runs-on: ubuntu-24.04 37 | outputs: 38 | filepath: ${{ steps.generate-metadata-file.outputs.filepath }} 39 | steps: 40 | - name: "Checkout directory" 41 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 42 | - name: Generate metadata file 43 | id: generate-metadata-file 44 | uses: hashicorp/actions-generate-metadata@fdbc8803a0e53bcbb912ddeee3808329033d6357 # v1 45 | with: 46 | version: ${{ needs.get-product-version.outputs.product-version }} 47 | product: ${{ env.PKG_NAME }} 48 | repositoryOwner: "hashicorp" 49 | - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 50 | if: ${{ !env.ACT }} 51 | with: 52 | name: metadata.json 53 | path: ${{ steps.generate-metadata-file.outputs.filepath }} 54 | 55 | build-linux: 56 | needs: 57 | - get-go-version 58 | - get-product-version 59 | runs-on: ubuntu-24.04 60 | strategy: 61 | matrix: 62 | goos: ["linux"] 63 | goarch: ["amd64"] 64 | fail-fast: true 65 | name: Go ${{ needs.get-go-version.outputs.go-version }} ${{ matrix.goos }} ${{ matrix.goarch }} build 66 | steps: 67 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 68 | - uses: hashicorp/setup-golang@36878950ae8f21c1bc25accaf67a4df88c29b01d # v3 69 | - name: Build 70 | env: 71 | GOOS: ${{ matrix.goos }} 72 | GOARCH: ${{ matrix.goarch }} 73 | run: | 74 | make dist/${{ matrix.goos }}_${{ matrix.goarch }}.zip 75 | mv \ 76 | dist/${{ matrix.goos }}_${{ matrix.goarch }}.zip \ 77 | ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip 78 | - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 79 | if: ${{ !env.ACT }} 80 | with: 81 | name: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip 82 | path: ${{ env.PKG_NAME }}_${{ needs.get-product-version.outputs.product-version }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip 83 | - name: Package 84 | if: ${{ matrix.goos == 'linux' }} 85 | uses: hashicorp/actions-packaging-linux@514d75d0961adeddf1f928fb93b82f41735fc488 # V1.6 86 | with: 87 | name: ${{ github.event.repository.name }} 88 | description: "nomad-device-nvidia is a HashiCorp Nomad device driver for Linux." 89 | arch: ${{ matrix.goarch }} 90 | version: ${{ needs.get-product-version.outputs.product-version }} 91 | maintainer: "HashiCorp" 92 | homepage: "https://github.com/hashicorp/nomad-device-nvidia" 93 | license: "MPL-2.0" 94 | binary: "dist/${{ matrix.goos }}_${{ matrix.goarch }}/${{ env.PKG_NAME }}" 95 | - name: Set Package Names 96 | if: ${{ matrix.goos == 'linux' }} 97 | run: | 98 | echo "RPM_PACKAGE=$(basename out/*.rpm)" >> "$GITHUB_ENV" 99 | echo "DEB_PACKAGE=$(basename out/*.deb)" >> "$GITHUB_ENV" 100 | - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 101 | if: ${{ matrix.goos == 'linux' }} 102 | with: 103 | name: ${{ env.RPM_PACKAGE }} 104 | path: out/${{ env.RPM_PACKAGE }} 105 | - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 106 | if: ${{ matrix.goos == 'linux' }} 107 | with: 108 | name: ${{ env.DEB_PACKAGE }} 109 | path: out/${{ env.DEB_PACKAGE }} 110 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: Run CI Tests 2 | on: 3 | pull_request: 4 | paths-ignore: 5 | - 'README.md' 6 | push: 7 | branches: 8 | - 'main' 9 | paths-ignore: 10 | - 'README.md' 11 | jobs: 12 | run-lint: 13 | timeout-minutes: 10 14 | runs-on: ubuntu-24.04 15 | steps: 16 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 17 | - uses: hashicorp/setup-golang@36878950ae8f21c1bc25accaf67a4df88c29b01d # v3 18 | with: 19 | version-file: go.mod 20 | - uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0 21 | with: 22 | version: v1.64.2 23 | skip-cache: true 24 | args: --timeout=9m 25 | run-copywrite: 26 | timeout-minutes: 10 27 | runs-on: ubuntu-24.04 28 | steps: 29 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 30 | - uses: hashicorp/setup-copywrite@32638da2d4e81d56a0764aa1547882fc4d209636 # v1.1.3 31 | - name: verify copywrite headers 32 | run: | 33 | copywrite --config .copywrite.hcl headers --spdx "MPL-2.0" --plan 34 | run-tests: 35 | timeout-minutes: 10 36 | strategy: 37 | fail-fast: false 38 | matrix: 39 | os: 40 | - ubuntu-22.04 41 | - ubuntu-24.04 42 | runs-on: ${{matrix.os}} 43 | steps: 44 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 45 | - uses: hashicorp/setup-golang@36878950ae8f21c1bc25accaf67a4df88c29b01d # v3 46 | with: 47 | version-file: go.mod 48 | - name: Run Go Test 49 | run: | 50 | make test 51 | permissions: 52 | contents: read 53 | -------------------------------------------------------------------------------- /.github/workflows/jira-sync.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | on: 5 | issues: 6 | types: [labeled, opened, closed, deleted, reopened] 7 | issue_comment: 8 | types: [created] 9 | workflow_dispatch: 10 | 11 | name: Jira Issue Sync 12 | 13 | jobs: 14 | sync: 15 | runs-on: ubuntu-latest 16 | name: Jira Issue sync 17 | steps: 18 | - name: Login 19 | uses: atlassian/gajira-login@45fd029b9f1d6d8926c6f04175aa80c0e42c9026 # v3.0.1 20 | env: 21 | JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} 22 | JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} 23 | JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} 24 | 25 | - name: Set ticket type 26 | id: set-ticket-type 27 | run: | 28 | echo "TYPE=GH Issue" >> "$GITHUB_OUTPUT" 29 | 30 | - name: Create ticket if an issue is labeled with hcc/jira 31 | if: github.event.action == 'labeled' && github.event.label.name == 'hcc/jira' 32 | uses: tomhjp/gh-action-jira-create@3ed1789cad3521292e591a7cfa703215ec1348bf # v0.2.1 33 | with: 34 | project: NMD 35 | issuetype: "${{ steps.set-ticket-type.outputs.TYPE }}" 36 | summary: "${{ github.event.repository.name }} [${{ steps.set-ticket-type.outputs.TYPE }} #${{ github.event.issue.number }}]: ${{ github.event.issue.title }}" 37 | description: "${{ github.event.issue.body || github.event.pull_request.body }}\n\n_Created in GitHub by ${{ github.actor }}._" 38 | # customfield_10089 is "Issue Link" 39 | # customfield_10371 is "Source" (use JIRA API to retrieve) 40 | # customerfield_10091 is "Team (R&D) 41 | # customfield_10001 is Team (jira default teams?) 42 | extraFields: '{ "customfield_10089": "${{ github.event.issue.html_url || github.event.pull_request.html_url }}", 43 | "customfield_10001": "72e166fb-d26c-4a61-b0de-7a290d91708f", 44 | "components": [{ "name": "nomad-device-nvidia" }], 45 | "labels": ["community", "GitHub"] }' 46 | env: 47 | JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} 48 | JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} 49 | JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} 50 | 51 | - name: Search 52 | if: github.event.action != 'opened' 53 | id: search 54 | uses: tomhjp/gh-action-jira-search@04700b457f317c3e341ce90da5a3ff4ce058f2fa # v0.2.2 55 | with: 56 | # cf[10089] is Issue Link (use JIRA API to retrieve) 57 | jql: 'cf[10089] = "${{ github.event.issue.html_url || github.event.pull_request.html_url }}"' 58 | 59 | - name: Sync comment 60 | if: github.event.action == 'created' && steps.search.outputs.issue 61 | uses: tomhjp/gh-action-jira-comment@6eb6b9ead70221916b6badd118c24535ed220bd9 # v0.2.0 62 | with: 63 | issue: ${{ steps.search.outputs.issue }} 64 | comment: "${{ github.actor }} ${{ github.event.review.state || 'commented' }}:\n\n${{ github.event.comment.body || github.event.review.body }}\n\n${{ github.event.comment.html_url || github.event.review.html_url }}" 65 | 66 | - name: Close ticket 67 | if: ( github.event.action == 'closed' || github.event.action == 'deleted' ) && steps.search.outputs.issue 68 | uses: atlassian/gajira-transition@38fc9cd61b03d6a53dd35fcccda172fe04b36de3 # v3.0.1 69 | with: 70 | issue: ${{ steps.search.outputs.issue }} 71 | transition: "Closed" 72 | 73 | - name: Reopen ticket 74 | if: github.event.action == 'reopened' && steps.search.outputs.issue 75 | uses: atlassian/gajira-transition@38fc9cd61b03d6a53dd35fcccda172fe04b36de3 # v3.0.1 76 | with: 77 | issue: ${{ steps.search.outputs.issue }} 78 | transition: "To Do" 79 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # Build output 18 | bin/ 19 | pkg/ 20 | nomad-device-nvidia 21 | -------------------------------------------------------------------------------- /.go-version: -------------------------------------------------------------------------------- 1 | 1.24.2 2 | -------------------------------------------------------------------------------- /.release/ci.hcl: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | schema = "1" 5 | 6 | project "nomad-device-nvidia" { 7 | team = "nomad" 8 | slack { 9 | notification_channel = "C03B5EWFW01" 10 | } 11 | github { 12 | organization = "hashicorp" 13 | repository = "nomad-device-nvidia" 14 | release_branches = [ 15 | "main", 16 | "release/**", 17 | ] 18 | } 19 | } 20 | 21 | event "merge" { 22 | // "entrypoint" to use if build is not run automatically 23 | // i.e. send "merge" complete signal to orchestrator to trigger build 24 | } 25 | 26 | event "build" { 27 | depends = ["merge"] 28 | action "build" { 29 | organization = "hashicorp" 30 | repository = "nomad-device-nvidia" 31 | workflow = "build" 32 | } 33 | } 34 | 35 | event "prepare" { 36 | depends = ["build"] 37 | action "prepare" { 38 | organization = "hashicorp" 39 | repository = "crt-workflows-common" 40 | workflow = "prepare" 41 | depends = ["build"] 42 | } 43 | 44 | notification { 45 | on = "fail" 46 | } 47 | } 48 | 49 | ## These are promotion and post-publish events 50 | ## they should be added to the end of the file after the verify event stanza. 51 | 52 | event "trigger-staging" { 53 | // This event is dispatched by the bob trigger-promotion command 54 | // and is required - do not delete. 55 | } 56 | 57 | event "promote-staging" { 58 | depends = ["trigger-staging"] 59 | action "promote-staging" { 60 | organization = "hashicorp" 61 | repository = "crt-workflows-common" 62 | workflow = "promote-staging" 63 | config = "release-metadata.hcl" 64 | } 65 | 66 | notification { 67 | on = "always" 68 | } 69 | } 70 | 71 | event "trigger-production" { 72 | // This event is dispatched by the bob trigger-promotion command 73 | // and is required - do not delete. 74 | } 75 | 76 | event "promote-production" { 77 | depends = ["trigger-production"] 78 | action "promote-production" { 79 | organization = "hashicorp" 80 | repository = "crt-workflows-common" 81 | workflow = "promote-production" 82 | } 83 | 84 | notification { 85 | on = "always" 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /.release/nomad-device-nvidia-artifacts.hcl: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | schema = 1 5 | artifacts { 6 | zip = [ 7 | "nomad-device-nvidia_${version}_linux_amd64.zip", 8 | ] 9 | rpm = [ 10 | "nomad-device-nvidia-${version_linux}-1.x86_64.rpm", 11 | ] 12 | deb = [ 13 | "nomad-device-nvidia_${version_linux}-1_amd64.deb", 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.release/release-metadata.hcl: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | url_license = "https://github.com/hashicorp/nomad-device-nvidia/blob/main/LICENSE" 5 | url_project_website = "https://www.nomadproject.io/plugins/devices/nvidia" 6 | url_source_repository = "https://github.com/hashicorp/nomad-device-nvidia" 7 | -------------------------------------------------------------------------------- /.release/security-scan.hcl: -------------------------------------------------------------------------------- 1 | # Copyright (c) HashiCorp, Inc. 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | binary { 5 | go_modules = true 6 | osv = true 7 | go_stdlib = true 8 | oss_index = false 9 | nvd = false 10 | 11 | secrets { 12 | all = true 13 | } 14 | 15 | # Triage items that are _safe_ to ignore here. Note that this list should be 16 | # periodically cleaned up to remove items that are no longer found by the scanner. 17 | triage { 18 | suppress { 19 | vulnerabilities = [ 20 | "GO-2025-3510", // TODO(dduzgun-security): false positive scan result, investigate why and fix 21 | "GO-2024-3262", // TODO(dduzgun-security): false positive scan result, investigate why and fix 22 | ] 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## UNRELEASED 2 | 3 | IMPROVEMENTS: 4 | * build: Updated Nomad to 1.10.0 5 | * build: Updated to Go 1.24.2 6 | 7 | ## 1.1.0 (August 22, 2024) 8 | 9 | IMPROVEMENTS: 10 | * driver: Added support for MIG instances. 11 | * build: Updated to Go 1.21.6 12 | * deps: Upgrade protobuf library to 1.33.0 to address CVE-2024-24786 13 | * deps: Upgrade Nomad to 1.8.3 14 | * deps: Upgrade go-nvml to v0.12.4-0 15 | 16 | ## 1.0.0 (October 14, 2021) 17 | 18 | * Initial realse of standalone Nomad Nvidia device plugin. 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 HashiCorp, Inc. 2 | 3 | Mozilla Public License Version 2.0 4 | ================================== 5 | 6 | 1. Definitions 7 | -------------- 8 | 9 | 1.1. "Contributor" 10 | means each individual or legal entity that creates, contributes to 11 | the creation of, or owns Covered Software. 12 | 13 | 1.2. "Contributor Version" 14 | means the combination of the Contributions of others (if any) used 15 | by a Contributor and that particular Contributor's Contribution. 16 | 17 | 1.3. "Contribution" 18 | means Covered Software of a particular Contributor. 19 | 20 | 1.4. "Covered Software" 21 | means Source Code Form to which the initial Contributor has attached 22 | the notice in Exhibit A, the Executable Form of such Source Code 23 | Form, and Modifications of such Source Code Form, in each case 24 | including portions thereof. 25 | 26 | 1.5. "Incompatible With Secondary Licenses" 27 | means 28 | 29 | (a) that the initial Contributor has attached the notice described 30 | in Exhibit B to the Covered Software; or 31 | 32 | (b) that the Covered Software was made available under the terms of 33 | version 1.1 or earlier of the License, but not also under the 34 | terms of a Secondary License. 35 | 36 | 1.6. "Executable Form" 37 | means any form of the work other than Source Code Form. 38 | 39 | 1.7. "Larger Work" 40 | means a work that combines Covered Software with other material, in 41 | a separate file or files, that is not Covered Software. 42 | 43 | 1.8. "License" 44 | means this document. 45 | 46 | 1.9. "Licensable" 47 | means having the right to grant, to the maximum extent possible, 48 | whether at the time of the initial grant or subsequently, any and 49 | all of the rights conveyed by this License. 50 | 51 | 1.10. "Modifications" 52 | means any of the following: 53 | 54 | (a) any file in Source Code Form that results from an addition to, 55 | deletion from, or modification of the contents of Covered 56 | Software; or 57 | 58 | (b) any new file in Source Code Form that contains any Covered 59 | Software. 60 | 61 | 1.11. "Patent Claims" of a Contributor 62 | means any patent claim(s), including without limitation, method, 63 | process, and apparatus claims, in any patent Licensable by such 64 | Contributor that would be infringed, but for the grant of the 65 | License, by the making, using, selling, offering for sale, having 66 | made, import, or transfer of either its Contributions or its 67 | Contributor Version. 68 | 69 | 1.12. "Secondary License" 70 | means either the GNU General Public License, Version 2.0, the GNU 71 | Lesser General Public License, Version 2.1, the GNU Affero General 72 | Public License, Version 3.0, or any later versions of those 73 | licenses. 74 | 75 | 1.13. "Source Code Form" 76 | means the form of the work preferred for making modifications. 77 | 78 | 1.14. "You" (or "Your") 79 | means an individual or a legal entity exercising rights under this 80 | License. For legal entities, "You" includes any entity that 81 | controls, is controlled by, or is under common control with You. For 82 | purposes of this definition, "control" means (a) the power, direct 83 | or indirect, to cause the direction or management of such entity, 84 | whether by contract or otherwise, or (b) ownership of more than 85 | fifty percent (50%) of the outstanding shares or beneficial 86 | ownership of such entity. 87 | 88 | 2. License Grants and Conditions 89 | -------------------------------- 90 | 91 | 2.1. Grants 92 | 93 | Each Contributor hereby grants You a world-wide, royalty-free, 94 | non-exclusive license: 95 | 96 | (a) under intellectual property rights (other than patent or trademark) 97 | Licensable by such Contributor to use, reproduce, make available, 98 | modify, display, perform, distribute, and otherwise exploit its 99 | Contributions, either on an unmodified basis, with Modifications, or 100 | as part of a Larger Work; and 101 | 102 | (b) under Patent Claims of such Contributor to make, use, sell, offer 103 | for sale, have made, import, and otherwise transfer either its 104 | Contributions or its Contributor Version. 105 | 106 | 2.2. Effective Date 107 | 108 | The licenses granted in Section 2.1 with respect to any Contribution 109 | become effective for each Contribution on the date the Contributor first 110 | distributes such Contribution. 111 | 112 | 2.3. Limitations on Grant Scope 113 | 114 | The licenses granted in this Section 2 are the only rights granted under 115 | this License. No additional rights or licenses will be implied from the 116 | distribution or licensing of Covered Software under this License. 117 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 118 | Contributor: 119 | 120 | (a) for any code that a Contributor has removed from Covered Software; 121 | or 122 | 123 | (b) for infringements caused by: (i) Your and any other third party's 124 | modifications of Covered Software, or (ii) the combination of its 125 | Contributions with other software (except as part of its Contributor 126 | Version); or 127 | 128 | (c) under Patent Claims infringed by Covered Software in the absence of 129 | its Contributions. 130 | 131 | This License does not grant any rights in the trademarks, service marks, 132 | or logos of any Contributor (except as may be necessary to comply with 133 | the notice requirements in Section 3.4). 134 | 135 | 2.4. Subsequent Licenses 136 | 137 | No Contributor makes additional grants as a result of Your choice to 138 | distribute the Covered Software under a subsequent version of this 139 | License (see Section 10.2) or under the terms of a Secondary License (if 140 | permitted under the terms of Section 3.3). 141 | 142 | 2.5. Representation 143 | 144 | Each Contributor represents that the Contributor believes its 145 | Contributions are its original creation(s) or it has sufficient rights 146 | to grant the rights to its Contributions conveyed by this License. 147 | 148 | 2.6. Fair Use 149 | 150 | This License is not intended to limit any rights You have under 151 | applicable copyright doctrines of fair use, fair dealing, or other 152 | equivalents. 153 | 154 | 2.7. Conditions 155 | 156 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 157 | in Section 2.1. 158 | 159 | 3. Responsibilities 160 | ------------------- 161 | 162 | 3.1. Distribution of Source Form 163 | 164 | All distribution of Covered Software in Source Code Form, including any 165 | Modifications that You create or to which You contribute, must be under 166 | the terms of this License. You must inform recipients that the Source 167 | Code Form of the Covered Software is governed by the terms of this 168 | License, and how they can obtain a copy of this License. You may not 169 | attempt to alter or restrict the recipients' rights in the Source Code 170 | Form. 171 | 172 | 3.2. Distribution of Executable Form 173 | 174 | If You distribute Covered Software in Executable Form then: 175 | 176 | (a) such Covered Software must also be made available in Source Code 177 | Form, as described in Section 3.1, and You must inform recipients of 178 | the Executable Form how they can obtain a copy of such Source Code 179 | Form by reasonable means in a timely manner, at a charge no more 180 | than the cost of distribution to the recipient; and 181 | 182 | (b) You may distribute such Executable Form under the terms of this 183 | License, or sublicense it under different terms, provided that the 184 | license for the Executable Form does not attempt to limit or alter 185 | the recipients' rights in the Source Code Form under this License. 186 | 187 | 3.3. Distribution of a Larger Work 188 | 189 | You may create and distribute a Larger Work under terms of Your choice, 190 | provided that You also comply with the requirements of this License for 191 | the Covered Software. If the Larger Work is a combination of Covered 192 | Software with a work governed by one or more Secondary Licenses, and the 193 | Covered Software is not Incompatible With Secondary Licenses, this 194 | License permits You to additionally distribute such Covered Software 195 | under the terms of such Secondary License(s), so that the recipient of 196 | the Larger Work may, at their option, further distribute the Covered 197 | Software under the terms of either this License or such Secondary 198 | License(s). 199 | 200 | 3.4. Notices 201 | 202 | You may not remove or alter the substance of any license notices 203 | (including copyright notices, patent notices, disclaimers of warranty, 204 | or limitations of liability) contained within the Source Code Form of 205 | the Covered Software, except that You may alter any license notices to 206 | the extent required to remedy known factual inaccuracies. 207 | 208 | 3.5. Application of Additional Terms 209 | 210 | You may choose to offer, and to charge a fee for, warranty, support, 211 | indemnity or liability obligations to one or more recipients of Covered 212 | Software. However, You may do so only on Your own behalf, and not on 213 | behalf of any Contributor. You must make it absolutely clear that any 214 | such warranty, support, indemnity, or liability obligation is offered by 215 | You alone, and You hereby agree to indemnify every Contributor for any 216 | liability incurred by such Contributor as a result of warranty, support, 217 | indemnity or liability terms You offer. You may include additional 218 | disclaimers of warranty and limitations of liability specific to any 219 | jurisdiction. 220 | 221 | 4. Inability to Comply Due to Statute or Regulation 222 | --------------------------------------------------- 223 | 224 | If it is impossible for You to comply with any of the terms of this 225 | License with respect to some or all of the Covered Software due to 226 | statute, judicial order, or regulation then You must: (a) comply with 227 | the terms of this License to the maximum extent possible; and (b) 228 | describe the limitations and the code they affect. Such description must 229 | be placed in a text file included with all distributions of the Covered 230 | Software under this License. Except to the extent prohibited by statute 231 | or regulation, such description must be sufficiently detailed for a 232 | recipient of ordinary skill to be able to understand it. 233 | 234 | 5. Termination 235 | -------------- 236 | 237 | 5.1. The rights granted under this License will terminate automatically 238 | if You fail to comply with any of its terms. However, if You become 239 | compliant, then the rights granted under this License from a particular 240 | Contributor are reinstated (a) provisionally, unless and until such 241 | Contributor explicitly and finally terminates Your grants, and (b) on an 242 | ongoing basis, if such Contributor fails to notify You of the 243 | non-compliance by some reasonable means prior to 60 days after You have 244 | come back into compliance. Moreover, Your grants from a particular 245 | Contributor are reinstated on an ongoing basis if such Contributor 246 | notifies You of the non-compliance by some reasonable means, this is the 247 | first time You have received notice of non-compliance with this License 248 | from such Contributor, and You become compliant prior to 30 days after 249 | Your receipt of the notice. 250 | 251 | 5.2. If You initiate litigation against any entity by asserting a patent 252 | infringement claim (excluding declaratory judgment actions, 253 | counter-claims, and cross-claims) alleging that a Contributor Version 254 | directly or indirectly infringes any patent, then the rights granted to 255 | You by any and all Contributors for the Covered Software under Section 256 | 2.1 of this License shall terminate. 257 | 258 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 259 | end user license agreements (excluding distributors and resellers) which 260 | have been validly granted by You or Your distributors under this License 261 | prior to termination shall survive termination. 262 | 263 | ************************************************************************ 264 | * * 265 | * 6. Disclaimer of Warranty * 266 | * ------------------------- * 267 | * * 268 | * Covered Software is provided under this License on an "as is" * 269 | * basis, without warranty of any kind, either expressed, implied, or * 270 | * statutory, including, without limitation, warranties that the * 271 | * Covered Software is free of defects, merchantable, fit for a * 272 | * particular purpose or non-infringing. The entire risk as to the * 273 | * quality and performance of the Covered Software is with You. * 274 | * Should any Covered Software prove defective in any respect, You * 275 | * (not any Contributor) assume the cost of any necessary servicing, * 276 | * repair, or correction. This disclaimer of warranty constitutes an * 277 | * essential part of this License. No use of any Covered Software is * 278 | * authorized under this License except under this disclaimer. * 279 | * * 280 | ************************************************************************ 281 | 282 | ************************************************************************ 283 | * * 284 | * 7. Limitation of Liability * 285 | * -------------------------- * 286 | * * 287 | * Under no circumstances and under no legal theory, whether tort * 288 | * (including negligence), contract, or otherwise, shall any * 289 | * Contributor, or anyone who distributes Covered Software as * 290 | * permitted above, be liable to You for any direct, indirect, * 291 | * special, incidental, or consequential damages of any character * 292 | * including, without limitation, damages for lost profits, loss of * 293 | * goodwill, work stoppage, computer failure or malfunction, or any * 294 | * and all other commercial damages or losses, even if such party * 295 | * shall have been informed of the possibility of such damages. This * 296 | * limitation of liability shall not apply to liability for death or * 297 | * personal injury resulting from such party's negligence to the * 298 | * extent applicable law prohibits such limitation. Some * 299 | * jurisdictions do not allow the exclusion or limitation of * 300 | * incidental or consequential damages, so this exclusion and * 301 | * limitation may not apply to You. * 302 | * * 303 | ************************************************************************ 304 | 305 | 8. Litigation 306 | ------------- 307 | 308 | Any litigation relating to this License may be brought only in the 309 | courts of a jurisdiction where the defendant maintains its principal 310 | place of business and such litigation shall be governed by laws of that 311 | jurisdiction, without reference to its conflict-of-law provisions. 312 | Nothing in this Section shall prevent a party's ability to bring 313 | cross-claims or counter-claims. 314 | 315 | 9. Miscellaneous 316 | ---------------- 317 | 318 | This License represents the complete agreement concerning the subject 319 | matter hereof. If any provision of this License is held to be 320 | unenforceable, such provision shall be reformed only to the extent 321 | necessary to make it enforceable. Any law or regulation which provides 322 | that the language of a contract shall be construed against the drafter 323 | shall not be used to construe this License against a Contributor. 324 | 325 | 10. Versions of the License 326 | --------------------------- 327 | 328 | 10.1. New Versions 329 | 330 | Mozilla Foundation is the license steward. Except as provided in Section 331 | 10.3, no one other than the license steward has the right to modify or 332 | publish new versions of this License. Each version will be given a 333 | distinguishing version number. 334 | 335 | 10.2. Effect of New Versions 336 | 337 | You may distribute the Covered Software under the terms of the version 338 | of the License under which You originally received the Covered Software, 339 | or under the terms of any subsequent version published by the license 340 | steward. 341 | 342 | 10.3. Modified Versions 343 | 344 | If you create software not governed by this License, and you want to 345 | create a new license for such software, you may create and use a 346 | modified version of this License if you rename the license and remove 347 | any references to the name of the license steward (except to note that 348 | such modified license differs from this License). 349 | 350 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 351 | Licenses 352 | 353 | If You choose to distribute Source Code Form that is Incompatible With 354 | Secondary Licenses under the terms of this version of the License, the 355 | notice described in Exhibit B of this License must be attached. 356 | 357 | Exhibit A - Source Code Form License Notice 358 | ------------------------------------------- 359 | 360 | This Source Code Form is subject to the terms of the Mozilla Public 361 | License, v. 2.0. If a copy of the MPL was not distributed with this 362 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 363 | 364 | If it is not possible or desirable to put the notice in a particular 365 | file, then You may include the notice in a location (such as a LICENSE 366 | file in a relevant directory) where a recipient would be likely to look 367 | for such a notice. 368 | 369 | You may add additional accurate notices of copyright ownership. 370 | 371 | Exhibit B - "Incompatible With Secondary Licenses" Notice 372 | --------------------------------------------------------- 373 | 374 | This Source Code Form is "Incompatible With Secondary Licenses", as 375 | defined by the Mozilla Public License, v. 2.0. 376 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL = bash 2 | 3 | NOMAD_PLUGIN_DIR ?= /tmp/nomad-plugins 4 | 5 | .PHONY: clean 6 | clean: 7 | @echo "==> Cleanup previous build" 8 | rm -f $(NOMAD_PLUGIN_DIR)/nomad-device-nvidia 9 | 10 | .PHONY: copywrite 11 | copywrite: 12 | @echo "==> Checking copywrite headers" 13 | copywrite --config .copywrite.hcl headers --spdx "MPL-2.0" 14 | 15 | .PHONY: compile 16 | compile: clean 17 | @echo "==> Compile nvidia driver plugin ..." 18 | mkdir -p $(NOMAD_PLUGIN_DIR) 19 | go build -race -trimpath -o $(NOMAD_PLUGIN_DIR)/nomad-device-nvidia cmd/main.go 20 | 21 | .PHONY: test 22 | test: 23 | @echo "==> Running tests ..." 24 | go test -v -race ./... 25 | 26 | .PHONY: lint 27 | lint: 28 | @echo "==> Lint nvidia driver ..." 29 | go vet ./... 30 | golangci-lint --timeout=5m run 31 | 32 | .PHONY: hack 33 | hack: compile 34 | hack: 35 | @echo "==> Run dev Nomad with nomad plugin" 36 | sudo nomad agent -dev -plugin-dir=$(NOMAD_PLUGIN_DIR) 37 | 38 | # CRT release compilation 39 | dist/%/nomad-device-nvidia: GO_OUT ?= $@ 40 | dist/%/nomad-device-nvidia: 41 | @echo "==> RELEASE BUILD of $@ ..." 42 | GOOS=linux GOARCH=$(lastword $(subst _, ,$*)) \ 43 | go build -trimpath -o $(GO_OUT) cmd/main.go 44 | 45 | # CRT release packaging (zip only) 46 | .PRECIOUS: dist/%/nomad-device-nvidia 47 | dist/%.zip: dist/%/nomad-device-nvidia 48 | @echo "==> RELEASE PACKAGING of $@ ..." 49 | @cp LICENSE $(dir $<)LICENSE.txt 50 | zip -j $@ $(dir $<)* 51 | 52 | # CRT version generation 53 | .PHONY: version 54 | version: 55 | @$(CURDIR)/version/generate.sh version/version.go version/version.go 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nomad Nvidia Device Plugin 2 | 3 | This repository provides an implementation of a Nomad 4 | [`device`](https://www.nomadproject.io/docs/job-specification/device) plugin 5 | for Nvidia GPUs. 6 | 7 | ## Behavior 8 | 9 | The Nvidia device plugin uses 10 | [NVML](https://github.com/NVIDIA/gpu-monitoring-tools) bindings to get data 11 | regarding available Nvidia devices and will expose them via 12 | [`Fingerprint`](https://www.nomadproject.io/docs/internals/plugins/devices#fingerprint-context-context-chan-fingerprintresponse-error) 13 | RPC. GPUs can be excluded from fingerprinting by setting the `ignored_gpu_ids` 14 | field (see below). Plugin sends statistics for fingerprinted devices periodically. 15 | 16 | 17 | The plugin detects whether the GPU has [`Multi-Instance GPU (MIG)`](https://www.nvidia.com/en-us/technologies/multi-instance-gpu/) enabled. 18 | When enabled all instances will be fingerprinted as individual GPUs that can be addressed accordingly. 19 | 20 | ## Config 21 | 22 | The plugin is configured in the Nomad client's 23 | [`plugin`](https://www.nomadproject.io/docs/configuration/plugin) block: 24 | 25 | ```hcl 26 | plugin "nvidia" { 27 | config { 28 | ignored_gpu_ids = ["uuid1", "uuid2"] 29 | fingerprint_period = "5s" 30 | } 31 | } 32 | ``` 33 | 34 | The valid configuration options are: 35 | 36 | * `ignored_gpu_ids` (`list(string)`: `[]`): list of GPU UUIDs strings that 37 | should not be exposed to nomad 38 | * `fingerprint_period` (`string`: `"1m"`): interval to repeat the fingerprint 39 | process to identify possible changes. 40 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | 9 | "github.com/hashicorp/go-hclog" 10 | "github.com/hashicorp/nomad-device-nvidia" 11 | "github.com/hashicorp/nomad/plugins" 12 | ) 13 | 14 | func main() { 15 | // Serve the plugin 16 | plugins.ServeCtx(factory) 17 | } 18 | 19 | // factory returns a new instance of the Nvidia GPU plugin 20 | func factory(ctx context.Context, log hclog.Logger) interface{} { 21 | return nvidia.NewNvidiaDevice(ctx, log) 22 | } 23 | -------------------------------------------------------------------------------- /device.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvidia 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/hashicorp/go-hclog" 14 | "github.com/hashicorp/nomad-device-nvidia/nvml" 15 | "github.com/hashicorp/nomad-device-nvidia/version" 16 | "github.com/hashicorp/nomad/helper/pluginutils/loader" 17 | "github.com/hashicorp/nomad/plugins/base" 18 | "github.com/hashicorp/nomad/plugins/device" 19 | "github.com/hashicorp/nomad/plugins/shared/hclspec" 20 | ) 21 | 22 | const ( 23 | // pluginName is the name of the plugin 24 | pluginName = "nvidia-gpu" 25 | 26 | // vendor is the vendor providing the devices 27 | vendor = "nvidia" 28 | 29 | // deviceType is the type of device being returned 30 | deviceType = device.DeviceTypeGPU 31 | 32 | // notAvailable value is returned to nomad server in case some properties were 33 | // undetected by nvml driver 34 | notAvailable = "N/A" 35 | 36 | // Nvidia-container-runtime environment variable names 37 | NvidiaVisibleDevices = "NVIDIA_VISIBLE_DEVICES" 38 | ) 39 | 40 | var ( 41 | // PluginID is the nvidia plugin metadata registered in the plugin 42 | // catalog. 43 | PluginID = loader.PluginID{ 44 | Name: pluginName, 45 | PluginType: base.PluginTypeDevice, 46 | } 47 | 48 | // PluginConfig is the nvidia factory function registered in the 49 | // plugin catalog. 50 | PluginConfig = &loader.InternalPluginConfig{ 51 | Factory: func(ctx context.Context, l hclog.Logger) interface{} { return NewNvidiaDevice(ctx, l) }, 52 | } 53 | 54 | // pluginInfo describes the plugin 55 | pluginInfo = &base.PluginInfoResponse{ 56 | Type: base.PluginTypeDevice, 57 | PluginApiVersions: []string{device.ApiVersion010}, 58 | PluginVersion: version.Version, 59 | Name: pluginName, 60 | } 61 | 62 | // configSpec is the specification of the plugin's configuration 63 | configSpec = hclspec.NewObject(map[string]*hclspec.Spec{ 64 | "enabled": hclspec.NewDefault( 65 | hclspec.NewAttr("enabled", "bool", false), 66 | hclspec.NewLiteral("true"), 67 | ), 68 | "ignored_gpu_ids": hclspec.NewDefault( 69 | hclspec.NewAttr("ignored_gpu_ids", "list(string)", false), 70 | hclspec.NewLiteral("[]"), 71 | ), 72 | "fingerprint_period": hclspec.NewDefault( 73 | hclspec.NewAttr("fingerprint_period", "string", false), 74 | hclspec.NewLiteral("\"1m\""), 75 | ), 76 | }) 77 | ) 78 | 79 | // Config contains configuration information for the plugin. 80 | type Config struct { 81 | Enabled bool `codec:"enabled"` 82 | IgnoredGPUIDs []string `codec:"ignored_gpu_ids"` 83 | FingerprintPeriod string `codec:"fingerprint_period"` 84 | } 85 | 86 | // NvidiaDevice contains all plugin specific data 87 | type NvidiaDevice struct { 88 | // enabled indicates whether the plugin should be enabled 89 | enabled bool 90 | 91 | // nvmlClient is used to get data from nvidia 92 | nvmlClient nvml.NvmlClient 93 | 94 | // initErr holds an error retrieved during 95 | // nvmlClient initialization 96 | initErr error 97 | 98 | // ignoredGPUIDs is a set of UUIDs that would not be exposed to nomad 99 | ignoredGPUIDs map[string]struct{} 100 | 101 | // fingerprintPeriod is how often we should call nvml to get list of devices 102 | fingerprintPeriod time.Duration 103 | 104 | // devices is the set of detected eligible devices 105 | devices map[string]struct{} 106 | deviceLock sync.RWMutex 107 | 108 | logger hclog.Logger 109 | } 110 | 111 | // NewNvidiaDevice returns a new nvidia device plugin. 112 | func NewNvidiaDevice(_ context.Context, log hclog.Logger) *NvidiaDevice { 113 | nvmlClient, err := nvml.NewNvmlClient() 114 | logger := log.Named(pluginName) 115 | if err != nil && err.Error() != nvml.UnavailableLib.Error() { 116 | logger.Error("unable to initialize Nvidia driver", "reason", err) 117 | } 118 | return &NvidiaDevice{ 119 | logger: logger, 120 | devices: make(map[string]struct{}), 121 | ignoredGPUIDs: make(map[string]struct{}), 122 | nvmlClient: nvmlClient, 123 | initErr: err, 124 | } 125 | } 126 | 127 | // PluginInfo returns information describing the plugin. 128 | func (d *NvidiaDevice) PluginInfo() (*base.PluginInfoResponse, error) { 129 | return pluginInfo, nil 130 | } 131 | 132 | // ConfigSchema returns the plugins configuration schema. 133 | func (d *NvidiaDevice) ConfigSchema() (*hclspec.Spec, error) { 134 | return configSpec, nil 135 | } 136 | 137 | // SetConfig is used to set the configuration of the plugin. 138 | func (d *NvidiaDevice) SetConfig(cfg *base.Config) error { 139 | var config Config 140 | if len(cfg.PluginConfig) != 0 { 141 | if err := base.MsgPackDecode(cfg.PluginConfig, &config); err != nil { 142 | return err 143 | } 144 | } 145 | 146 | d.enabled = config.Enabled 147 | 148 | for _, ignoredGPUId := range config.IgnoredGPUIDs { 149 | d.ignoredGPUIDs[ignoredGPUId] = struct{}{} 150 | } 151 | 152 | period, err := time.ParseDuration(config.FingerprintPeriod) 153 | if err != nil { 154 | return fmt.Errorf("failed to parse fingerprint period %q: %v", config.FingerprintPeriod, err) 155 | } 156 | d.fingerprintPeriod = period 157 | 158 | return nil 159 | } 160 | 161 | // Fingerprint streams detected devices. If device changes are detected or the 162 | // devices health changes, messages will be emitted. 163 | func (d *NvidiaDevice) Fingerprint(ctx context.Context) (<-chan *device.FingerprintResponse, error) { 164 | if !d.enabled { 165 | return nil, device.ErrPluginDisabled 166 | } 167 | 168 | outCh := make(chan *device.FingerprintResponse) 169 | go d.fingerprint(ctx, outCh) 170 | return outCh, nil 171 | } 172 | 173 | type reservationError struct { 174 | notExistingIDs []string 175 | } 176 | 177 | func (e *reservationError) Error() string { 178 | return fmt.Sprintf("unknown device IDs: %s", strings.Join(e.notExistingIDs, ",")) 179 | } 180 | 181 | // Reserve returns information on how to mount given devices. 182 | // Assumption is made that nomad server is responsible for correctness of 183 | // GPU allocations, handling tricky cases such as double-allocation of single GPU 184 | func (d *NvidiaDevice) Reserve(deviceIDs []string) (*device.ContainerReservation, error) { 185 | if len(deviceIDs) == 0 { 186 | return &device.ContainerReservation{}, nil 187 | } 188 | if !d.enabled { 189 | return nil, device.ErrPluginDisabled 190 | } 191 | 192 | // Due to the asynchronous nature of NvidiaPlugin, there is a possibility 193 | // of race condition 194 | // 195 | // Timeline: 196 | // 1 - fingerprint reports that GPU with id "1" is present 197 | // 2 - the following events happen at the same time: 198 | // a) server decides to allocate GPU with id "1" 199 | // b) fingerprint check reports that GPU with id "1" is no more present 200 | // 201 | // The latest and always valid version of fingerprinted ids are stored in 202 | // d.devices map. To avoid this race condition an error is returned if 203 | // any of provided deviceIDs is not found in d.devices map 204 | d.deviceLock.RLock() 205 | var notExistingIDs []string 206 | for _, id := range deviceIDs { 207 | if _, deviceIDExists := d.devices[id]; !deviceIDExists { 208 | notExistingIDs = append(notExistingIDs, id) 209 | } 210 | } 211 | d.deviceLock.RUnlock() 212 | if len(notExistingIDs) != 0 { 213 | return nil, &reservationError{notExistingIDs} 214 | } 215 | 216 | return &device.ContainerReservation{ 217 | Envs: map[string]string{ 218 | NvidiaVisibleDevices: strings.Join(deviceIDs, ","), 219 | }, 220 | }, nil 221 | } 222 | 223 | // Stats streams statistics for the detected devices. 224 | func (d *NvidiaDevice) Stats(ctx context.Context, interval time.Duration) (<-chan *device.StatsResponse, error) { 225 | if !d.enabled { 226 | return nil, device.ErrPluginDisabled 227 | } 228 | 229 | outCh := make(chan *device.StatsResponse) 230 | go d.stats(ctx, outCh, interval) 231 | return outCh, nil 232 | } 233 | -------------------------------------------------------------------------------- /device_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvidia 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/hashicorp/go-hclog" 10 | "github.com/hashicorp/nomad-device-nvidia/nvml" 11 | "github.com/hashicorp/nomad/plugins/device" 12 | "github.com/shoenig/test/must" 13 | ) 14 | 15 | type MockNvmlClient struct { 16 | FingerprintError error 17 | FingerprintResponseReturned *nvml.FingerprintData 18 | 19 | StatsError error 20 | StatsResponseReturned []*nvml.StatsData 21 | } 22 | 23 | func (c *MockNvmlClient) GetFingerprintData() (*nvml.FingerprintData, error) { 24 | return c.FingerprintResponseReturned, c.FingerprintError 25 | } 26 | 27 | func (c *MockNvmlClient) GetStatsData() ([]*nvml.StatsData, error) { 28 | return c.StatsResponseReturned, c.StatsError 29 | } 30 | 31 | func TestReserve(t *testing.T) { 32 | cases := []struct { 33 | Name string 34 | ExpectedReservation *device.ContainerReservation 35 | ExpectedError error 36 | Device *NvidiaDevice 37 | RequestedIDs []string 38 | }{ 39 | { 40 | Name: "All RequestedIDs are not managed by Device", 41 | ExpectedReservation: nil, 42 | ExpectedError: &reservationError{[]string{ 43 | "UUID1", 44 | "UUID2", 45 | "UUID3", 46 | }}, 47 | RequestedIDs: []string{ 48 | "UUID1", 49 | "UUID2", 50 | "UUID3", 51 | }, 52 | Device: &NvidiaDevice{ 53 | logger: hclog.NewNullLogger(), 54 | enabled: true, 55 | }, 56 | }, 57 | { 58 | Name: "Some RequestedIDs are not managed by Device", 59 | ExpectedReservation: nil, 60 | ExpectedError: &reservationError{[]string{ 61 | "UUID1", 62 | "UUID2", 63 | }}, 64 | RequestedIDs: []string{ 65 | "UUID1", 66 | "UUID2", 67 | "UUID3", 68 | }, 69 | Device: &NvidiaDevice{ 70 | devices: map[string]struct{}{ 71 | "UUID3": {}, 72 | }, 73 | logger: hclog.NewNullLogger(), 74 | enabled: true, 75 | }, 76 | }, 77 | { 78 | Name: "All RequestedIDs are managed by Device", 79 | ExpectedReservation: &device.ContainerReservation{ 80 | Envs: map[string]string{ 81 | NvidiaVisibleDevices: "UUID1,UUID2,UUID3", 82 | }, 83 | }, 84 | ExpectedError: nil, 85 | RequestedIDs: []string{ 86 | "UUID1", 87 | "UUID2", 88 | "UUID3", 89 | }, 90 | Device: &NvidiaDevice{ 91 | devices: map[string]struct{}{ 92 | "UUID1": {}, 93 | "UUID2": {}, 94 | "UUID3": {}, 95 | }, 96 | logger: hclog.NewNullLogger(), 97 | enabled: true, 98 | }, 99 | }, 100 | { 101 | Name: "No IDs requested", 102 | ExpectedReservation: &device.ContainerReservation{}, 103 | ExpectedError: nil, 104 | RequestedIDs: nil, 105 | Device: &NvidiaDevice{ 106 | devices: map[string]struct{}{ 107 | "UUID1": {}, 108 | "UUID2": {}, 109 | "UUID3": {}, 110 | }, 111 | logger: hclog.NewNullLogger(), 112 | enabled: true, 113 | }, 114 | }, 115 | { 116 | Name: "Device is disabled", 117 | ExpectedReservation: nil, 118 | ExpectedError: device.ErrPluginDisabled, 119 | RequestedIDs: []string{ 120 | "UUID1", 121 | "UUID2", 122 | "UUID3", 123 | }, 124 | Device: &NvidiaDevice{ 125 | devices: map[string]struct{}{ 126 | "UUID1": {}, 127 | "UUID2": {}, 128 | "UUID3": {}, 129 | }, 130 | logger: hclog.NewNullLogger(), 131 | enabled: false, 132 | }, 133 | }, 134 | } 135 | 136 | for _, c := range cases { 137 | t.Run(c.Name, func(t *testing.T) { 138 | actualReservation, actualError := c.Device.Reserve(c.RequestedIDs) 139 | must.Eq(t, c.ExpectedReservation, actualReservation) 140 | must.Eq(t, c.ExpectedError, actualError) 141 | }) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /fingerprint.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvidia 5 | 6 | import ( 7 | "context" 8 | "time" 9 | 10 | "github.com/hashicorp/nomad-device-nvidia/nvml" 11 | "github.com/hashicorp/nomad/helper/pointer" 12 | "github.com/hashicorp/nomad/plugins/device" 13 | "github.com/hashicorp/nomad/plugins/shared/structs" 14 | ) 15 | 16 | const ( 17 | // Attribute names and units for reporting Fingerprint output 18 | MemoryAttr = "memory" 19 | PowerAttr = "power" 20 | BAR1Attr = "bar1" 21 | DriverVersionAttr = "driver_version" 22 | CoresClockAttr = "cores_clock" 23 | MemoryClockAttr = "memory_clock" 24 | PCIBandwidthAttr = "pci_bandwidth" 25 | DisplayStateAttr = "display_state" 26 | PersistenceModeAttr = "persistence_mode" 27 | ) 28 | 29 | // fingerprint is the long running goroutine that detects hardware 30 | func (d *NvidiaDevice) fingerprint(ctx context.Context, devices chan<- *device.FingerprintResponse) { 31 | defer close(devices) 32 | 33 | if d.initErr != nil { 34 | if d.initErr.Error() != nvml.UnavailableLib.Error() { 35 | d.logger.Error("exiting fingerprinting due to problems with NVML loading", "error", d.initErr) 36 | devices <- device.NewFingerprintError(d.initErr) 37 | } 38 | 39 | // Just close the channel to let server know that there are no working 40 | // Nvidia GPU units 41 | return 42 | } 43 | 44 | // Create a timer that will fire immediately for the first detection 45 | ticker := time.NewTimer(0) 46 | 47 | for { 48 | select { 49 | case <-ctx.Done(): 50 | return 51 | case <-ticker.C: 52 | ticker.Reset(d.fingerprintPeriod) 53 | } 54 | d.writeFingerprintToChannel(devices) 55 | } 56 | } 57 | 58 | // writeFingerprintToChannel makes nvml call and writes response to channel 59 | func (d *NvidiaDevice) writeFingerprintToChannel(devices chan<- *device.FingerprintResponse) { 60 | fingerprintData, err := d.nvmlClient.GetFingerprintData() 61 | if err != nil { 62 | d.logger.Error("failed to get fingerprint nvidia devices", "error", err) 63 | devices <- device.NewFingerprintError(err) 64 | return 65 | } 66 | 67 | // ignore devices from fingerprint output 68 | fingerprintDevices := ignoreFingerprintedDevices(fingerprintData.Devices, d.ignoredGPUIDs) 69 | // check if any device health was updated or any device was added to host 70 | if !d.fingerprintChanged(fingerprintDevices) { 71 | return 72 | } 73 | 74 | commonAttributes := map[string]*structs.Attribute{ 75 | DriverVersionAttr: { 76 | String: pointer.Of(fingerprintData.DriverVersion), 77 | }, 78 | } 79 | 80 | // Group all FingerprintDevices by DeviceName attribute 81 | deviceListByDeviceName := make(map[string][]*nvml.FingerprintDeviceData) 82 | for _, device := range fingerprintDevices { 83 | deviceName := device.DeviceName 84 | if deviceName == nil { 85 | // nvml driver was not able to detect device name. This kind 86 | // of devices are placed to single group with 'notAvailable' name 87 | notAvailableCopy := notAvailable 88 | deviceName = ¬AvailableCopy 89 | } 90 | 91 | deviceListByDeviceName[*deviceName] = append(deviceListByDeviceName[*deviceName], device) 92 | } 93 | 94 | // Build Fingerprint response with computed groups and send it over the channel 95 | deviceGroups := make([]*device.DeviceGroup, 0, len(deviceListByDeviceName)) 96 | for groupName, devices := range deviceListByDeviceName { 97 | deviceGroups = append(deviceGroups, deviceGroupFromFingerprintData(groupName, devices, commonAttributes)) 98 | } 99 | devices <- device.NewFingerprint(deviceGroups...) 100 | } 101 | 102 | // ignoreFingerprintedDevices excludes ignored devices from fingerprint output 103 | func ignoreFingerprintedDevices(deviceData []*nvml.FingerprintDeviceData, ignoredGPUIDs map[string]struct{}) []*nvml.FingerprintDeviceData { 104 | var result []*nvml.FingerprintDeviceData 105 | for _, fingerprintDevice := range deviceData { 106 | if _, ignored := ignoredGPUIDs[fingerprintDevice.UUID]; !ignored { 107 | result = append(result, fingerprintDevice) 108 | } 109 | } 110 | return result 111 | } 112 | 113 | // fingerprintChanged checks if there are any previously unseen nvidia devices located 114 | // or any of fingerprinted nvidia devices disappeared since the last fingerprint run. 115 | // Also, this func updates device map on NvidiaDevice with the latest data 116 | func (d *NvidiaDevice) fingerprintChanged(allDevices []*nvml.FingerprintDeviceData) bool { 117 | d.deviceLock.Lock() 118 | defer d.deviceLock.Unlock() 119 | 120 | changeDetected := false 121 | // check if every device in allDevices is in d.devices 122 | for _, device := range allDevices { 123 | if _, ok := d.devices[device.UUID]; !ok { 124 | changeDetected = true 125 | } 126 | } 127 | 128 | // check if every device in d.devices is in allDevices 129 | fingerprintDeviceMap := make(map[string]struct{}) 130 | for _, device := range allDevices { 131 | fingerprintDeviceMap[device.UUID] = struct{}{} 132 | } 133 | for id := range d.devices { 134 | if _, ok := fingerprintDeviceMap[id]; !ok { 135 | changeDetected = true 136 | } 137 | } 138 | 139 | d.devices = fingerprintDeviceMap 140 | return changeDetected 141 | } 142 | 143 | // deviceGroupFromFingerprintData composes deviceGroup from FingerprintDeviceData slice 144 | func deviceGroupFromFingerprintData(groupName string, deviceList []*nvml.FingerprintDeviceData, commonAttributes map[string]*structs.Attribute) *device.DeviceGroup { 145 | // deviceGroup without devices makes no sense -> return nil when no devices are provided 146 | if len(deviceList) == 0 { 147 | return nil 148 | } 149 | 150 | devices := make([]*device.Device, len(deviceList)) 151 | for index, dev := range deviceList { 152 | devices[index] = &device.Device{ 153 | ID: dev.UUID, 154 | // all fingerprinted devices are "healthy" for now 155 | // to get real health data -> dcgm bindings should be used 156 | Healthy: true, 157 | HwLocality: &device.DeviceLocality{ 158 | PciBusID: dev.PCIBusID, 159 | }, 160 | } 161 | } 162 | 163 | deviceGroup := &device.DeviceGroup{ 164 | Vendor: vendor, 165 | Type: deviceType, 166 | Name: groupName, 167 | Devices: devices, 168 | // Assumption made that devices with the same DeviceName have the same 169 | // attributes like amount of memory, power, bar1memory etc 170 | Attributes: attributesFromFingerprintDeviceData(deviceList[0]), 171 | } 172 | 173 | // Extend attribute map with common attributes 174 | for attributeKey, attributeValue := range commonAttributes { 175 | deviceGroup.Attributes[attributeKey] = attributeValue 176 | } 177 | 178 | return deviceGroup 179 | } 180 | 181 | // attributesFromFingerprintDeviceData converts nvml.FingerprintDeviceData 182 | // struct to device.DeviceGroup.Attributes format (map[string]string) 183 | // this function performs all nil checks for FingerprintDeviceData pointers 184 | func attributesFromFingerprintDeviceData(d *nvml.FingerprintDeviceData) map[string]*structs.Attribute { 185 | attrs := map[string]*structs.Attribute{ 186 | DisplayStateAttr: { 187 | String: pointer.Of(d.DisplayState), 188 | }, 189 | PersistenceModeAttr: { 190 | String: pointer.Of(d.PersistenceMode), 191 | }, 192 | } 193 | 194 | if d.MemoryMiB != nil { 195 | attrs[MemoryAttr] = &structs.Attribute{ 196 | Int: pointer.Of(int64(*d.MemoryMiB)), 197 | Unit: structs.UnitMiB, 198 | } 199 | } 200 | if d.PowerW != nil { 201 | attrs[PowerAttr] = &structs.Attribute{ 202 | Int: pointer.Of(int64(*d.PowerW)), 203 | Unit: structs.UnitW, 204 | } 205 | } 206 | if d.BAR1MiB != nil { 207 | attrs[BAR1Attr] = &structs.Attribute{ 208 | Int: pointer.Of(int64(*d.BAR1MiB)), 209 | Unit: structs.UnitMiB, 210 | } 211 | } 212 | if d.CoresClockMHz != nil { 213 | attrs[CoresClockAttr] = &structs.Attribute{ 214 | Int: pointer.Of(int64(*d.CoresClockMHz)), 215 | Unit: structs.UnitMHz, 216 | } 217 | } 218 | if d.MemoryClockMHz != nil { 219 | attrs[MemoryClockAttr] = &structs.Attribute{ 220 | Int: pointer.Of(int64(*d.MemoryClockMHz)), 221 | Unit: structs.UnitMHz, 222 | } 223 | } 224 | if d.PCIBandwidthMBPerS != nil { 225 | attrs[PCIBandwidthAttr] = &structs.Attribute{ 226 | Int: pointer.Of(int64(*d.PCIBandwidthMBPerS)), 227 | Unit: structs.UnitMBPerS, 228 | } 229 | } 230 | 231 | return attrs 232 | } 233 | -------------------------------------------------------------------------------- /fingerprint_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvidia 5 | 6 | import ( 7 | "context" 8 | "errors" 9 | "sort" 10 | "testing" 11 | 12 | hclog "github.com/hashicorp/go-hclog" 13 | "github.com/hashicorp/nomad-device-nvidia/nvml" 14 | "github.com/hashicorp/nomad/helper/pointer" 15 | "github.com/hashicorp/nomad/plugins/device" 16 | "github.com/hashicorp/nomad/plugins/shared/structs" 17 | "github.com/shoenig/test/must" 18 | ) 19 | 20 | func TestIgnoreFingerprintedDevices(t *testing.T) { 21 | for _, testCase := range []struct { 22 | Name string 23 | DeviceData []*nvml.FingerprintDeviceData 24 | IgnoredGPUIds map[string]struct{} 25 | ExpectedResult []*nvml.FingerprintDeviceData 26 | }{ 27 | { 28 | Name: "Odd ignored", 29 | DeviceData: []*nvml.FingerprintDeviceData{ 30 | { 31 | DeviceData: &nvml.DeviceData{ 32 | DeviceName: pointer.Of("DeviceName1"), 33 | UUID: "UUID1", 34 | MemoryMiB: pointer.Of(uint64(1000)), 35 | }, 36 | }, 37 | { 38 | DeviceData: &nvml.DeviceData{ 39 | DeviceName: pointer.Of("DeviceName2"), 40 | UUID: "UUID2", 41 | MemoryMiB: pointer.Of(uint64(1000)), 42 | }, 43 | }, 44 | { 45 | DeviceData: &nvml.DeviceData{ 46 | DeviceName: pointer.Of("DeviceName3"), 47 | UUID: "UUID3", 48 | MemoryMiB: pointer.Of(uint64(1000)), 49 | }, 50 | }, 51 | }, 52 | IgnoredGPUIds: map[string]struct{}{ 53 | "UUID2": {}, 54 | }, 55 | ExpectedResult: []*nvml.FingerprintDeviceData{ 56 | { 57 | DeviceData: &nvml.DeviceData{ 58 | DeviceName: pointer.Of("DeviceName1"), 59 | UUID: "UUID1", 60 | MemoryMiB: pointer.Of(uint64(1000)), 61 | }, 62 | }, 63 | { 64 | DeviceData: &nvml.DeviceData{ 65 | DeviceName: pointer.Of("DeviceName3"), 66 | UUID: "UUID3", 67 | MemoryMiB: pointer.Of(uint64(1000)), 68 | }, 69 | }, 70 | }, 71 | }, 72 | { 73 | Name: "Even ignored", 74 | DeviceData: []*nvml.FingerprintDeviceData{ 75 | { 76 | DeviceData: &nvml.DeviceData{ 77 | DeviceName: pointer.Of("DeviceName1"), 78 | UUID: "UUID1", 79 | MemoryMiB: pointer.Of(uint64(1000)), 80 | }, 81 | }, 82 | { 83 | DeviceData: &nvml.DeviceData{ 84 | DeviceName: pointer.Of("DeviceName2"), 85 | UUID: "UUID2", 86 | MemoryMiB: pointer.Of(uint64(1000)), 87 | }, 88 | }, 89 | { 90 | DeviceData: &nvml.DeviceData{ 91 | DeviceName: pointer.Of("DeviceName3"), 92 | UUID: "UUID3", 93 | MemoryMiB: pointer.Of(uint64(1000)), 94 | }, 95 | }, 96 | }, 97 | IgnoredGPUIds: map[string]struct{}{ 98 | "UUID1": {}, 99 | "UUID3": {}, 100 | }, 101 | ExpectedResult: []*nvml.FingerprintDeviceData{ 102 | { 103 | DeviceData: &nvml.DeviceData{ 104 | DeviceName: pointer.Of("DeviceName2"), 105 | UUID: "UUID2", 106 | MemoryMiB: pointer.Of(uint64(1000)), 107 | }, 108 | }, 109 | }, 110 | }, 111 | { 112 | Name: "All ignored", 113 | DeviceData: []*nvml.FingerprintDeviceData{ 114 | { 115 | DeviceData: &nvml.DeviceData{ 116 | DeviceName: pointer.Of("DeviceName1"), 117 | UUID: "UUID1", 118 | MemoryMiB: pointer.Of(uint64(1000)), 119 | }, 120 | }, 121 | { 122 | DeviceData: &nvml.DeviceData{ 123 | DeviceName: pointer.Of("DeviceName2"), 124 | UUID: "UUID2", 125 | MemoryMiB: pointer.Of(uint64(1000)), 126 | }, 127 | }, 128 | { 129 | DeviceData: &nvml.DeviceData{ 130 | DeviceName: pointer.Of("DeviceName3"), 131 | UUID: "UUID3", 132 | MemoryMiB: pointer.Of(uint64(1000)), 133 | }, 134 | }, 135 | }, 136 | IgnoredGPUIds: map[string]struct{}{ 137 | "UUID1": {}, 138 | "UUID2": {}, 139 | "UUID3": {}, 140 | }, 141 | ExpectedResult: nil, 142 | }, 143 | { 144 | Name: "No ignored", 145 | DeviceData: []*nvml.FingerprintDeviceData{ 146 | { 147 | DeviceData: &nvml.DeviceData{ 148 | DeviceName: pointer.Of("DeviceName1"), 149 | UUID: "UUID1", 150 | MemoryMiB: pointer.Of(uint64(1000)), 151 | }, 152 | }, 153 | { 154 | DeviceData: &nvml.DeviceData{ 155 | DeviceName: pointer.Of("DeviceName2"), 156 | UUID: "UUID2", 157 | MemoryMiB: pointer.Of(uint64(1000)), 158 | }, 159 | }, 160 | { 161 | DeviceData: &nvml.DeviceData{ 162 | DeviceName: pointer.Of("DeviceName3"), 163 | UUID: "UUID3", 164 | MemoryMiB: pointer.Of(uint64(1000)), 165 | }, 166 | }, 167 | }, 168 | IgnoredGPUIds: map[string]struct{}{}, 169 | ExpectedResult: []*nvml.FingerprintDeviceData{ 170 | { 171 | DeviceData: &nvml.DeviceData{ 172 | DeviceName: pointer.Of("DeviceName1"), 173 | UUID: "UUID1", 174 | MemoryMiB: pointer.Of(uint64(1000)), 175 | }, 176 | }, 177 | { 178 | DeviceData: &nvml.DeviceData{ 179 | DeviceName: pointer.Of("DeviceName2"), 180 | UUID: "UUID2", 181 | MemoryMiB: pointer.Of(uint64(1000)), 182 | }, 183 | }, 184 | { 185 | DeviceData: &nvml.DeviceData{ 186 | DeviceName: pointer.Of("DeviceName3"), 187 | UUID: "UUID3", 188 | MemoryMiB: pointer.Of(uint64(1000)), 189 | }, 190 | }, 191 | }, 192 | }, 193 | { 194 | Name: "No DeviceData provided", 195 | DeviceData: nil, 196 | IgnoredGPUIds: map[string]struct{}{ 197 | "UUID1": {}, 198 | "UUID2": {}, 199 | "UUID3": {}, 200 | }, 201 | ExpectedResult: nil, 202 | }, 203 | } { 204 | t.Run(testCase.Name, func(t *testing.T) { 205 | actualResult := ignoreFingerprintedDevices(testCase.DeviceData, testCase.IgnoredGPUIds) 206 | must.Eq(t, testCase.ExpectedResult, actualResult) 207 | }) 208 | } 209 | } 210 | 211 | func TestCheckFingerprintUpdates(t *testing.T) { 212 | for _, testCase := range []struct { 213 | Name string 214 | Device *NvidiaDevice 215 | AllDevices []*nvml.FingerprintDeviceData 216 | DeviceMapAfterMethodCall map[string]struct{} 217 | ExpectedResult bool 218 | }{ 219 | { 220 | Name: "No updates", 221 | Device: &NvidiaDevice{devices: map[string]struct{}{ 222 | "1": {}, 223 | "2": {}, 224 | "3": {}, 225 | }}, 226 | AllDevices: []*nvml.FingerprintDeviceData{ 227 | { 228 | DeviceData: &nvml.DeviceData{ 229 | UUID: "1", 230 | }, 231 | }, 232 | { 233 | DeviceData: &nvml.DeviceData{ 234 | UUID: "2", 235 | }, 236 | }, 237 | { 238 | DeviceData: &nvml.DeviceData{ 239 | UUID: "3", 240 | }, 241 | }, 242 | }, 243 | ExpectedResult: false, 244 | DeviceMapAfterMethodCall: map[string]struct{}{ 245 | "1": {}, 246 | "2": {}, 247 | "3": {}, 248 | }, 249 | }, 250 | { 251 | Name: "New Device Appeared", 252 | Device: &NvidiaDevice{devices: map[string]struct{}{ 253 | "1": {}, 254 | "2": {}, 255 | "3": {}, 256 | }}, 257 | AllDevices: []*nvml.FingerprintDeviceData{ 258 | { 259 | DeviceData: &nvml.DeviceData{ 260 | UUID: "1", 261 | }, 262 | }, 263 | { 264 | DeviceData: &nvml.DeviceData{ 265 | UUID: "2", 266 | }, 267 | }, 268 | { 269 | DeviceData: &nvml.DeviceData{ 270 | UUID: "3", 271 | }, 272 | }, 273 | { 274 | DeviceData: &nvml.DeviceData{ 275 | UUID: "I am new", 276 | }, 277 | }, 278 | }, 279 | ExpectedResult: true, 280 | DeviceMapAfterMethodCall: map[string]struct{}{ 281 | "1": {}, 282 | "2": {}, 283 | "3": {}, 284 | "I am new": {}, 285 | }, 286 | }, 287 | { 288 | Name: "Device disappeared", 289 | Device: &NvidiaDevice{devices: map[string]struct{}{ 290 | "1": {}, 291 | "2": {}, 292 | "3": {}, 293 | }}, 294 | AllDevices: []*nvml.FingerprintDeviceData{ 295 | { 296 | DeviceData: &nvml.DeviceData{ 297 | UUID: "1", 298 | }, 299 | }, 300 | { 301 | DeviceData: &nvml.DeviceData{ 302 | UUID: "2", 303 | }, 304 | }, 305 | }, 306 | ExpectedResult: true, 307 | DeviceMapAfterMethodCall: map[string]struct{}{ 308 | "1": {}, 309 | "2": {}, 310 | }, 311 | }, 312 | { 313 | Name: "No devices in NvidiaDevice map", 314 | Device: &NvidiaDevice{}, 315 | AllDevices: []*nvml.FingerprintDeviceData{ 316 | { 317 | DeviceData: &nvml.DeviceData{ 318 | UUID: "1", 319 | }, 320 | }, 321 | { 322 | DeviceData: &nvml.DeviceData{ 323 | UUID: "2", 324 | }, 325 | }, 326 | { 327 | DeviceData: &nvml.DeviceData{ 328 | UUID: "3", 329 | }, 330 | }, 331 | }, 332 | ExpectedResult: true, 333 | DeviceMapAfterMethodCall: map[string]struct{}{ 334 | "1": {}, 335 | "2": {}, 336 | "3": {}, 337 | }, 338 | }, 339 | { 340 | Name: "No devices detected", 341 | Device: &NvidiaDevice{devices: map[string]struct{}{ 342 | "1": {}, 343 | "2": {}, 344 | "3": {}, 345 | }}, 346 | AllDevices: nil, 347 | ExpectedResult: true, 348 | DeviceMapAfterMethodCall: map[string]struct{}{}, 349 | }, 350 | } { 351 | t.Run(testCase.Name, func(t *testing.T) { 352 | actualResult := testCase.Device.fingerprintChanged(testCase.AllDevices) 353 | // check that function returns valid "updated / not updated" state 354 | must.Eq(t, testCase.ExpectedResult, actualResult) 355 | // check that function propely updates devices map 356 | must.Eq(t, testCase.Device.devices, testCase.DeviceMapAfterMethodCall) 357 | }) 358 | } 359 | } 360 | 361 | func TestAttributesFromFingerprintDeviceData(t *testing.T) { 362 | for _, testCase := range []struct { 363 | Name string 364 | FingerprintDeviceData *nvml.FingerprintDeviceData 365 | ExpectedResult map[string]*structs.Attribute 366 | }{ 367 | { 368 | Name: "All attributes are not nil", 369 | FingerprintDeviceData: &nvml.FingerprintDeviceData{ 370 | DeviceData: &nvml.DeviceData{ 371 | UUID: "1", 372 | DeviceName: pointer.Of("Type1"), 373 | MemoryMiB: pointer.Of(uint64(256)), 374 | PowerW: pointer.Of(uint(2)), 375 | BAR1MiB: pointer.Of(uint64(256)), 376 | }, 377 | PCIBusID: "pciBusID1", 378 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 379 | CoresClockMHz: pointer.Of(uint(1)), 380 | MemoryClockMHz: pointer.Of(uint(1)), 381 | DisplayState: "Enabled", 382 | PersistenceMode: "Enabled", 383 | }, 384 | ExpectedResult: map[string]*structs.Attribute{ 385 | MemoryAttr: { 386 | Int: pointer.Of(int64(256)), 387 | Unit: structs.UnitMiB, 388 | }, 389 | PowerAttr: { 390 | Int: pointer.Of(int64(2)), 391 | Unit: structs.UnitW, 392 | }, 393 | BAR1Attr: { 394 | Int: pointer.Of(int64(256)), 395 | Unit: structs.UnitMiB, 396 | }, 397 | PCIBandwidthAttr: { 398 | Int: pointer.Of(int64(1)), 399 | Unit: structs.UnitMBPerS, 400 | }, 401 | CoresClockAttr: { 402 | Int: pointer.Of(int64(1)), 403 | Unit: structs.UnitMHz, 404 | }, 405 | MemoryClockAttr: { 406 | Int: pointer.Of(int64(1)), 407 | Unit: structs.UnitMHz, 408 | }, 409 | DisplayStateAttr: { 410 | String: pointer.Of("Enabled"), 411 | }, 412 | PersistenceModeAttr: { 413 | String: pointer.Of("Enabled"), 414 | }, 415 | }, 416 | }, 417 | { 418 | Name: "nil values are omitted", 419 | FingerprintDeviceData: &nvml.FingerprintDeviceData{ 420 | DeviceData: &nvml.DeviceData{ 421 | UUID: "1", 422 | DeviceName: pointer.Of("Type1"), 423 | MemoryMiB: nil, 424 | PowerW: pointer.Of(uint(2)), 425 | BAR1MiB: pointer.Of(uint64(256)), 426 | }, 427 | PCIBusID: "pciBusID1", 428 | DisplayState: "Enabled", 429 | PersistenceMode: "Enabled", 430 | }, 431 | ExpectedResult: map[string]*structs.Attribute{ 432 | PowerAttr: { 433 | Int: pointer.Of(int64(2)), 434 | Unit: structs.UnitW, 435 | }, 436 | BAR1Attr: { 437 | Int: pointer.Of(int64(256)), 438 | Unit: structs.UnitMiB, 439 | }, 440 | DisplayStateAttr: { 441 | String: pointer.Of("Enabled"), 442 | }, 443 | PersistenceModeAttr: { 444 | String: pointer.Of("Enabled"), 445 | }, 446 | }, 447 | }, 448 | } { 449 | t.Run(testCase.Name, func(t *testing.T) { 450 | actualResult := attributesFromFingerprintDeviceData(testCase.FingerprintDeviceData) 451 | must.Eq(t, testCase.ExpectedResult, actualResult) 452 | }) 453 | } 454 | } 455 | 456 | func TestDeviceGroupFromFingerprintData(t *testing.T) { 457 | for _, testCase := range []struct { 458 | Name string 459 | GroupName string 460 | Devices []*nvml.FingerprintDeviceData 461 | CommonAttributes map[string]*structs.Attribute 462 | ExpectedResult *device.DeviceGroup 463 | }{ 464 | { 465 | Name: "Devices are provided", 466 | GroupName: "Type1", 467 | Devices: []*nvml.FingerprintDeviceData{ 468 | { 469 | DeviceData: &nvml.DeviceData{ 470 | UUID: "1", 471 | DeviceName: pointer.Of("Type1"), 472 | MemoryMiB: pointer.Of(uint64(100)), 473 | PowerW: pointer.Of(uint(2)), 474 | BAR1MiB: pointer.Of(uint64(256)), 475 | }, 476 | PCIBusID: "pciBusID1", 477 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 478 | CoresClockMHz: pointer.Of(uint(1)), 479 | MemoryClockMHz: pointer.Of(uint(1)), 480 | DisplayState: "Enabled", 481 | PersistenceMode: "Enabled", 482 | }, 483 | { 484 | DeviceData: &nvml.DeviceData{ 485 | UUID: "2", 486 | DeviceName: pointer.Of("Type1"), 487 | MemoryMiB: pointer.Of(uint64(100)), 488 | PowerW: pointer.Of(uint(2)), 489 | BAR1MiB: pointer.Of(uint64(256)), 490 | }, 491 | PCIBusID: "pciBusID2", 492 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 493 | CoresClockMHz: pointer.Of(uint(1)), 494 | MemoryClockMHz: pointer.Of(uint(1)), 495 | DisplayState: "Enabled", 496 | PersistenceMode: "Enabled", 497 | }, 498 | }, 499 | ExpectedResult: &device.DeviceGroup{ 500 | Vendor: vendor, 501 | Type: deviceType, 502 | Name: "Type1", 503 | Devices: []*device.Device{ 504 | { 505 | ID: "1", 506 | Healthy: true, 507 | HwLocality: &device.DeviceLocality{ 508 | PciBusID: "pciBusID1", 509 | }, 510 | }, 511 | { 512 | ID: "2", 513 | Healthy: true, 514 | HwLocality: &device.DeviceLocality{ 515 | PciBusID: "pciBusID2", 516 | }, 517 | }, 518 | }, 519 | Attributes: map[string]*structs.Attribute{ 520 | MemoryAttr: { 521 | Int: pointer.Of(int64(100)), 522 | Unit: structs.UnitMiB, 523 | }, 524 | PowerAttr: { 525 | Int: pointer.Of(int64(2)), 526 | Unit: structs.UnitW, 527 | }, 528 | BAR1Attr: { 529 | Int: pointer.Of(int64(256)), 530 | Unit: structs.UnitMiB, 531 | }, 532 | PCIBandwidthAttr: { 533 | Int: pointer.Of(int64(1)), 534 | Unit: structs.UnitMBPerS, 535 | }, 536 | CoresClockAttr: { 537 | Int: pointer.Of(int64(1)), 538 | Unit: structs.UnitMHz, 539 | }, 540 | MemoryClockAttr: { 541 | Int: pointer.Of(int64(1)), 542 | Unit: structs.UnitMHz, 543 | }, 544 | DisplayStateAttr: { 545 | String: pointer.Of("Enabled"), 546 | }, 547 | PersistenceModeAttr: { 548 | String: pointer.Of("Enabled"), 549 | }, 550 | }, 551 | }, 552 | }, 553 | { 554 | Name: "Devices and common attributes are provided", 555 | GroupName: "Type1", 556 | Devices: []*nvml.FingerprintDeviceData{ 557 | { 558 | DeviceData: &nvml.DeviceData{ 559 | UUID: "1", 560 | DeviceName: pointer.Of("Type1"), 561 | MemoryMiB: pointer.Of(uint64(100)), 562 | PowerW: pointer.Of(uint(2)), 563 | BAR1MiB: pointer.Of(uint64(256)), 564 | }, 565 | PCIBusID: "pciBusID1", 566 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 567 | CoresClockMHz: pointer.Of(uint(1)), 568 | MemoryClockMHz: pointer.Of(uint(1)), 569 | DisplayState: "Enabled", 570 | PersistenceMode: "Enabled", 571 | }, 572 | { 573 | DeviceData: &nvml.DeviceData{ 574 | UUID: "2", 575 | DeviceName: pointer.Of("Type1"), 576 | MemoryMiB: pointer.Of(uint64(100)), 577 | PowerW: pointer.Of(uint(2)), 578 | BAR1MiB: pointer.Of(uint64(256)), 579 | }, 580 | PCIBusID: "pciBusID2", 581 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 582 | CoresClockMHz: pointer.Of(uint(1)), 583 | MemoryClockMHz: pointer.Of(uint(1)), 584 | DisplayState: "Enabled", 585 | PersistenceMode: "Enabled", 586 | }, 587 | }, 588 | CommonAttributes: map[string]*structs.Attribute{ 589 | DriverVersionAttr: { 590 | String: pointer.Of("1"), 591 | }, 592 | }, 593 | ExpectedResult: &device.DeviceGroup{ 594 | Vendor: vendor, 595 | Type: deviceType, 596 | Name: "Type1", 597 | Devices: []*device.Device{ 598 | { 599 | ID: "1", 600 | Healthy: true, 601 | HwLocality: &device.DeviceLocality{ 602 | PciBusID: "pciBusID1", 603 | }, 604 | }, 605 | { 606 | ID: "2", 607 | Healthy: true, 608 | HwLocality: &device.DeviceLocality{ 609 | PciBusID: "pciBusID2", 610 | }, 611 | }, 612 | }, 613 | Attributes: map[string]*structs.Attribute{ 614 | MemoryAttr: { 615 | Int: pointer.Of(int64(100)), 616 | Unit: structs.UnitMiB, 617 | }, 618 | PowerAttr: { 619 | Int: pointer.Of(int64(2)), 620 | Unit: structs.UnitW, 621 | }, 622 | BAR1Attr: { 623 | Int: pointer.Of(int64(256)), 624 | Unit: structs.UnitMiB, 625 | }, 626 | PCIBandwidthAttr: { 627 | Int: pointer.Of(int64(1)), 628 | Unit: structs.UnitMBPerS, 629 | }, 630 | CoresClockAttr: { 631 | Int: pointer.Of(int64(1)), 632 | Unit: structs.UnitMHz, 633 | }, 634 | MemoryClockAttr: { 635 | Int: pointer.Of(int64(1)), 636 | Unit: structs.UnitMHz, 637 | }, 638 | DisplayStateAttr: { 639 | String: pointer.Of("Enabled"), 640 | }, 641 | PersistenceModeAttr: { 642 | String: pointer.Of("Enabled"), 643 | }, 644 | DriverVersionAttr: { 645 | String: pointer.Of("1"), 646 | }, 647 | }, 648 | }, 649 | }, 650 | { 651 | Name: "Devices are not provided", 652 | GroupName: "Type1", 653 | CommonAttributes: map[string]*structs.Attribute{ 654 | DriverVersionAttr: { 655 | String: pointer.Of("1"), 656 | }, 657 | }, 658 | Devices: nil, 659 | ExpectedResult: nil, 660 | }, 661 | } { 662 | t.Run(testCase.Name, func(t *testing.T) { 663 | actualResult := deviceGroupFromFingerprintData(testCase.GroupName, testCase.Devices, testCase.CommonAttributes) 664 | must.Eq(t, testCase.ExpectedResult, actualResult) 665 | }) 666 | } 667 | } 668 | 669 | func TestWriteFingerprintToChannel(t *testing.T) { 670 | for _, testCase := range []struct { 671 | Name string 672 | Device *NvidiaDevice 673 | ExpectedWriteToChannel *device.FingerprintResponse 674 | }{ 675 | { 676 | Name: "Check that FingerprintError is handled properly", 677 | Device: &NvidiaDevice{ 678 | nvmlClient: &MockNvmlClient{ 679 | FingerprintError: errors.New(""), 680 | }, 681 | logger: hclog.NewNullLogger(), 682 | }, 683 | ExpectedWriteToChannel: &device.FingerprintResponse{ 684 | Error: errors.New(""), 685 | }, 686 | }, 687 | { 688 | Name: "Check ignore devices works correctly", 689 | Device: &NvidiaDevice{ 690 | nvmlClient: &MockNvmlClient{ 691 | FingerprintResponseReturned: &nvml.FingerprintData{ 692 | DriverVersion: "1", 693 | Devices: []*nvml.FingerprintDeviceData{ 694 | { 695 | DeviceData: &nvml.DeviceData{ 696 | UUID: "1", 697 | DeviceName: pointer.Of("Name"), 698 | MemoryMiB: pointer.Of(uint64(10)), 699 | PowerW: pointer.Of(uint(100)), 700 | BAR1MiB: pointer.Of(uint64(256)), 701 | }, 702 | PCIBusID: "pciBusID1", 703 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 704 | CoresClockMHz: pointer.Of(uint(1)), 705 | MemoryClockMHz: pointer.Of(uint(1)), 706 | DisplayState: "Enabled", 707 | PersistenceMode: "Enabled", 708 | }, 709 | { 710 | DeviceData: &nvml.DeviceData{ 711 | UUID: "2", 712 | DeviceName: pointer.Of("Name"), 713 | MemoryMiB: pointer.Of(uint64(10)), 714 | PowerW: pointer.Of(uint(100)), 715 | BAR1MiB: pointer.Of(uint64(256)), 716 | }, 717 | PCIBusID: "pciBusID2", 718 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 719 | CoresClockMHz: pointer.Of(uint(1)), 720 | MemoryClockMHz: pointer.Of(uint(1)), 721 | DisplayState: "Enabled", 722 | PersistenceMode: "Enabled", 723 | }, 724 | }, 725 | }, 726 | }, 727 | ignoredGPUIDs: map[string]struct{}{ 728 | "1": {}, 729 | }, 730 | logger: hclog.NewNullLogger(), 731 | }, 732 | ExpectedWriteToChannel: &device.FingerprintResponse{ 733 | Devices: []*device.DeviceGroup{ 734 | { 735 | Vendor: vendor, 736 | Type: deviceType, 737 | Name: "Name", 738 | Devices: []*device.Device{ 739 | { 740 | ID: "2", 741 | Healthy: true, 742 | HwLocality: &device.DeviceLocality{ 743 | PciBusID: "pciBusID2", 744 | }, 745 | }, 746 | }, 747 | Attributes: map[string]*structs.Attribute{ 748 | MemoryAttr: { 749 | Int: pointer.Of(int64(10)), 750 | Unit: structs.UnitMiB, 751 | }, 752 | PowerAttr: { 753 | Int: pointer.Of(int64(100)), 754 | Unit: structs.UnitW, 755 | }, 756 | BAR1Attr: { 757 | Int: pointer.Of(int64(256)), 758 | Unit: structs.UnitMiB, 759 | }, 760 | PCIBandwidthAttr: { 761 | Int: pointer.Of(int64(1)), 762 | Unit: structs.UnitMBPerS, 763 | }, 764 | CoresClockAttr: { 765 | Int: pointer.Of(int64(1)), 766 | Unit: structs.UnitMHz, 767 | }, 768 | MemoryClockAttr: { 769 | Int: pointer.Of(int64(1)), 770 | Unit: structs.UnitMHz, 771 | }, 772 | DisplayStateAttr: { 773 | String: pointer.Of("Enabled"), 774 | }, 775 | PersistenceModeAttr: { 776 | String: pointer.Of("Enabled"), 777 | }, 778 | DriverVersionAttr: { 779 | String: pointer.Of("1"), 780 | }, 781 | }, 782 | }, 783 | }, 784 | }, 785 | }, 786 | { 787 | Name: "Check devices are split to multiple device groups 1", 788 | Device: &NvidiaDevice{ 789 | nvmlClient: &MockNvmlClient{ 790 | FingerprintResponseReturned: &nvml.FingerprintData{ 791 | DriverVersion: "1", 792 | Devices: []*nvml.FingerprintDeviceData{ 793 | { 794 | DeviceData: &nvml.DeviceData{ 795 | UUID: "1", 796 | DeviceName: pointer.Of("Name1"), 797 | MemoryMiB: pointer.Of(uint64(10)), 798 | PowerW: pointer.Of(uint(100)), 799 | BAR1MiB: pointer.Of(uint64(256)), 800 | }, 801 | PCIBusID: "pciBusID1", 802 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 803 | CoresClockMHz: pointer.Of(uint(1)), 804 | MemoryClockMHz: pointer.Of(uint(1)), 805 | DisplayState: "Enabled", 806 | PersistenceMode: "Enabled", 807 | }, 808 | { 809 | DeviceData: &nvml.DeviceData{ 810 | UUID: "2", 811 | DeviceName: pointer.Of("Name2"), 812 | MemoryMiB: pointer.Of(uint64(11)), 813 | PowerW: pointer.Of(uint(100)), 814 | BAR1MiB: pointer.Of(uint64(256)), 815 | }, 816 | PCIBusID: "pciBusID2", 817 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 818 | CoresClockMHz: pointer.Of(uint(1)), 819 | MemoryClockMHz: pointer.Of(uint(1)), 820 | DisplayState: "Enabled", 821 | PersistenceMode: "Enabled", 822 | }, 823 | { 824 | DeviceData: &nvml.DeviceData{ 825 | UUID: "3", 826 | DeviceName: pointer.Of("Name3"), 827 | MemoryMiB: pointer.Of(uint64(12)), 828 | PowerW: pointer.Of(uint(100)), 829 | BAR1MiB: pointer.Of(uint64(256)), 830 | }, 831 | PCIBusID: "pciBusID3", 832 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 833 | CoresClockMHz: pointer.Of(uint(1)), 834 | MemoryClockMHz: pointer.Of(uint(1)), 835 | DisplayState: "Enabled", 836 | PersistenceMode: "Enabled", 837 | }, 838 | }, 839 | }, 840 | }, 841 | logger: hclog.NewNullLogger(), 842 | }, 843 | ExpectedWriteToChannel: &device.FingerprintResponse{ 844 | Devices: []*device.DeviceGroup{ 845 | { 846 | Vendor: vendor, 847 | Type: deviceType, 848 | Name: "Name1", 849 | Devices: []*device.Device{ 850 | { 851 | ID: "1", 852 | Healthy: true, 853 | HwLocality: &device.DeviceLocality{ 854 | PciBusID: "pciBusID1", 855 | }, 856 | }, 857 | }, 858 | Attributes: map[string]*structs.Attribute{ 859 | MemoryAttr: { 860 | Int: pointer.Of(int64(10)), 861 | Unit: structs.UnitMiB, 862 | }, 863 | PowerAttr: { 864 | Int: pointer.Of(int64(100)), 865 | Unit: structs.UnitW, 866 | }, 867 | BAR1Attr: { 868 | Int: pointer.Of(int64(256)), 869 | Unit: structs.UnitMiB, 870 | }, 871 | PCIBandwidthAttr: { 872 | Int: pointer.Of(int64(1)), 873 | Unit: structs.UnitMBPerS, 874 | }, 875 | CoresClockAttr: { 876 | Int: pointer.Of(int64(1)), 877 | Unit: structs.UnitMHz, 878 | }, 879 | MemoryClockAttr: { 880 | Int: pointer.Of(int64(1)), 881 | Unit: structs.UnitMHz, 882 | }, 883 | DisplayStateAttr: { 884 | String: pointer.Of("Enabled"), 885 | }, 886 | PersistenceModeAttr: { 887 | String: pointer.Of("Enabled"), 888 | }, 889 | DriverVersionAttr: { 890 | String: pointer.Of("1"), 891 | }, 892 | }, 893 | }, 894 | { 895 | Vendor: vendor, 896 | Type: deviceType, 897 | Name: "Name2", 898 | Devices: []*device.Device{ 899 | { 900 | ID: "2", 901 | Healthy: true, 902 | HwLocality: &device.DeviceLocality{ 903 | PciBusID: "pciBusID2", 904 | }, 905 | }, 906 | }, 907 | Attributes: map[string]*structs.Attribute{ 908 | MemoryAttr: { 909 | Int: pointer.Of(int64(11)), 910 | Unit: structs.UnitMiB, 911 | }, 912 | PowerAttr: { 913 | Int: pointer.Of(int64(100)), 914 | Unit: structs.UnitW, 915 | }, 916 | BAR1Attr: { 917 | Int: pointer.Of(int64(256)), 918 | Unit: structs.UnitMiB, 919 | }, 920 | PCIBandwidthAttr: { 921 | Int: pointer.Of(int64(1)), 922 | Unit: structs.UnitMBPerS, 923 | }, 924 | CoresClockAttr: { 925 | Int: pointer.Of(int64(1)), 926 | Unit: structs.UnitMHz, 927 | }, 928 | MemoryClockAttr: { 929 | Int: pointer.Of(int64(1)), 930 | Unit: structs.UnitMHz, 931 | }, 932 | DisplayStateAttr: { 933 | String: pointer.Of("Enabled"), 934 | }, 935 | PersistenceModeAttr: { 936 | String: pointer.Of("Enabled"), 937 | }, 938 | DriverVersionAttr: { 939 | String: pointer.Of("1"), 940 | }, 941 | }, 942 | }, 943 | { 944 | Vendor: vendor, 945 | Type: deviceType, 946 | Name: "Name3", 947 | Devices: []*device.Device{ 948 | { 949 | ID: "3", 950 | Healthy: true, 951 | HwLocality: &device.DeviceLocality{ 952 | PciBusID: "pciBusID3", 953 | }, 954 | }, 955 | }, 956 | Attributes: map[string]*structs.Attribute{ 957 | MemoryAttr: { 958 | Int: pointer.Of(int64(12)), 959 | Unit: structs.UnitMiB, 960 | }, 961 | PowerAttr: { 962 | Int: pointer.Of(int64(100)), 963 | Unit: structs.UnitW, 964 | }, 965 | BAR1Attr: { 966 | Int: pointer.Of(int64(256)), 967 | Unit: structs.UnitMiB, 968 | }, 969 | PCIBandwidthAttr: { 970 | Int: pointer.Of(int64(1)), 971 | Unit: structs.UnitMBPerS, 972 | }, 973 | CoresClockAttr: { 974 | Int: pointer.Of(int64(1)), 975 | Unit: structs.UnitMHz, 976 | }, 977 | MemoryClockAttr: { 978 | Int: pointer.Of(int64(1)), 979 | Unit: structs.UnitMHz, 980 | }, 981 | DisplayStateAttr: { 982 | String: pointer.Of("Enabled"), 983 | }, 984 | PersistenceModeAttr: { 985 | String: pointer.Of("Enabled"), 986 | }, 987 | DriverVersionAttr: { 988 | String: pointer.Of("1"), 989 | }, 990 | }, 991 | }, 992 | }, 993 | }, 994 | }, 995 | { 996 | Name: "Check devices are split to multiple device groups 2", 997 | Device: &NvidiaDevice{ 998 | nvmlClient: &MockNvmlClient{ 999 | FingerprintResponseReturned: &nvml.FingerprintData{ 1000 | DriverVersion: "1", 1001 | Devices: []*nvml.FingerprintDeviceData{ 1002 | { 1003 | DeviceData: &nvml.DeviceData{ 1004 | UUID: "1", 1005 | DeviceName: pointer.Of("Name1"), 1006 | MemoryMiB: pointer.Of(uint64(10)), 1007 | PowerW: pointer.Of(uint(100)), 1008 | BAR1MiB: pointer.Of(uint64(256)), 1009 | }, 1010 | PCIBusID: "pciBusID1", 1011 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 1012 | CoresClockMHz: pointer.Of(uint(1)), 1013 | MemoryClockMHz: pointer.Of(uint(1)), 1014 | DisplayState: "Enabled", 1015 | PersistenceMode: "Enabled", 1016 | }, 1017 | { 1018 | DeviceData: &nvml.DeviceData{ 1019 | UUID: "2", 1020 | DeviceName: pointer.Of("Name2"), 1021 | MemoryMiB: pointer.Of(uint64(11)), 1022 | PowerW: pointer.Of(uint(100)), 1023 | BAR1MiB: pointer.Of(uint64(256)), 1024 | }, 1025 | PCIBusID: "pciBusID2", 1026 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 1027 | CoresClockMHz: pointer.Of(uint(1)), 1028 | MemoryClockMHz: pointer.Of(uint(1)), 1029 | DisplayState: "Enabled", 1030 | PersistenceMode: "Enabled", 1031 | }, 1032 | { 1033 | DeviceData: &nvml.DeviceData{ 1034 | UUID: "3", 1035 | DeviceName: pointer.Of("Name2"), 1036 | MemoryMiB: pointer.Of(uint64(12)), 1037 | PowerW: pointer.Of(uint(100)), 1038 | BAR1MiB: pointer.Of(uint64(256)), 1039 | }, 1040 | PCIBusID: "pciBusID3", 1041 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 1042 | CoresClockMHz: pointer.Of(uint(1)), 1043 | MemoryClockMHz: pointer.Of(uint(1)), 1044 | DisplayState: "Enabled", 1045 | PersistenceMode: "Enabled", 1046 | }, 1047 | }, 1048 | }, 1049 | }, 1050 | logger: hclog.NewNullLogger(), 1051 | }, 1052 | ExpectedWriteToChannel: &device.FingerprintResponse{ 1053 | Devices: []*device.DeviceGroup{ 1054 | { 1055 | Vendor: vendor, 1056 | Type: deviceType, 1057 | Name: "Name1", 1058 | Devices: []*device.Device{ 1059 | { 1060 | ID: "1", 1061 | Healthy: true, 1062 | HwLocality: &device.DeviceLocality{ 1063 | PciBusID: "pciBusID1", 1064 | }, 1065 | }, 1066 | }, 1067 | Attributes: map[string]*structs.Attribute{ 1068 | MemoryAttr: { 1069 | Int: pointer.Of(int64(10)), 1070 | Unit: structs.UnitMiB, 1071 | }, 1072 | PowerAttr: { 1073 | Int: pointer.Of(int64(100)), 1074 | Unit: structs.UnitW, 1075 | }, 1076 | BAR1Attr: { 1077 | Int: pointer.Of(int64(256)), 1078 | Unit: structs.UnitMiB, 1079 | }, 1080 | PCIBandwidthAttr: { 1081 | Int: pointer.Of(int64(1)), 1082 | Unit: structs.UnitMBPerS, 1083 | }, 1084 | CoresClockAttr: { 1085 | Int: pointer.Of(int64(1)), 1086 | Unit: structs.UnitMHz, 1087 | }, 1088 | MemoryClockAttr: { 1089 | Int: pointer.Of(int64(1)), 1090 | Unit: structs.UnitMHz, 1091 | }, 1092 | DisplayStateAttr: { 1093 | String: pointer.Of("Enabled"), 1094 | }, 1095 | PersistenceModeAttr: { 1096 | String: pointer.Of("Enabled"), 1097 | }, 1098 | DriverVersionAttr: { 1099 | String: pointer.Of("1"), 1100 | }, 1101 | }, 1102 | }, 1103 | { 1104 | Vendor: vendor, 1105 | Type: deviceType, 1106 | Name: "Name2", 1107 | Devices: []*device.Device{ 1108 | { 1109 | ID: "2", 1110 | Healthy: true, 1111 | HwLocality: &device.DeviceLocality{ 1112 | PciBusID: "pciBusID2", 1113 | }, 1114 | }, 1115 | { 1116 | ID: "3", 1117 | Healthy: true, 1118 | HwLocality: &device.DeviceLocality{ 1119 | PciBusID: "pciBusID3", 1120 | }, 1121 | }, 1122 | }, 1123 | Attributes: map[string]*structs.Attribute{ 1124 | MemoryAttr: { 1125 | Int: pointer.Of(int64(11)), 1126 | Unit: structs.UnitMiB, 1127 | }, 1128 | PowerAttr: { 1129 | Int: pointer.Of(int64(100)), 1130 | Unit: structs.UnitW, 1131 | }, 1132 | BAR1Attr: { 1133 | Int: pointer.Of(int64(256)), 1134 | Unit: structs.UnitMiB, 1135 | }, 1136 | PCIBandwidthAttr: { 1137 | Int: pointer.Of(int64(1)), 1138 | Unit: structs.UnitMBPerS, 1139 | }, 1140 | CoresClockAttr: { 1141 | Int: pointer.Of(int64(1)), 1142 | Unit: structs.UnitMHz, 1143 | }, 1144 | MemoryClockAttr: { 1145 | Int: pointer.Of(int64(1)), 1146 | Unit: structs.UnitMHz, 1147 | }, 1148 | DisplayStateAttr: { 1149 | String: pointer.Of("Enabled"), 1150 | }, 1151 | PersistenceModeAttr: { 1152 | String: pointer.Of("Enabled"), 1153 | }, 1154 | DriverVersionAttr: { 1155 | String: pointer.Of("1"), 1156 | }, 1157 | }, 1158 | }, 1159 | }, 1160 | }, 1161 | }, 1162 | } { 1163 | t.Run(testCase.Name, func(t *testing.T) { 1164 | channel := make(chan *device.FingerprintResponse, 1) 1165 | testCase.Device.writeFingerprintToChannel(channel) 1166 | actualResult := <-channel 1167 | // writeFingerprintToChannel iterates over map keys 1168 | // and insterts results to an array, so order of elements in output array 1169 | // may be different 1170 | // actualResult, expectedResult arrays has to be sorted firsted 1171 | sort.Slice(actualResult.Devices, func(i, j int) bool { 1172 | return actualResult.Devices[i].Name < actualResult.Devices[j].Name 1173 | }) 1174 | sort.Slice(testCase.ExpectedWriteToChannel.Devices, func(i, j int) bool { 1175 | return testCase.ExpectedWriteToChannel.Devices[i].Name < testCase.ExpectedWriteToChannel.Devices[j].Name 1176 | }) 1177 | must.Eq(t, testCase.ExpectedWriteToChannel, actualResult) 1178 | }) 1179 | } 1180 | } 1181 | 1182 | // Test if nonworking driver returns empty fingerprint data 1183 | func TestFingerprint(t *testing.T) { 1184 | for _, testCase := range []struct { 1185 | Name string 1186 | Device *NvidiaDevice 1187 | ExpectedWriteToChannel *device.FingerprintResponse 1188 | }{ 1189 | { 1190 | Name: "Check that working driver returns valid fingeprint data", 1191 | Device: &NvidiaDevice{ 1192 | initErr: nil, 1193 | nvmlClient: &MockNvmlClient{ 1194 | FingerprintResponseReturned: &nvml.FingerprintData{ 1195 | DriverVersion: "1", 1196 | Devices: []*nvml.FingerprintDeviceData{ 1197 | { 1198 | DeviceData: &nvml.DeviceData{ 1199 | UUID: "1", 1200 | DeviceName: pointer.Of("Name1"), 1201 | MemoryMiB: pointer.Of(uint64(10)), 1202 | PowerW: pointer.Of(uint(100)), 1203 | BAR1MiB: pointer.Of(uint64(256)), 1204 | }, 1205 | PCIBusID: "pciBusID1", 1206 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 1207 | CoresClockMHz: pointer.Of(uint(1)), 1208 | MemoryClockMHz: pointer.Of(uint(1)), 1209 | DisplayState: "Enabled", 1210 | PersistenceMode: "Enabled", 1211 | }, 1212 | { 1213 | DeviceData: &nvml.DeviceData{ 1214 | UUID: "2", 1215 | DeviceName: pointer.Of("Name1"), 1216 | MemoryMiB: pointer.Of(uint64(10)), 1217 | PowerW: pointer.Of(uint(100)), 1218 | BAR1MiB: pointer.Of(uint64(256)), 1219 | }, 1220 | PCIBusID: "pciBusID2", 1221 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 1222 | CoresClockMHz: pointer.Of(uint(1)), 1223 | MemoryClockMHz: pointer.Of(uint(1)), 1224 | DisplayState: "Enabled", 1225 | PersistenceMode: "Enabled", 1226 | }, 1227 | { 1228 | DeviceData: &nvml.DeviceData{ 1229 | UUID: "3", 1230 | DeviceName: pointer.Of("Name1"), 1231 | MemoryMiB: pointer.Of(uint64(10)), 1232 | PowerW: pointer.Of(uint(100)), 1233 | BAR1MiB: pointer.Of(uint64(256)), 1234 | }, 1235 | PCIBusID: "pciBusID3", 1236 | PCIBandwidthMBPerS: pointer.Of(uint(1)), 1237 | CoresClockMHz: pointer.Of(uint(1)), 1238 | MemoryClockMHz: pointer.Of(uint(1)), 1239 | DisplayState: "Enabled", 1240 | PersistenceMode: "Enabled", 1241 | }, 1242 | }, 1243 | }, 1244 | }, 1245 | logger: hclog.NewNullLogger(), 1246 | }, 1247 | ExpectedWriteToChannel: &device.FingerprintResponse{ 1248 | Devices: []*device.DeviceGroup{ 1249 | { 1250 | Vendor: vendor, 1251 | Type: deviceType, 1252 | Name: "Name1", 1253 | Devices: []*device.Device{ 1254 | { 1255 | ID: "1", 1256 | Healthy: true, 1257 | HwLocality: &device.DeviceLocality{ 1258 | PciBusID: "pciBusID1", 1259 | }, 1260 | }, 1261 | { 1262 | ID: "2", 1263 | Healthy: true, 1264 | HwLocality: &device.DeviceLocality{ 1265 | PciBusID: "pciBusID2", 1266 | }, 1267 | }, 1268 | { 1269 | ID: "3", 1270 | Healthy: true, 1271 | HwLocality: &device.DeviceLocality{ 1272 | PciBusID: "pciBusID3", 1273 | }, 1274 | }, 1275 | }, 1276 | Attributes: map[string]*structs.Attribute{ 1277 | MemoryAttr: { 1278 | Int: pointer.Of(int64(10)), 1279 | Unit: structs.UnitMiB, 1280 | }, 1281 | PowerAttr: { 1282 | Int: pointer.Of(int64(100)), 1283 | Unit: structs.UnitW, 1284 | }, 1285 | BAR1Attr: { 1286 | Int: pointer.Of(int64(256)), 1287 | Unit: structs.UnitMiB, 1288 | }, 1289 | PCIBandwidthAttr: { 1290 | Int: pointer.Of(int64(1)), 1291 | Unit: structs.UnitMBPerS, 1292 | }, 1293 | CoresClockAttr: { 1294 | Int: pointer.Of(int64(1)), 1295 | Unit: structs.UnitMHz, 1296 | }, 1297 | MemoryClockAttr: { 1298 | Int: pointer.Of(int64(1)), 1299 | Unit: structs.UnitMHz, 1300 | }, 1301 | DisplayStateAttr: { 1302 | String: pointer.Of("Enabled"), 1303 | }, 1304 | PersistenceModeAttr: { 1305 | String: pointer.Of("Enabled"), 1306 | }, 1307 | DriverVersionAttr: { 1308 | String: pointer.Of("1"), 1309 | }, 1310 | }, 1311 | }, 1312 | }, 1313 | }, 1314 | }, 1315 | { 1316 | Name: "Check that not working driver returns error fingeprint data", 1317 | Device: &NvidiaDevice{ 1318 | initErr: errors.New("foo"), 1319 | nvmlClient: &MockNvmlClient{ 1320 | FingerprintResponseReturned: &nvml.FingerprintData{ 1321 | DriverVersion: "1", 1322 | Devices: []*nvml.FingerprintDeviceData{ 1323 | { 1324 | DeviceData: &nvml.DeviceData{ 1325 | UUID: "1", 1326 | DeviceName: pointer.Of("Name1"), 1327 | MemoryMiB: pointer.Of(uint64(10)), 1328 | }, 1329 | }, 1330 | { 1331 | DeviceData: &nvml.DeviceData{ 1332 | UUID: "2", 1333 | DeviceName: pointer.Of("Name1"), 1334 | MemoryMiB: pointer.Of(uint64(10)), 1335 | }, 1336 | }, 1337 | { 1338 | DeviceData: &nvml.DeviceData{ 1339 | UUID: "3", 1340 | DeviceName: pointer.Of("Name1"), 1341 | MemoryMiB: pointer.Of(uint64(10)), 1342 | }, 1343 | }, 1344 | }, 1345 | }, 1346 | }, 1347 | logger: hclog.NewNullLogger(), 1348 | }, 1349 | ExpectedWriteToChannel: &device.FingerprintResponse{ 1350 | Error: errors.New("foo"), 1351 | }, 1352 | }, 1353 | } { 1354 | t.Run(testCase.Name, func(t *testing.T) { 1355 | outCh := make(chan *device.FingerprintResponse) 1356 | ctx, cancel := context.WithCancel(context.Background()) 1357 | go testCase.Device.fingerprint(ctx, outCh) 1358 | result := <-outCh 1359 | cancel() 1360 | must.Eq(t, result, testCase.ExpectedWriteToChannel) 1361 | }) 1362 | } 1363 | } 1364 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hashicorp/nomad-device-nvidia 2 | 3 | go 1.24.2 4 | 5 | // maintain the go-metrics version required by nomad 6 | replace github.com/armon/go-metrics => github.com/armon/go-metrics v0.0.0-20230509193637-d9ca9af9f1f9 7 | 8 | // switched to a broken vanity url 9 | replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.16 10 | 11 | require ( 12 | github.com/NVIDIA/go-nvml v0.12.4-0 13 | github.com/hashicorp/go-hclog v1.6.3 14 | github.com/hashicorp/nomad v1.10.0 15 | github.com/shoenig/test v1.12.1 16 | ) 17 | 18 | require ( 19 | dario.cat/mergo v1.0.1 // indirect 20 | github.com/LK4D4/joincontext v0.0.0-20171026170139-1724345da6d5 // indirect 21 | github.com/Masterminds/goutils v1.1.1 // indirect 22 | github.com/Masterminds/semver/v3 v3.3.1 // indirect 23 | github.com/Masterminds/sprig/v3 v3.3.0 // indirect 24 | github.com/Microsoft/go-winio v0.6.2 // indirect 25 | github.com/agext/levenshtein v1.2.3 // indirect 26 | github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect 27 | github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect 28 | github.com/armon/go-metrics v0.5.3 // indirect 29 | github.com/armon/go-radix v1.0.0 // indirect 30 | github.com/bgentry/speakeasy v0.2.0 // indirect 31 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 32 | github.com/container-storage-interface/spec v1.11.0 // indirect 33 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 34 | github.com/docker/cli v28.0.4+incompatible // indirect 35 | github.com/docker/docker-credential-helpers v0.9.3 // indirect 36 | github.com/dustin/go-humanize v1.0.1 // indirect 37 | github.com/fatih/color v1.18.0 // indirect 38 | github.com/go-jose/go-jose/v3 v3.0.4 // indirect 39 | github.com/go-jose/go-jose/v4 v4.1.0 // indirect 40 | github.com/go-ole/go-ole v1.3.0 // indirect 41 | github.com/go-test/deep v1.1.0 // indirect 42 | github.com/gojuno/minimock/v3 v3.4.5 // indirect 43 | github.com/golang/protobuf v1.5.4 // indirect 44 | github.com/google/btree v1.1.3 // indirect 45 | github.com/google/go-cmp v0.7.0 // indirect 46 | github.com/google/uuid v1.6.0 // indirect 47 | github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect 48 | github.com/hashicorp/cli v1.1.7 // indirect 49 | github.com/hashicorp/consul/api v1.32.0 // indirect 50 | github.com/hashicorp/cronexpr v1.1.2 // indirect 51 | github.com/hashicorp/errwrap v1.1.0 // indirect 52 | github.com/hashicorp/go-bexpr v0.1.14 // indirect 53 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 54 | github.com/hashicorp/go-immutable-radix v1.3.1 // indirect 55 | github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect 56 | github.com/hashicorp/go-kms-wrapping/v2 v2.0.18 // indirect 57 | github.com/hashicorp/go-metrics v0.5.4 // indirect 58 | github.com/hashicorp/go-msgpack/v2 v2.1.3 // indirect 59 | github.com/hashicorp/go-multierror v1.1.1 // indirect 60 | github.com/hashicorp/go-plugin v1.6.3 // indirect 61 | github.com/hashicorp/go-retryablehttp v0.7.7 // indirect 62 | github.com/hashicorp/go-rootcerts v1.0.2 // indirect 63 | github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.10 // indirect 64 | github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect 65 | github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 // indirect 66 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect 67 | github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.3 // indirect 68 | github.com/hashicorp/go-set/v2 v2.1.0 // indirect 69 | github.com/hashicorp/go-set/v3 v3.0.0 // indirect 70 | github.com/hashicorp/go-sockaddr v1.0.7 // indirect 71 | github.com/hashicorp/go-uuid v1.0.3 // indirect 72 | github.com/hashicorp/go-version v1.7.0 // indirect 73 | github.com/hashicorp/golang-lru v1.0.2 // indirect 74 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 75 | github.com/hashicorp/hcl v1.0.1-vault-7 // indirect 76 | github.com/hashicorp/hcl/v2 v2.23.0 // indirect 77 | github.com/hashicorp/memberlist v0.5.3 // indirect 78 | github.com/hashicorp/raft v1.7.3 // indirect 79 | github.com/hashicorp/raft-autopilot v0.3.0 // indirect 80 | github.com/hashicorp/serf v0.10.2 // indirect 81 | github.com/hashicorp/vault/api v1.16.0 // indirect 82 | github.com/hashicorp/yamux v0.1.2 // indirect 83 | github.com/hpcloud/tail v1.0.1-0.20170814160653-37f427138745 // indirect 84 | github.com/huandu/xstrings v1.5.0 // indirect 85 | github.com/ishidawataru/sctp v0.0.0-20250303034628-ecf9ed6df987 // indirect 86 | github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f // indirect 87 | github.com/kr/pretty v0.3.1 // indirect 88 | github.com/kr/text v0.2.0 // indirect 89 | github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect 90 | github.com/mattn/go-colorable v0.1.14 // indirect 91 | github.com/mattn/go-isatty v0.0.20 // indirect 92 | github.com/miekg/dns v1.1.65 // indirect 93 | github.com/mitchellh/copystructure v1.2.0 // indirect 94 | github.com/mitchellh/go-homedir v1.1.0 // indirect 95 | github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect 96 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 97 | github.com/mitchellh/hashstructure v1.1.0 // indirect 98 | github.com/mitchellh/mapstructure v1.5.0 // indirect 99 | github.com/mitchellh/pointerstructure v1.2.1 // indirect 100 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 101 | github.com/moby/sys/mount v0.3.4 // indirect 102 | github.com/moby/sys/mountinfo v0.7.2 // indirect 103 | github.com/oklog/run v1.1.0 // indirect 104 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 105 | github.com/posener/complete v1.2.3 // indirect 106 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect 107 | github.com/prometheus/client_golang v1.22.0 // indirect 108 | github.com/prometheus/procfs v0.16.0 // indirect 109 | github.com/rogpeppe/go-internal v1.14.1 // indirect 110 | github.com/ryanuber/go-glob v1.0.0 // indirect 111 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect 112 | github.com/shirou/gopsutil/v3 v3.24.5 // indirect 113 | github.com/shoenig/go-landlock v1.2.2 // indirect 114 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 115 | github.com/shopspring/decimal v1.4.0 // indirect 116 | github.com/spf13/cast v1.7.1 // indirect 117 | github.com/stretchr/objx v0.5.2 // indirect 118 | github.com/stretchr/testify v1.10.0 // indirect 119 | github.com/tklauser/go-sysconf v0.3.15 // indirect 120 | github.com/tklauser/numcpus v0.10.0 // indirect 121 | github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect 122 | github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect 123 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 124 | github.com/zclconf/go-cty v1.16.2 // indirect 125 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect 126 | go.opentelemetry.io/otel v1.35.0 // indirect 127 | go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect 128 | golang.org/x/crypto v0.37.0 // indirect 129 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect 130 | golang.org/x/mod v0.24.0 // indirect 131 | golang.org/x/net v0.39.0 // indirect 132 | golang.org/x/sync v0.13.0 // indirect 133 | golang.org/x/sys v0.32.0 // indirect 134 | golang.org/x/text v0.24.0 // indirect 135 | golang.org/x/time v0.11.0 // indirect 136 | golang.org/x/tools v0.32.0 // indirect 137 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a // indirect 138 | google.golang.org/grpc v1.71.1 // indirect 139 | google.golang.org/protobuf v1.36.6 // indirect 140 | gopkg.in/fsnotify.v1 v1.4.7 // indirect 141 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 142 | gopkg.in/yaml.v3 v3.0.1 // indirect 143 | kernel.org/pub/linux/libs/security/libcap/psx v1.2.75 // indirect 144 | oss.indeed.com/go/libtime v1.6.0 // indirect 145 | ) 146 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 4 | dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 5 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= 6 | github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 7 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 8 | github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= 9 | github.com/LK4D4/joincontext v0.0.0-20171026170139-1724345da6d5 h1:U7q69tqXiCf6m097GRlNQB0/6SI1qWIOHYHhCEvDxF4= 10 | github.com/LK4D4/joincontext v0.0.0-20171026170139-1724345da6d5/go.mod h1:nxQPcNPR/34g+HcK2hEsF99O+GJgIkW/OmPl8wtzhmk= 11 | github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= 12 | github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= 13 | github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= 14 | github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= 15 | github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= 16 | github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= 17 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= 18 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= 19 | github.com/NVIDIA/go-nvml v0.12.4-0 h1:4tkbB3pT1O77JGr0gQ6uD8FrsUPqP1A/EOEm2wI1TUg= 20 | github.com/NVIDIA/go-nvml v0.12.4-0/go.mod h1:8Llmj+1Rr+9VGGwZuRer5N/aCjxGuR5nPb/9ebBiIEQ= 21 | github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= 22 | github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= 23 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 24 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 25 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 26 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 27 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 28 | github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= 29 | github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= 30 | github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= 31 | github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= 32 | github.com/armon/go-metrics v0.0.0-20230509193637-d9ca9af9f1f9 h1:51N4T44k8crLrlHy1zgBKGdYKjzjquaXw/RPbq/bH+o= 33 | github.com/armon/go-metrics v0.0.0-20230509193637-d9ca9af9f1f9/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= 34 | github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= 35 | github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 36 | github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= 37 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 38 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 39 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 40 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 41 | github.com/bgentry/speakeasy v0.2.0 h1:tgObeVOf8WAvtuAX6DhJ4xks4CFNwPDZiqzGqIHE51E= 42 | github.com/bgentry/speakeasy v0.2.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 43 | github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= 44 | github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= 45 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 46 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 47 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 48 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 49 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 50 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 51 | github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= 52 | github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= 53 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 54 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 55 | github.com/container-storage-interface/spec v1.11.0 h1:H/YKTOeUZwHtyPOr9raR+HgFmGluGCklulxDYxSdVNM= 56 | github.com/container-storage-interface/spec v1.11.0/go.mod h1:DtUvaQszPml1YJfIK7c00mlv6/g4wNMLanLgiUbKFRI= 57 | github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= 58 | github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= 59 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 60 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 61 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 62 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= 63 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 64 | github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= 65 | github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= 66 | github.com/docker/cli v28.0.4+incompatible h1:pBJSJeNd9QeIWPjRcV91RVJihd/TXB77q1ef64XEu4A= 67 | github.com/docker/cli v28.0.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= 68 | github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= 69 | github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 70 | github.com/docker/docker v28.0.4+incompatible h1:JNNkBctYKurkw6FrHfKqY0nKIDf5nrbxjVBtS+cdcok= 71 | github.com/docker/docker v28.0.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 72 | github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= 73 | github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= 74 | github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= 75 | github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= 76 | github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= 77 | github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= 78 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 79 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 80 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 81 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 82 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 83 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 84 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 85 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 86 | github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= 87 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 88 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 89 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 90 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 91 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 92 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 93 | github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= 94 | github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= 95 | github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= 96 | github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= 97 | github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= 98 | github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= 99 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 100 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 101 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 102 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 103 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 104 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 105 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 106 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 107 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 108 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 109 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 110 | github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= 111 | github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= 112 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 113 | github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= 114 | github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= 115 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 116 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 117 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 118 | github.com/gojuno/minimock/v3 v3.0.4/go.mod h1:HqeqnwV8mAABn3pO5hqF+RE7gjA0jsN8cbbSogoGrzI= 119 | github.com/gojuno/minimock/v3 v3.0.6/go.mod h1:v61ZjAKHr+WnEkND63nQPCZ/DTfQgJdvbCi3IuoMblY= 120 | github.com/gojuno/minimock/v3 v3.4.5 h1:Jcb0tEYZvVlQNtAAYpg3jCOoSwss2c1/rNugYTzj304= 121 | github.com/gojuno/minimock/v3 v3.4.5/go.mod h1:o9F8i2IT8v3yirA7mmdpNGzh1WNesm6iQakMtQV6KiE= 122 | github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= 123 | github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= 124 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 125 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 126 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 127 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 128 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 129 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 130 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 131 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 132 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 133 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 134 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 135 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 136 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 137 | github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= 138 | github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= 139 | github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= 140 | github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= 141 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 142 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 143 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 144 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 145 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 146 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 147 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 148 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 149 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 150 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 151 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 152 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 153 | github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= 154 | github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= 155 | github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= 156 | github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= 157 | github.com/hashicorp/cli v1.1.7 h1:/fZJ+hNdwfTSfsxMBa9WWMlfjUZbX8/LnUxgAd7lCVU= 158 | github.com/hashicorp/cli v1.1.7/go.mod h1:e6Mfpga9OCT1vqzFuoGZiiF/KaG9CbUfO5s3ghU3YgU= 159 | github.com/hashicorp/consul-template v0.40.0 h1:hEBUdCgC4+NgtLvG+Rjmotyi9trKzE0/81ZYXvdyLCU= 160 | github.com/hashicorp/consul-template v0.40.0/go.mod h1:KBUHKdBmpIgITW3vgRWptpVVPpsJRMGno3KY/NV3lAA= 161 | github.com/hashicorp/consul/api v1.32.0 h1:5wp5u780Gri7c4OedGEPzmlUEzi0g2KyiPphSr6zjVg= 162 | github.com/hashicorp/consul/api v1.32.0/go.mod h1:Z8YgY0eVPukT/17ejW+l+C7zJmKwgPHtjU1q16v/Y40= 163 | github.com/hashicorp/consul/sdk v0.16.2 h1:cGX/djeEe9r087ARiKVWwVWCF64J+yW0G6ftZMZYbj0= 164 | github.com/hashicorp/consul/sdk v0.16.2/go.mod h1:onxcZjYVsPx5XMveAC/OtoIsdr32fykB7INFltDoRE8= 165 | github.com/hashicorp/cronexpr v1.1.2 h1:wG/ZYIKT+RT3QkOdgYc+xsKWVRgnxJ1OJtjjy84fJ9A= 166 | github.com/hashicorp/cronexpr v1.1.2/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= 167 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 168 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 169 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 170 | github.com/hashicorp/go-bexpr v0.1.14 h1:uKDeyuOhWhT1r5CiMTjdVY4Aoxdxs6EtwgTGnlosyp4= 171 | github.com/hashicorp/go-bexpr v0.1.14/go.mod h1:gN7hRKB3s7yT+YvTdnhZVLTENejvhlkZ8UE4YVBS+Q8= 172 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 173 | github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= 174 | github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= 175 | github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= 176 | github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= 177 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 178 | github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= 179 | github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 180 | github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fUvGGCwJfdASSzbrfo= 181 | github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= 182 | github.com/hashicorp/go-kms-wrapping/v2 v2.0.18 h1:DLfC677GfKEpSAFpEWvl1vXsGpEcSHmbhBaPLrdDQHc= 183 | github.com/hashicorp/go-kms-wrapping/v2 v2.0.18/go.mod h1:t/eaR/mi2mw3klfl1WEAuiLKrlZ/Q8cosmsT+RIPLu0= 184 | github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY= 185 | github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI= 186 | github.com/hashicorp/go-msgpack/v2 v2.1.3 h1:cB1w4Zrk0O3jQBTcFMKqYQWRFfsSQ/TYKNyUUVyCP2c= 187 | github.com/hashicorp/go-msgpack/v2 v2.1.3/go.mod h1:SjlwKKFnwBXvxD/I1bEcfJIBbEJ+MCUn39TxymNR5ZU= 188 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 189 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 190 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 191 | github.com/hashicorp/go-plugin v1.6.3 h1:xgHB+ZUSYeuJi96WtxEjzi23uh7YQpznjGh0U0UUrwg= 192 | github.com/hashicorp/go-plugin v1.6.3/go.mod h1:MRobyh+Wc/nYy1V4KAXUiYfzxoYhs7V1mlH1Z7iY2h0= 193 | github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 194 | github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= 195 | github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= 196 | github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= 197 | github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= 198 | github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.10 h1:2iDz+t0JLl1W0tJhvmhsh/UBgT1JgC8Qxz8HxYMWXQo= 199 | github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.10/go.mod h1:eZkXE+osawMrAWR4wJRmyKauUwH6mNGbjFuiDujnbPk= 200 | github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= 201 | github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= 202 | github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 h1:SMGUnbpAcat8rIKHkBPjfv81yC46a8eCNZ2hsR2l1EI= 203 | github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1/go.mod h1:Ch/bf00Qnx77MZd49JRgHYqHQjtEmTgGU2faufpVZb0= 204 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= 205 | github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= 206 | github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.3 h1:xbrxd0U9XQW8qL1BAz2XrAjAF/P2vcqUTAues9c24B8= 207 | github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.3/go.mod h1:LWq2Sy8UoKKuK4lFuCNWSjJj57MhNNf2zzBWMtkAIX4= 208 | github.com/hashicorp/go-set/v2 v2.1.0 h1:iERPCQWks+I+4bTgy0CT2myZsCqNgBg79ZHqwniohXo= 209 | github.com/hashicorp/go-set/v2 v2.1.0/go.mod h1:6q4nh8UCVZODn2tJ5RbJi8+ki7pjZBsAEYGt6yaGeTo= 210 | github.com/hashicorp/go-set/v3 v3.0.0 h1:CaJBQvQCOWoftrBcDt7Nwgo0kdpmrKxar/x2o6pV9JA= 211 | github.com/hashicorp/go-set/v3 v3.0.0/go.mod h1:IEghM2MpE5IaNvL+D7X480dfNtxjRXZ6VMpK3C8s2ok= 212 | github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= 213 | github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= 214 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 215 | github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= 216 | github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 217 | github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= 218 | github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 219 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 220 | github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= 221 | github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 222 | github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= 223 | github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 224 | github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= 225 | github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= 226 | github.com/hashicorp/hcl/v2 v2.23.0 h1:Fphj1/gCylPxHutVSEOf2fBOh1VE4AuLV7+kbJf3qos= 227 | github.com/hashicorp/hcl/v2 v2.23.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= 228 | github.com/hashicorp/memberlist v0.5.3 h1:tQ1jOCypD0WvMemw/ZhhtH+PWpzcftQvgCorLu0hndk= 229 | github.com/hashicorp/memberlist v0.5.3/go.mod h1:h60o12SZn/ua/j0B6iKAZezA4eDaGsIuPO70eOaJ6WE= 230 | github.com/hashicorp/nomad v1.10.0 h1:Qkj6j9sez9SAuxjqi8Eu6k2gKN1Db60ULhHc4oRoY5s= 231 | github.com/hashicorp/nomad v1.10.0/go.mod h1:MYuOpkYXVO01m+iOHonlm0+LzuMixDGVD1aSwHg45ts= 232 | github.com/hashicorp/raft v1.7.3 h1:DxpEqZJysHN0wK+fviai5mFcSYsCkNpFUl1xpAW8Rbo= 233 | github.com/hashicorp/raft v1.7.3/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ= 234 | github.com/hashicorp/raft-autopilot v0.3.0 h1:KhXCecBFqAMpC0i77qVfuYd937cl2dNatSA/sSNs+2s= 235 | github.com/hashicorp/raft-autopilot v0.3.0/go.mod h1:pUBzcE8bXIm/NcFZ/xKz7O3aNOU/4T4Zkv11YqdxpUc= 236 | github.com/hashicorp/serf v0.10.2 h1:m5IORhuNSjaxeljg5DeQVDlQyVkhRIjJDimbkCa8aAc= 237 | github.com/hashicorp/serf v0.10.2/go.mod h1:T1CmSGfSeGfnfNy/w0odXQUR1rfECGd2Qdsp84DjOiY= 238 | github.com/hashicorp/vault/api v1.16.0 h1:nbEYGJiAPGzT9U4oWgaaB0g+Rj8E59QuHKyA5LhwQN4= 239 | github.com/hashicorp/vault/api v1.16.0/go.mod h1:KhuUhzOD8lDSk29AtzNjgAu2kxRA9jL9NAbkFlqvkBA= 240 | github.com/hashicorp/yamux v0.1.2 h1:XtB8kyFOyHXYVFnwT5C3+Bdo8gArse7j2AQ0DA0Uey8= 241 | github.com/hashicorp/yamux v0.1.2/go.mod h1:C+zze2n6e/7wshOZep2A70/aQU6QBRWJO/G6FT1wIns= 242 | github.com/hexdigest/gowrap v1.1.7/go.mod h1:Z+nBFUDLa01iaNM+/jzoOA1JJ7sm51rnYFauKFUB5fs= 243 | github.com/hpcloud/tail v1.0.1-0.20170814160653-37f427138745 h1:8as8OQ+RF1QrsHvWWsKBtBKINhD9QaD1iozA1wrO4aA= 244 | github.com/hpcloud/tail v1.0.1-0.20170814160653-37f427138745/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 245 | github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= 246 | github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= 247 | github.com/ishidawataru/sctp v0.0.0-20250303034628-ecf9ed6df987 h1:pf7+hef676aOjZ9XcvEw5qhdTPPaFVfavOQS+IntVOY= 248 | github.com/ishidawataru/sctp v0.0.0-20250303034628-ecf9ed6df987/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= 249 | github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f h1:E87tDTVS5W65euzixn7clSzK66puSt1H4I5SC0EmHH4= 250 | github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f/go.mod h1:3J2qVK16Lq8V+wfiL2lPeDZ7UWMxk5LemerHa1p6N00= 251 | github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= 252 | github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= 253 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 254 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 255 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 256 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 257 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 258 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 259 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 260 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 261 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 262 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 263 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 264 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 265 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 266 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 267 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 268 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 269 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 270 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 271 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 272 | github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= 273 | github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= 274 | github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 275 | github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= 276 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 277 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 278 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 279 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 280 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 281 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 282 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 283 | github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= 284 | github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck= 285 | github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= 286 | github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= 287 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 288 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 289 | github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 h1:drhDO54gdT/a15GBcMRmunZiNcLgPiFIJa23KzmcvcU= 290 | github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770/go.mod h1:SO/iHr6q2EzbqRApt+8/E9wqebTwQn5y+UlB04bxzo0= 291 | github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= 292 | github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= 293 | github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= 294 | github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= 295 | github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 296 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 297 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 298 | github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= 299 | github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= 300 | github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= 301 | github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 302 | github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= 303 | github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= 304 | github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= 305 | github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= 306 | github.com/moby/sys/mount v0.3.4 h1:yn5jq4STPztkkzSKpZkLcmjue+bZJ0u2AuQY1iNI1Ww= 307 | github.com/moby/sys/mount v0.3.4/go.mod h1:KcQJMbQdJHPlq5lcYT+/CjatWM4PuxKe+XLSVS4J6Os= 308 | github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= 309 | github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= 310 | github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= 311 | github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= 312 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 313 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 314 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 315 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 316 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 317 | github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 318 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 319 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 320 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 321 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 322 | github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= 323 | github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= 324 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 325 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 326 | github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= 327 | github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= 328 | github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 329 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 330 | github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= 331 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 332 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 333 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 334 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 335 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 336 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 337 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 338 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= 339 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 340 | github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= 341 | github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= 342 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= 343 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= 344 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 345 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 346 | github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 347 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 348 | github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= 349 | github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= 350 | github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= 351 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 352 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 353 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 354 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 355 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 356 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 357 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 358 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 359 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 360 | github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= 361 | github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= 362 | github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= 363 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 364 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 365 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 366 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 367 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 368 | github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= 369 | github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= 370 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 371 | github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= 372 | github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= 373 | github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= 374 | github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 375 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= 376 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 377 | github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= 378 | github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= 379 | github.com/shoenig/go-landlock v1.2.2 h1:cIEdRXuHkzapHJGMBM+GpWdDlZU5MSJWaxxCri7hiI8= 380 | github.com/shoenig/go-landlock v1.2.2/go.mod h1:MLSBZBAUvZh/4flRg+LysngJvz/0OdtpWTEAWuJViSY= 381 | github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= 382 | github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= 383 | github.com/shoenig/test v1.12.1 h1:mLHfnMv7gmhhP44WrvT+nKSxKkPDiNkIuHGdIGI9RLU= 384 | github.com/shoenig/test v1.12.1/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= 385 | github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= 386 | github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= 387 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 388 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 389 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 390 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 391 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 392 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 393 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 394 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 395 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 396 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 397 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 398 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 399 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 400 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 401 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 402 | github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= 403 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 404 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 405 | github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= 406 | github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= 407 | github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= 408 | github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= 409 | github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= 410 | github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= 411 | github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= 412 | github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= 413 | github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= 414 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 415 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 416 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 417 | github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 418 | github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 419 | github.com/zclconf/go-cty v1.16.2 h1:LAJSwc3v81IRBZyUVQDUdZ7hs3SYs9jv0eZJDWHD/70= 420 | github.com/zclconf/go-cty v1.16.2/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= 421 | github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= 422 | github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= 423 | go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= 424 | go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= 425 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= 426 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= 427 | go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= 428 | go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= 429 | go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= 430 | go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= 431 | go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= 432 | go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= 433 | go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= 434 | go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= 435 | go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= 436 | go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= 437 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 438 | go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= 439 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 440 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 441 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 442 | go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= 443 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 444 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 445 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 446 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 447 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 448 | golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= 449 | golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= 450 | golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= 451 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 452 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= 453 | golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= 454 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 455 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 456 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 457 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 458 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 459 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 460 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 461 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 462 | golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= 463 | golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= 464 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 465 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 466 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 467 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 468 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 469 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 470 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 471 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 472 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 473 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 474 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 475 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 476 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 477 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 478 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 479 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 480 | golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= 481 | golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= 482 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 483 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 484 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 485 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 486 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 487 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 488 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 489 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 490 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 491 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 492 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 493 | golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= 494 | golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 495 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 496 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 497 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 498 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 499 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 500 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 501 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 502 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 503 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 504 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 505 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 506 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 507 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 508 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 509 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 510 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 511 | golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 512 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 513 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 514 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 515 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 516 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 517 | golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 518 | golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 519 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 520 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 521 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 522 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 523 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 524 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 525 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 526 | golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= 527 | golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 528 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 529 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 530 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 531 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 532 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= 533 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 534 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 535 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 536 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 537 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 538 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 539 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 540 | golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= 541 | golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= 542 | golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= 543 | golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= 544 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 545 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 546 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 547 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 548 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 549 | golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 550 | golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 551 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 552 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 553 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 554 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 555 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 556 | golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= 557 | golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= 558 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 559 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 560 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 561 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 562 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 563 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 564 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 565 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 566 | google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 567 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a h1:GIqLhp/cYUkuGuiT+vJk8vhOP86L4+SP5j8yXgeVpvI= 568 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250409194420-de1ac958c67a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= 569 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 570 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 571 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 572 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 573 | google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= 574 | google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= 575 | google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= 576 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 577 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 578 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 579 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 580 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 581 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 582 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 583 | google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= 584 | google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= 585 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 586 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 587 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 588 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 589 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 590 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 591 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 592 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 593 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 594 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 595 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 596 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 597 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 598 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 599 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 600 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 601 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 602 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 603 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 604 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 605 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 606 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 607 | kernel.org/pub/linux/libs/security/libcap/psx v1.2.75 h1:cTgLaDzZqsIoKDomWTT6GEKKIdowAz5gwfKhfKhRP50= 608 | kernel.org/pub/linux/libs/security/libcap/psx v1.2.75/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= 609 | oss.indeed.com/go/libtime v1.6.0 h1:XQyczJihse/wQGo59OfPF3f4f+Sywv4R8vdGB3S9BfU= 610 | oss.indeed.com/go/libtime v1.6.0/go.mod h1:B2sdEcuzB0zhTKkAuHy4JInKRc7Al3tME4qWam6R7mA= 611 | -------------------------------------------------------------------------------- /nvml/client.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvml 5 | 6 | import ( 7 | "cmp" 8 | "fmt" 9 | "slices" 10 | ) 11 | 12 | // DeviceData represents common fields for Nvidia device 13 | type DeviceData struct { 14 | UUID string 15 | DeviceName *string 16 | MemoryMiB *uint64 17 | PowerW *uint 18 | BAR1MiB *uint64 19 | } 20 | 21 | // FingerprintDeviceData is a superset of DeviceData 22 | // it describes device specific fields returned from 23 | // nvml queries during fingerprinting call 24 | type FingerprintDeviceData struct { 25 | *DeviceData 26 | PCIBandwidthMBPerS *uint 27 | CoresClockMHz *uint 28 | MemoryClockMHz *uint 29 | DisplayState string 30 | PersistenceMode string 31 | PCIBusID string 32 | } 33 | 34 | // FingerprintData represets attributes of driver/devices 35 | type FingerprintData struct { 36 | Devices []*FingerprintDeviceData 37 | DriverVersion string 38 | } 39 | 40 | // StatsData is a superset of DeviceData 41 | // it represents statistics data returned for every Nvidia device 42 | type StatsData struct { 43 | *DeviceData 44 | PowerUsageW *uint 45 | GPUUtilization *uint 46 | MemoryUtilization *uint 47 | EncoderUtilization *uint 48 | DecoderUtilization *uint 49 | TemperatureC *uint 50 | UsedMemoryMiB *uint64 51 | BAR1UsedMiB *uint64 52 | ECCErrorsL1Cache *uint64 53 | ECCErrorsL2Cache *uint64 54 | ECCErrorsDevice *uint64 55 | } 56 | 57 | // NvmlClient describes how users would use nvml library 58 | type NvmlClient interface { 59 | GetFingerprintData() (*FingerprintData, error) 60 | GetStatsData() ([]*StatsData, error) 61 | } 62 | 63 | // nvmlClient implements NvmlClient 64 | // Users of this lib are expected to use this struct via NewNvmlClient func 65 | type nvmlClient struct { 66 | driver NvmlDriver 67 | } 68 | 69 | // NewNvmlClient function creates new nvmlClient with real 70 | // NvmlDriver implementation. Also, this func initializes NvmlDriver 71 | func NewNvmlClient() (*nvmlClient, error) { 72 | driver := &nvmlDriver{} 73 | err := driver.Initialize() 74 | if err != nil { 75 | return nil, err 76 | } 77 | return &nvmlClient{ 78 | driver: driver, 79 | }, nil 80 | } 81 | 82 | // GetFingerprintData returns FingerprintData for available Nvidia devices 83 | func (c *nvmlClient) GetFingerprintData() (*FingerprintData, error) { 84 | /* 85 | nvml fields to be fingerprinted # nvml_library_call 86 | 1 - Driver Version # nvmlSystemGetDriverVersion 87 | 2 - Product Name # nvmlDeviceGetName 88 | 3 - GPU UUID # nvmlDeviceGetUUID 89 | 4 - Total Memory # nvmlDeviceGetMemoryInfo 90 | 5 - Power # nvmlDeviceGetPowerManagementLimit 91 | 6 - PCIBusID # nvmlDeviceGetPciInfo 92 | 7 - BAR1 Memory # nvmlDeviceGetBAR1MemoryInfo( 93 | 8 - PCI Bandwidth 94 | 9 - Memory, Cores Clock # nvmlDeviceGetMaxClockInfo 95 | 10 - Display Mode # nvmlDeviceGetDisplayMode 96 | 11 - Persistence Mode # nvmlDeviceGetPersistenceMode 97 | */ 98 | 99 | // Assumed that this method is called with receiver retrieved from 100 | // NewNvmlClient because this method handles initialization of NVML library 101 | 102 | driverVersion, err := c.driver.SystemDriverVersion() 103 | if err != nil { 104 | return nil, fmt.Errorf("nvidia nvml SystemDriverVersion() error: %v\n", err) 105 | } 106 | 107 | deviceUUIDs, err := c.driver.ListDeviceUUIDs() 108 | if err != nil { 109 | return nil, fmt.Errorf("nvidia nvml ListDeviceUUIDs() error: %v\n", err) 110 | } 111 | 112 | allNvidiaGPUResources := make([]*FingerprintDeviceData, 0, len(deviceUUIDs)) 113 | 114 | for uuid, mode := range deviceUUIDs { 115 | // do not care about phsyical parents of MIGs 116 | if mode == parent { 117 | continue 118 | } 119 | 120 | deviceInfo, err := c.driver.DeviceInfoByUUID(uuid) 121 | if err != nil { 122 | return nil, fmt.Errorf("nvidia nvml DeviceInfoByUUID() error: %v\n", err) 123 | } 124 | 125 | allNvidiaGPUResources = append(allNvidiaGPUResources, &FingerprintDeviceData{ 126 | DeviceData: &DeviceData{ 127 | DeviceName: deviceInfo.Name, 128 | UUID: deviceInfo.UUID, 129 | MemoryMiB: deviceInfo.MemoryMiB, 130 | PowerW: deviceInfo.PowerW, 131 | BAR1MiB: deviceInfo.BAR1MiB, 132 | }, 133 | PCIBandwidthMBPerS: deviceInfo.PCIBandwidthMBPerS, 134 | CoresClockMHz: deviceInfo.CoresClockMHz, 135 | MemoryClockMHz: deviceInfo.MemoryClockMHz, 136 | DisplayState: deviceInfo.DisplayState, 137 | PersistenceMode: deviceInfo.PersistenceMode, 138 | PCIBusID: deviceInfo.PCIBusID, 139 | }) 140 | 141 | slices.SortFunc(allNvidiaGPUResources, func(a, b *FingerprintDeviceData) int { 142 | return cmp.Compare(a.DeviceData.UUID, b.DeviceData.UUID) 143 | }) 144 | } 145 | 146 | return &FingerprintData{ 147 | Devices: allNvidiaGPUResources, 148 | DriverVersion: driverVersion, 149 | }, nil 150 | } 151 | 152 | // GetStatsData returns statistics data for all devices on this machine 153 | func (c *nvmlClient) GetStatsData() ([]*StatsData, error) { 154 | /* 155 | nvml fields to be reported to stats api # nvml_library_call 156 | 1 - Used Memory # nvmlDeviceGetMemoryInfo 157 | 2 - Utilization of GPU # nvmlDeviceGetUtilizationRates 158 | 3 - Utilization of Memory # nvmlDeviceGetUtilizationRates 159 | 4 - Utilization of Decoder # nvmlDeviceGetDecoderUtilization 160 | 5 - Utilization of Encoder # nvmlDeviceGetEncoderUtilization 161 | 6 - Current GPU Temperature # nvmlDeviceGetTemperature 162 | 7 - Power Draw # nvmlDeviceGetPowerUsage 163 | 8 - BAR1 Used memory # nvmlDeviceGetBAR1MemoryInfo 164 | 9 - ECC Errors on requesting L1Cache # nvmlDeviceGetMemoryErrorCounter 165 | 10 - ECC Errors on requesting L2Cache # nvmlDeviceGetMemoryErrorCounter 166 | 11 - ECC Errors on requesting Device memory # nvmlDeviceGetMemoryErrorCounter 167 | */ 168 | 169 | // Assumed that this method is called with receiver retrieved from 170 | // NewNvmlClient because this method handles initialization of NVML library 171 | 172 | deviceUUIDs, err := c.driver.ListDeviceUUIDs() 173 | if err != nil { 174 | return nil, fmt.Errorf("nvidia nvml ListDeviceUUIDs() error: %v\n", err) 175 | } 176 | 177 | allNvidiaGPUStats := make([]*StatsData, 0, len(deviceUUIDs)) 178 | 179 | for uuid, mode := range deviceUUIDs { 180 | 181 | // A30/A100 MIG devices have no stats. 182 | // 183 | // https://docs.nvidia.com/datacenter/tesla/mig-user-guide/#telemetry 184 | // 185 | // Is this fixed on H100 or later? Maybe? 186 | if mode == mig || mode == parent { 187 | continue 188 | } 189 | 190 | deviceInfo, deviceStatus, err := c.driver.DeviceInfoAndStatusByUUID(uuid) 191 | if err != nil { 192 | return nil, fmt.Errorf("nvidia nvml DeviceInfoAndStatusByUUID() error: %v\n", err) 193 | } 194 | 195 | allNvidiaGPUStats = append(allNvidiaGPUStats, &StatsData{ 196 | DeviceData: &DeviceData{ 197 | DeviceName: deviceInfo.Name, 198 | UUID: deviceInfo.UUID, 199 | MemoryMiB: deviceInfo.MemoryMiB, 200 | PowerW: deviceInfo.PowerW, 201 | BAR1MiB: deviceInfo.BAR1MiB, 202 | }, 203 | PowerUsageW: deviceStatus.PowerUsageW, 204 | GPUUtilization: deviceStatus.GPUUtilization, 205 | MemoryUtilization: deviceStatus.MemoryUtilization, 206 | EncoderUtilization: deviceStatus.EncoderUtilization, 207 | DecoderUtilization: deviceStatus.DecoderUtilization, 208 | TemperatureC: deviceStatus.TemperatureC, 209 | UsedMemoryMiB: deviceStatus.UsedMemoryMiB, 210 | BAR1UsedMiB: deviceStatus.BAR1UsedMiB, 211 | ECCErrorsL1Cache: deviceStatus.ECCErrorsL1Cache, 212 | ECCErrorsL2Cache: deviceStatus.ECCErrorsL2Cache, 213 | ECCErrorsDevice: deviceStatus.ECCErrorsDevice, 214 | }) 215 | 216 | slices.SortFunc(allNvidiaGPUStats, func(a, b *StatsData) int { 217 | return cmp.Compare(a.DeviceData.UUID, b.DeviceData.UUID) 218 | }) 219 | } 220 | return allNvidiaGPUStats, nil 221 | } 222 | -------------------------------------------------------------------------------- /nvml/client_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvml 5 | 6 | import ( 7 | "errors" 8 | "testing" 9 | 10 | "github.com/hashicorp/nomad/helper/pointer" 11 | "github.com/shoenig/test/must" 12 | ) 13 | 14 | var _ NvmlDriver = (*MockNVMLDriver)(nil) 15 | 16 | type MockNVMLDriver struct { 17 | systemDriverCallSuccessful bool 18 | listDeviceUUIDsSuccessful bool 19 | deviceInfoByUUIDCallSuccessful bool 20 | deviceInfoAndStatusByUUIDCallSuccessful bool 21 | driverVersion string 22 | devices []*DeviceInfo 23 | deviceStatus []*DeviceStatus 24 | modes []mode 25 | } 26 | 27 | func (m *MockNVMLDriver) Initialize() error { 28 | return nil 29 | } 30 | 31 | func (m *MockNVMLDriver) Shutdown() error { 32 | return nil 33 | } 34 | 35 | func (m *MockNVMLDriver) SystemDriverVersion() (string, error) { 36 | if !m.systemDriverCallSuccessful { 37 | return "", errors.New("failed to get system driver") 38 | } 39 | return m.driverVersion, nil 40 | } 41 | 42 | func (m *MockNVMLDriver) ListDeviceUUIDs() (map[string]mode, error) { 43 | if !m.listDeviceUUIDsSuccessful { 44 | return nil, errors.New("failed to get device length") 45 | } 46 | 47 | allNvidiaGPUUUIDs := make(map[string]mode) 48 | 49 | for i, device := range m.devices { 50 | allNvidiaGPUUUIDs[device.UUID] = m.modes[i] 51 | } 52 | 53 | return allNvidiaGPUUUIDs, nil 54 | } 55 | 56 | func (m *MockNVMLDriver) DeviceInfoByUUID(uuid string) (*DeviceInfo, error) { 57 | if !m.deviceInfoByUUIDCallSuccessful { 58 | return nil, errors.New("failed to get device info by UUID") 59 | } 60 | 61 | for _, device := range m.devices { 62 | if uuid == device.UUID { 63 | return device, nil 64 | } 65 | } 66 | 67 | return nil, errors.New("failed to get device handle") 68 | } 69 | 70 | func (m *MockNVMLDriver) DeviceInfoAndStatusByUUID(uuid string) (*DeviceInfo, *DeviceStatus, error) { 71 | if !m.deviceInfoAndStatusByUUIDCallSuccessful { 72 | return nil, nil, errors.New("failed to get device info and status by index") 73 | } 74 | 75 | for i, device := range m.devices { 76 | if uuid == device.UUID { 77 | return device, m.deviceStatus[i], nil 78 | } 79 | } 80 | 81 | return nil, nil, errors.New("failed to get device handle") 82 | } 83 | 84 | func TestGetFingerprintDataFromNVML(t *testing.T) { 85 | for _, testCase := range []struct { 86 | Name string 87 | DriverConfiguration *MockNVMLDriver 88 | ExpectedError bool 89 | ExpectedResult *FingerprintData 90 | }{ 91 | { 92 | Name: "fail on systemDriverCallSuccessful", 93 | ExpectedError: true, 94 | ExpectedResult: nil, 95 | DriverConfiguration: &MockNVMLDriver{ 96 | systemDriverCallSuccessful: false, 97 | listDeviceUUIDsSuccessful: true, 98 | deviceInfoByUUIDCallSuccessful: true, 99 | }, 100 | }, 101 | { 102 | Name: "fail on deviceCountCallSuccessful", 103 | ExpectedError: true, 104 | ExpectedResult: nil, 105 | DriverConfiguration: &MockNVMLDriver{ 106 | systemDriverCallSuccessful: true, 107 | listDeviceUUIDsSuccessful: false, 108 | deviceInfoByUUIDCallSuccessful: true, 109 | }, 110 | }, 111 | { 112 | Name: "fail on deviceInfoByUUIDCall", 113 | ExpectedError: true, 114 | ExpectedResult: nil, 115 | DriverConfiguration: &MockNVMLDriver{ 116 | systemDriverCallSuccessful: true, 117 | listDeviceUUIDsSuccessful: true, 118 | deviceInfoByUUIDCallSuccessful: false, 119 | modes: []mode{normal, normal}, 120 | devices: []*DeviceInfo{ 121 | { 122 | UUID: "UUID1", 123 | Name: pointer.Of("ModelName1"), 124 | MemoryMiB: pointer.Of(uint64(16)), 125 | PCIBusID: "busId", 126 | PowerW: pointer.Of(uint(100)), 127 | BAR1MiB: pointer.Of(uint64(100)), 128 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 129 | CoresClockMHz: pointer.Of(uint(100)), 130 | MemoryClockMHz: pointer.Of(uint(100)), 131 | }, { 132 | UUID: "UUID2", 133 | Name: pointer.Of("ModelName2"), 134 | MemoryMiB: pointer.Of(uint64(8)), 135 | PCIBusID: "busId", 136 | PowerW: pointer.Of(uint(100)), 137 | BAR1MiB: pointer.Of(uint64(100)), 138 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 139 | CoresClockMHz: pointer.Of(uint(100)), 140 | MemoryClockMHz: pointer.Of(uint(100)), 141 | }, 142 | }, 143 | }, 144 | }, 145 | { 146 | Name: "successful outcome", 147 | ExpectedError: false, 148 | ExpectedResult: &FingerprintData{ 149 | DriverVersion: "driverVersion", 150 | Devices: []*FingerprintDeviceData{ 151 | { 152 | DeviceData: &DeviceData{ 153 | DeviceName: pointer.Of("ModelName1"), 154 | UUID: "UUID1", 155 | MemoryMiB: pointer.Of(uint64(16)), 156 | PowerW: pointer.Of(uint(100)), 157 | BAR1MiB: pointer.Of(uint64(100)), 158 | }, 159 | PCIBusID: "busId1", 160 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 161 | CoresClockMHz: pointer.Of(uint(100)), 162 | MemoryClockMHz: pointer.Of(uint(100)), 163 | DisplayState: "Enabled", 164 | PersistenceMode: "Enabled", 165 | }, { 166 | DeviceData: &DeviceData{ 167 | DeviceName: pointer.Of("ModelName2"), 168 | UUID: "UUID2", 169 | MemoryMiB: pointer.Of(uint64(8)), 170 | PowerW: pointer.Of(uint(200)), 171 | BAR1MiB: pointer.Of(uint64(200)), 172 | }, 173 | PCIBusID: "busId2", 174 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 175 | CoresClockMHz: pointer.Of(uint(200)), 176 | MemoryClockMHz: pointer.Of(uint(200)), 177 | DisplayState: "Enabled", 178 | PersistenceMode: "Enabled", 179 | }, 180 | }, 181 | }, 182 | DriverConfiguration: &MockNVMLDriver{ 183 | systemDriverCallSuccessful: true, 184 | listDeviceUUIDsSuccessful: true, 185 | deviceInfoByUUIDCallSuccessful: true, 186 | driverVersion: "driverVersion", 187 | modes: []mode{normal, normal}, 188 | devices: []*DeviceInfo{ 189 | { 190 | UUID: "UUID1", 191 | Name: pointer.Of("ModelName1"), 192 | MemoryMiB: pointer.Of(uint64(16)), 193 | PCIBusID: "busId1", 194 | PowerW: pointer.Of(uint(100)), 195 | BAR1MiB: pointer.Of(uint64(100)), 196 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 197 | CoresClockMHz: pointer.Of(uint(100)), 198 | MemoryClockMHz: pointer.Of(uint(100)), 199 | DisplayState: "Enabled", 200 | PersistenceMode: "Enabled", 201 | }, { 202 | UUID: "UUID2", 203 | Name: pointer.Of("ModelName2"), 204 | MemoryMiB: pointer.Of(uint64(8)), 205 | PCIBusID: "busId2", 206 | PowerW: pointer.Of(uint(200)), 207 | BAR1MiB: pointer.Of(uint64(200)), 208 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 209 | CoresClockMHz: pointer.Of(uint(200)), 210 | MemoryClockMHz: pointer.Of(uint(200)), 211 | DisplayState: "Enabled", 212 | PersistenceMode: "Enabled", 213 | }, 214 | }, 215 | }, 216 | }, 217 | { 218 | Name: "successful migs", 219 | ExpectedError: false, 220 | ExpectedResult: &FingerprintData{ 221 | DriverVersion: "driverVersion", 222 | Devices: []*FingerprintDeviceData{ 223 | { 224 | DeviceData: &DeviceData{ 225 | DeviceName: pointer.Of("ModelName"), 226 | UUID: "UUID1", 227 | MemoryMiB: pointer.Of(uint64(16)), 228 | PowerW: pointer.Of(uint(100)), 229 | BAR1MiB: pointer.Of(uint64(100)), 230 | }, 231 | PCIBusID: "busId1", 232 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 233 | CoresClockMHz: pointer.Of(uint(100)), 234 | MemoryClockMHz: pointer.Of(uint(100)), 235 | DisplayState: "Enabled", 236 | PersistenceMode: "Enabled", 237 | }, 238 | { 239 | DeviceData: &DeviceData{ 240 | DeviceName: pointer.Of("ModelName"), 241 | UUID: "UUID2", 242 | MemoryMiB: pointer.Of(uint64(8)), 243 | PowerW: pointer.Of(uint(200)), 244 | BAR1MiB: pointer.Of(uint64(200)), 245 | }, 246 | PCIBusID: "busId2", 247 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 248 | CoresClockMHz: pointer.Of(uint(200)), 249 | MemoryClockMHz: pointer.Of(uint(200)), 250 | DisplayState: "Enabled", 251 | PersistenceMode: "Enabled", 252 | }, 253 | { 254 | DeviceData: &DeviceData{ 255 | DeviceName: pointer.Of("ModelName"), 256 | UUID: "UUID4", 257 | MemoryMiB: pointer.Of(uint64(8)), 258 | PowerW: pointer.Of(uint(200)), 259 | BAR1MiB: pointer.Of(uint64(200)), 260 | }, 261 | PCIBusID: "busId3", 262 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 263 | CoresClockMHz: pointer.Of(uint(200)), 264 | MemoryClockMHz: pointer.Of(uint(200)), 265 | DisplayState: "Enabled", 266 | PersistenceMode: "Enabled", 267 | }, 268 | }, 269 | }, 270 | DriverConfiguration: &MockNVMLDriver{ 271 | systemDriverCallSuccessful: true, 272 | listDeviceUUIDsSuccessful: true, 273 | deviceInfoByUUIDCallSuccessful: true, 274 | driverVersion: "driverVersion", 275 | modes: []mode{normal, normal, parent, mig}, 276 | devices: []*DeviceInfo{ 277 | { 278 | UUID: "UUID1", 279 | Name: pointer.Of("ModelName"), 280 | MemoryMiB: pointer.Of(uint64(16)), 281 | PCIBusID: "busId1", 282 | PowerW: pointer.Of(uint(100)), 283 | BAR1MiB: pointer.Of(uint64(100)), 284 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 285 | CoresClockMHz: pointer.Of(uint(100)), 286 | MemoryClockMHz: pointer.Of(uint(100)), 287 | DisplayState: "Enabled", 288 | PersistenceMode: "Enabled", 289 | }, 290 | { 291 | UUID: "UUID2", 292 | Name: pointer.Of("ModelName"), 293 | MemoryMiB: pointer.Of(uint64(8)), 294 | PCIBusID: "busId2", 295 | PowerW: pointer.Of(uint(200)), 296 | BAR1MiB: pointer.Of(uint64(200)), 297 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 298 | CoresClockMHz: pointer.Of(uint(200)), 299 | MemoryClockMHz: pointer.Of(uint(200)), 300 | DisplayState: "Enabled", 301 | PersistenceMode: "Enabled", 302 | }, 303 | { 304 | UUID: "UUID3", 305 | Name: pointer.Of("ModelName"), 306 | MemoryMiB: pointer.Of(uint64(8)), 307 | PCIBusID: "busId3", 308 | PowerW: pointer.Of(uint(200)), 309 | BAR1MiB: pointer.Of(uint64(200)), 310 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 311 | CoresClockMHz: pointer.Of(uint(200)), 312 | MemoryClockMHz: pointer.Of(uint(200)), 313 | DisplayState: "Enabled", 314 | PersistenceMode: "Enabled", 315 | }, 316 | { 317 | UUID: "UUID4", 318 | Name: pointer.Of("ModelName"), 319 | MemoryMiB: pointer.Of(uint64(8)), 320 | PCIBusID: "busId3", 321 | PowerW: pointer.Of(uint(200)), 322 | BAR1MiB: pointer.Of(uint64(200)), 323 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 324 | CoresClockMHz: pointer.Of(uint(200)), 325 | MemoryClockMHz: pointer.Of(uint(200)), 326 | DisplayState: "Enabled", 327 | PersistenceMode: "Enabled", 328 | }, 329 | }, 330 | }, 331 | }, 332 | } { 333 | 334 | t.Run(testCase.Name, func(t *testing.T) { 335 | cli := nvmlClient{driver: testCase.DriverConfiguration} 336 | fingerprintData, err := cli.GetFingerprintData() 337 | if testCase.ExpectedError { 338 | must.Error(t, err) 339 | } 340 | if !testCase.ExpectedError && err != nil { 341 | must.NoError(t, err) 342 | } 343 | must.Eq(t, testCase.ExpectedResult, fingerprintData) 344 | }) 345 | } 346 | } 347 | 348 | func TestGetStatsDataFromNVML(t *testing.T) { 349 | for _, testCase := range []struct { 350 | Name string 351 | DriverConfiguration *MockNVMLDriver 352 | ExpectedError bool 353 | ExpectedResult []*StatsData 354 | }{ 355 | { 356 | Name: "fail on listDeviceUUIDsCallSuccessful", 357 | ExpectedError: true, 358 | ExpectedResult: nil, 359 | DriverConfiguration: &MockNVMLDriver{ 360 | systemDriverCallSuccessful: true, 361 | listDeviceUUIDsSuccessful: false, 362 | deviceInfoByUUIDCallSuccessful: true, 363 | deviceInfoAndStatusByUUIDCallSuccessful: true, 364 | }, 365 | }, 366 | { 367 | Name: "fail on DeviceInfoAndStatusByUUID call", 368 | ExpectedError: true, 369 | ExpectedResult: nil, 370 | DriverConfiguration: &MockNVMLDriver{ 371 | systemDriverCallSuccessful: true, 372 | listDeviceUUIDsSuccessful: true, 373 | deviceInfoAndStatusByUUIDCallSuccessful: false, 374 | modes: []mode{normal, normal}, 375 | devices: []*DeviceInfo{ 376 | { 377 | UUID: "UUID1", 378 | Name: pointer.Of("ModelName1"), 379 | MemoryMiB: pointer.Of(uint64(16)), 380 | PCIBusID: "busId1", 381 | PowerW: pointer.Of(uint(100)), 382 | BAR1MiB: pointer.Of(uint64(100)), 383 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 384 | CoresClockMHz: pointer.Of(uint(100)), 385 | MemoryClockMHz: pointer.Of(uint(100)), 386 | }, { 387 | UUID: "UUID2", 388 | Name: pointer.Of("ModelName2"), 389 | MemoryMiB: pointer.Of(uint64(8)), 390 | PCIBusID: "busId2", 391 | PowerW: pointer.Of(uint(200)), 392 | BAR1MiB: pointer.Of(uint64(200)), 393 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 394 | CoresClockMHz: pointer.Of(uint(200)), 395 | MemoryClockMHz: pointer.Of(uint(200)), 396 | }, 397 | }, 398 | deviceStatus: []*DeviceStatus{ 399 | { 400 | TemperatureC: pointer.Of(uint(1)), 401 | GPUUtilization: pointer.Of(uint(1)), 402 | MemoryUtilization: pointer.Of(uint(1)), 403 | EncoderUtilization: pointer.Of(uint(1)), 404 | DecoderUtilization: pointer.Of(uint(1)), 405 | UsedMemoryMiB: pointer.Of(uint64(1)), 406 | ECCErrorsL1Cache: pointer.Of(uint64(1)), 407 | ECCErrorsL2Cache: pointer.Of(uint64(1)), 408 | ECCErrorsDevice: pointer.Of(uint64(1)), 409 | PowerUsageW: pointer.Of(uint(1)), 410 | BAR1UsedMiB: pointer.Of(uint64(1)), 411 | }, 412 | { 413 | TemperatureC: pointer.Of(uint(2)), 414 | GPUUtilization: pointer.Of(uint(2)), 415 | MemoryUtilization: pointer.Of(uint(2)), 416 | EncoderUtilization: pointer.Of(uint(2)), 417 | DecoderUtilization: pointer.Of(uint(2)), 418 | UsedMemoryMiB: pointer.Of(uint64(2)), 419 | ECCErrorsL1Cache: pointer.Of(uint64(2)), 420 | ECCErrorsL2Cache: pointer.Of(uint64(2)), 421 | ECCErrorsDevice: pointer.Of(uint64(2)), 422 | PowerUsageW: pointer.Of(uint(2)), 423 | BAR1UsedMiB: pointer.Of(uint64(2)), 424 | }, 425 | }, 426 | }, 427 | }, 428 | { 429 | Name: "successful outcome", 430 | ExpectedError: false, 431 | ExpectedResult: []*StatsData{ 432 | { 433 | DeviceData: &DeviceData{ 434 | DeviceName: pointer.Of("ModelName1"), 435 | UUID: "UUID1", 436 | MemoryMiB: pointer.Of(uint64(16)), 437 | PowerW: pointer.Of(uint(100)), 438 | BAR1MiB: pointer.Of(uint64(100)), 439 | }, 440 | TemperatureC: pointer.Of(uint(1)), 441 | GPUUtilization: pointer.Of(uint(1)), 442 | MemoryUtilization: pointer.Of(uint(1)), 443 | EncoderUtilization: pointer.Of(uint(1)), 444 | DecoderUtilization: pointer.Of(uint(1)), 445 | UsedMemoryMiB: pointer.Of(uint64(1)), 446 | ECCErrorsL1Cache: pointer.Of(uint64(1)), 447 | ECCErrorsL2Cache: pointer.Of(uint64(1)), 448 | ECCErrorsDevice: pointer.Of(uint64(1)), 449 | PowerUsageW: pointer.Of(uint(1)), 450 | BAR1UsedMiB: pointer.Of(uint64(1)), 451 | }, 452 | { 453 | DeviceData: &DeviceData{ 454 | DeviceName: pointer.Of("ModelName2"), 455 | UUID: "UUID2", 456 | MemoryMiB: pointer.Of(uint64(8)), 457 | PowerW: pointer.Of(uint(200)), 458 | BAR1MiB: pointer.Of(uint64(200)), 459 | }, 460 | TemperatureC: pointer.Of(uint(2)), 461 | GPUUtilization: pointer.Of(uint(2)), 462 | MemoryUtilization: pointer.Of(uint(2)), 463 | EncoderUtilization: pointer.Of(uint(2)), 464 | DecoderUtilization: pointer.Of(uint(2)), 465 | UsedMemoryMiB: pointer.Of(uint64(2)), 466 | ECCErrorsL1Cache: pointer.Of(uint64(2)), 467 | ECCErrorsL2Cache: pointer.Of(uint64(2)), 468 | ECCErrorsDevice: pointer.Of(uint64(2)), 469 | PowerUsageW: pointer.Of(uint(2)), 470 | BAR1UsedMiB: pointer.Of(uint64(2)), 471 | }, 472 | }, 473 | DriverConfiguration: &MockNVMLDriver{ 474 | listDeviceUUIDsSuccessful: true, 475 | deviceInfoByUUIDCallSuccessful: true, 476 | deviceInfoAndStatusByUUIDCallSuccessful: true, 477 | modes: []mode{normal, normal}, 478 | devices: []*DeviceInfo{ 479 | { 480 | UUID: "UUID1", 481 | Name: pointer.Of("ModelName1"), 482 | MemoryMiB: pointer.Of(uint64(16)), 483 | PCIBusID: "busId1", 484 | PowerW: pointer.Of(uint(100)), 485 | BAR1MiB: pointer.Of(uint64(100)), 486 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 487 | CoresClockMHz: pointer.Of(uint(100)), 488 | MemoryClockMHz: pointer.Of(uint(100)), 489 | }, { 490 | UUID: "UUID2", 491 | Name: pointer.Of("ModelName2"), 492 | MemoryMiB: pointer.Of(uint64(8)), 493 | PCIBusID: "busId2", 494 | PowerW: pointer.Of(uint(200)), 495 | BAR1MiB: pointer.Of(uint64(200)), 496 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 497 | CoresClockMHz: pointer.Of(uint(200)), 498 | MemoryClockMHz: pointer.Of(uint(200)), 499 | }, 500 | }, 501 | deviceStatus: []*DeviceStatus{ 502 | { 503 | TemperatureC: pointer.Of(uint(1)), 504 | GPUUtilization: pointer.Of(uint(1)), 505 | MemoryUtilization: pointer.Of(uint(1)), 506 | EncoderUtilization: pointer.Of(uint(1)), 507 | DecoderUtilization: pointer.Of(uint(1)), 508 | UsedMemoryMiB: pointer.Of(uint64(1)), 509 | ECCErrorsL1Cache: pointer.Of(uint64(1)), 510 | ECCErrorsL2Cache: pointer.Of(uint64(1)), 511 | ECCErrorsDevice: pointer.Of(uint64(1)), 512 | PowerUsageW: pointer.Of(uint(1)), 513 | BAR1UsedMiB: pointer.Of(uint64(1)), 514 | }, 515 | { 516 | TemperatureC: pointer.Of(uint(2)), 517 | GPUUtilization: pointer.Of(uint(2)), 518 | MemoryUtilization: pointer.Of(uint(2)), 519 | EncoderUtilization: pointer.Of(uint(2)), 520 | DecoderUtilization: pointer.Of(uint(2)), 521 | UsedMemoryMiB: pointer.Of(uint64(2)), 522 | ECCErrorsL1Cache: pointer.Of(uint64(2)), 523 | ECCErrorsL2Cache: pointer.Of(uint64(2)), 524 | ECCErrorsDevice: pointer.Of(uint64(2)), 525 | PowerUsageW: pointer.Of(uint(2)), 526 | BAR1UsedMiB: pointer.Of(uint64(2)), 527 | }, 528 | }, 529 | }, 530 | }, 531 | { 532 | Name: "successful migs", 533 | // stats not available on migs 534 | ExpectedError: false, 535 | ExpectedResult: []*StatsData{ 536 | { 537 | DeviceData: &DeviceData{ 538 | DeviceName: pointer.Of("ModelName"), 539 | UUID: "UUID1", 540 | MemoryMiB: pointer.Of(uint64(16)), 541 | PowerW: pointer.Of(uint(100)), 542 | BAR1MiB: pointer.Of(uint64(100)), 543 | }, 544 | TemperatureC: pointer.Of(uint(1)), 545 | GPUUtilization: pointer.Of(uint(1)), 546 | MemoryUtilization: pointer.Of(uint(1)), 547 | EncoderUtilization: pointer.Of(uint(1)), 548 | DecoderUtilization: pointer.Of(uint(1)), 549 | UsedMemoryMiB: pointer.Of(uint64(1)), 550 | ECCErrorsL1Cache: pointer.Of(uint64(1)), 551 | ECCErrorsL2Cache: pointer.Of(uint64(1)), 552 | ECCErrorsDevice: pointer.Of(uint64(1)), 553 | PowerUsageW: pointer.Of(uint(1)), 554 | BAR1UsedMiB: pointer.Of(uint64(1)), 555 | }, 556 | { 557 | DeviceData: &DeviceData{ 558 | DeviceName: pointer.Of("ModelName"), 559 | UUID: "UUID2", 560 | MemoryMiB: pointer.Of(uint64(8)), 561 | PowerW: pointer.Of(uint(200)), 562 | BAR1MiB: pointer.Of(uint64(200)), 563 | }, 564 | TemperatureC: pointer.Of(uint(2)), 565 | GPUUtilization: pointer.Of(uint(2)), 566 | MemoryUtilization: pointer.Of(uint(2)), 567 | EncoderUtilization: pointer.Of(uint(2)), 568 | DecoderUtilization: pointer.Of(uint(2)), 569 | UsedMemoryMiB: pointer.Of(uint64(2)), 570 | ECCErrorsL1Cache: pointer.Of(uint64(2)), 571 | ECCErrorsL2Cache: pointer.Of(uint64(2)), 572 | ECCErrorsDevice: pointer.Of(uint64(2)), 573 | PowerUsageW: pointer.Of(uint(2)), 574 | BAR1UsedMiB: pointer.Of(uint64(2)), 575 | }, 576 | }, 577 | DriverConfiguration: &MockNVMLDriver{ 578 | listDeviceUUIDsSuccessful: true, 579 | deviceInfoByUUIDCallSuccessful: true, 580 | deviceInfoAndStatusByUUIDCallSuccessful: true, 581 | modes: []mode{normal, normal, parent, mig}, 582 | devices: []*DeviceInfo{ 583 | { 584 | UUID: "UUID1", 585 | Name: pointer.Of("ModelName"), 586 | MemoryMiB: pointer.Of(uint64(16)), 587 | PCIBusID: "busId1", 588 | PowerW: pointer.Of(uint(100)), 589 | BAR1MiB: pointer.Of(uint64(100)), 590 | PCIBandwidthMBPerS: pointer.Of(uint(100)), 591 | CoresClockMHz: pointer.Of(uint(100)), 592 | MemoryClockMHz: pointer.Of(uint(100)), 593 | }, 594 | { 595 | UUID: "UUID2", 596 | Name: pointer.Of("ModelName"), 597 | MemoryMiB: pointer.Of(uint64(8)), 598 | PCIBusID: "busId2", 599 | PowerW: pointer.Of(uint(200)), 600 | BAR1MiB: pointer.Of(uint64(200)), 601 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 602 | CoresClockMHz: pointer.Of(uint(200)), 603 | MemoryClockMHz: pointer.Of(uint(200)), 604 | }, 605 | { // parent, no stats 606 | UUID: "UUID3", 607 | Name: pointer.Of("ModelName"), 608 | MemoryMiB: pointer.Of(uint64(8)), 609 | PCIBusID: "busId3", 610 | PowerW: pointer.Of(uint(200)), 611 | BAR1MiB: pointer.Of(uint64(200)), 612 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 613 | CoresClockMHz: pointer.Of(uint(200)), 614 | MemoryClockMHz: pointer.Of(uint(200)), 615 | }, 616 | { // mig, no stats 617 | UUID: "UUID4", 618 | Name: pointer.Of("ModelName"), 619 | MemoryMiB: pointer.Of(uint64(8)), 620 | PCIBusID: "busId3", 621 | PowerW: pointer.Of(uint(200)), 622 | BAR1MiB: pointer.Of(uint64(200)), 623 | PCIBandwidthMBPerS: pointer.Of(uint(200)), 624 | CoresClockMHz: pointer.Of(uint(200)), 625 | MemoryClockMHz: pointer.Of(uint(200)), 626 | }, 627 | }, 628 | deviceStatus: []*DeviceStatus{ 629 | { 630 | TemperatureC: pointer.Of(uint(1)), 631 | GPUUtilization: pointer.Of(uint(1)), 632 | MemoryUtilization: pointer.Of(uint(1)), 633 | EncoderUtilization: pointer.Of(uint(1)), 634 | DecoderUtilization: pointer.Of(uint(1)), 635 | UsedMemoryMiB: pointer.Of(uint64(1)), 636 | ECCErrorsL1Cache: pointer.Of(uint64(1)), 637 | ECCErrorsL2Cache: pointer.Of(uint64(1)), 638 | ECCErrorsDevice: pointer.Of(uint64(1)), 639 | PowerUsageW: pointer.Of(uint(1)), 640 | BAR1UsedMiB: pointer.Of(uint64(1)), 641 | }, 642 | { 643 | TemperatureC: pointer.Of(uint(2)), 644 | GPUUtilization: pointer.Of(uint(2)), 645 | MemoryUtilization: pointer.Of(uint(2)), 646 | EncoderUtilization: pointer.Of(uint(2)), 647 | DecoderUtilization: pointer.Of(uint(2)), 648 | UsedMemoryMiB: pointer.Of(uint64(2)), 649 | ECCErrorsL1Cache: pointer.Of(uint64(2)), 650 | ECCErrorsL2Cache: pointer.Of(uint64(2)), 651 | ECCErrorsDevice: pointer.Of(uint64(2)), 652 | PowerUsageW: pointer.Of(uint(2)), 653 | BAR1UsedMiB: pointer.Of(uint64(2)), 654 | }, 655 | }, 656 | }, 657 | }, 658 | } { 659 | cli := nvmlClient{driver: testCase.DriverConfiguration} 660 | statsData, err := cli.GetStatsData() 661 | 662 | if testCase.ExpectedError { 663 | must.Error(t, err) 664 | } 665 | if !testCase.ExpectedError && err != nil { 666 | must.NoError(t, err) 667 | } 668 | must.Eq(t, testCase.ExpectedResult, statsData) 669 | } 670 | } 671 | -------------------------------------------------------------------------------- /nvml/driver_default.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | //go:build !linux 5 | 6 | package nvml 7 | 8 | // Initialize nvml library by locating nvml shared object file and calling ldopen 9 | func (n *nvmlDriver) Initialize() error { 10 | return UnavailableLib 11 | } 12 | 13 | // Shutdown stops any further interaction with nvml 14 | func (n *nvmlDriver) Shutdown() error { 15 | return UnavailableLib 16 | } 17 | 18 | // SystemDriverVersion returns installed driver version 19 | func (n *nvmlDriver) SystemDriverVersion() (string, error) { 20 | return "", UnavailableLib 21 | } 22 | 23 | // ListDeviceUUIDs reports number of available GPU devices 24 | func (n *nvmlDriver) ListDeviceUUIDs() (map[string]mode, error) { 25 | return nil, UnavailableLib 26 | } 27 | 28 | // DeviceInfoByUUID returns DeviceInfo for the GPU matching the given UUID 29 | func (n *nvmlDriver) DeviceInfoByUUID(uuid string) (*DeviceInfo, error) { 30 | return nil, UnavailableLib 31 | } 32 | 33 | // DeviceInfoAndStatusByUUID returns DeviceInfo and DeviceStatus for the GPU matching the given UUID 34 | func (n *nvmlDriver) DeviceInfoAndStatusByUUID(uuid string) (*DeviceInfo, *DeviceStatus, error) { 35 | return nil, nil, UnavailableLib 36 | } 37 | -------------------------------------------------------------------------------- /nvml/driver_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | //go:build linux 5 | 6 | package nvml 7 | 8 | import ( 9 | "fmt" 10 | 11 | "github.com/NVIDIA/go-nvml/pkg/nvml" 12 | ) 13 | 14 | func decode(msg string, code nvml.Return) error { 15 | return fmt.Errorf("%s: %s", msg, nvml.ErrorString(code)) 16 | } 17 | 18 | // Initialize nvml library by locating nvml shared object file and calling ldopen 19 | func (n *nvmlDriver) Initialize() error { 20 | if code := nvml.Init(); code != nvml.SUCCESS { 21 | return decode("failed to initialize", code) 22 | } 23 | return nil 24 | } 25 | 26 | // Shutdown stops any further interaction with nvml 27 | func (n *nvmlDriver) Shutdown() error { 28 | if code := nvml.Shutdown(); code != nvml.SUCCESS { 29 | return decode("failed to shutdown", code) 30 | } 31 | return nil 32 | } 33 | 34 | // SystemDriverVersion returns installed driver version 35 | func (n *nvmlDriver) SystemDriverVersion() (string, error) { 36 | version, code := nvml.SystemGetDriverVersion() 37 | if code != nvml.SUCCESS { 38 | return "", decode("failed to get system driver version", code) 39 | } 40 | return version, nil 41 | } 42 | 43 | // List all compute device UUIDs in the system. 44 | // Includes all instances, including normal GPUs, MIGs, and their physical parents. 45 | // Each UUID is associated with a mode indication which type it is. 46 | func (n *nvmlDriver) ListDeviceUUIDs() (map[string]mode, error) { 47 | count, code := nvml.DeviceGetCount() 48 | if code != nvml.SUCCESS { 49 | return nil, decode("failed to get device count", code) 50 | } 51 | 52 | uuids := make(map[string]mode) 53 | 54 | for i := 0; i < int(count); i++ { 55 | device, code := nvml.DeviceGetHandleByIndex(int(i)) 56 | if code != nvml.SUCCESS { 57 | return nil, decode(fmt.Sprintf("failed to get device handle %d/%d", i, count), code) 58 | } 59 | 60 | // Get the device MIG mode, and if MIG is not enabled 61 | // or the device doesn't support MIG at all (indicated 62 | // by error code ERROR_NOT_SUPPORTED), then add the 63 | // device UUID to the list and continue. 64 | migMode, _, code := nvml.DeviceGetMigMode(device) 65 | if code == nvml.ERROR_NOT_SUPPORTED || migMode == nvml.DEVICE_MIG_DISABLE { 66 | uuid, code := nvml.DeviceGetUUID(device) 67 | if code != nvml.SUCCESS { 68 | return nil, decode("failed to get device %d uuid", code) 69 | } 70 | 71 | uuids[uuid] = normal 72 | continue 73 | } 74 | if code != nvml.SUCCESS { 75 | return nil, decode("failed to get device MIG mode", code) 76 | } 77 | 78 | migCount, code := nvml.DeviceGetMaxMigDeviceCount(device) 79 | if code != nvml.SUCCESS { 80 | return nil, decode("failed to get device MIG device count", code) 81 | } 82 | 83 | uuid, code := nvml.DeviceGetUUID(device) 84 | if code == nvml.SUCCESS { 85 | uuids[uuid] = parent 86 | } 87 | 88 | for j := 0; j < int(migCount); j++ { 89 | migDevice, code := nvml.DeviceGetMigDeviceHandleByIndex(device, int(j)) 90 | if code == nvml.ERROR_NOT_FOUND || code == nvml.ERROR_INVALID_ARGUMENT { 91 | continue 92 | } 93 | if code != nvml.SUCCESS { 94 | return nil, decode("failed to get device MIG device handle", code) 95 | } 96 | 97 | uuid, code := nvml.DeviceGetUUID(migDevice) 98 | if code != nvml.SUCCESS { 99 | return nil, decode(fmt.Sprintf("failed to get mig device uuid %d", j), code) 100 | } 101 | uuids[uuid] = mig 102 | } 103 | } 104 | 105 | return uuids, nil 106 | } 107 | 108 | func bytesToMegabytes(size uint64) uint64 { 109 | return size / (1 << 20) 110 | } 111 | 112 | // DeviceInfoByUUID returns DeviceInfo for the given GPU's UUID. 113 | func (n *nvmlDriver) DeviceInfoByUUID(uuid string) (*DeviceInfo, error) { 114 | device, code := nvml.DeviceGetHandleByUUID(uuid) 115 | if code != nvml.SUCCESS { 116 | return nil, decode("failed to get device handle", code) 117 | } 118 | 119 | name, code := nvml.Device.GetName(device) 120 | if code != nvml.SUCCESS { 121 | return nil, decode("failed to get device name", code) 122 | } 123 | 124 | memory, code := nvml.DeviceGetMemoryInfo(device) 125 | if code != nvml.SUCCESS { 126 | return nil, decode("failed to get device memory info", code) 127 | } 128 | memoryTotal := bytesToMegabytes(memory.Total) 129 | 130 | parentDevice, code := nvml.DeviceGetDeviceHandleFromMigDeviceHandle(device) 131 | if code == nvml.ERROR_NOT_FOUND || code == nvml.ERROR_INVALID_ARGUMENT { 132 | // Device is not a MIG device, so nothing to do. 133 | } else if code != nvml.SUCCESS { 134 | return nil, decode("failed to get device parent device handle", code) 135 | } else { 136 | // Device is a MIG device, and get the auxilary properties (such as PCIE 137 | // bandwidth) from the parent device. 138 | device = parentDevice 139 | } 140 | 141 | power, code := nvml.DeviceGetPowerUsage(device) 142 | if code != nvml.SUCCESS { 143 | if code == nvml.ERROR_NOT_SUPPORTED { 144 | power = 0 145 | } else { 146 | return nil, decode("failed to get device power info", code) 147 | } 148 | } 149 | powerU := uint(power) / 1000 150 | 151 | bar1, code := nvml.DeviceGetBAR1MemoryInfo(device) 152 | if code != nvml.SUCCESS { 153 | return nil, decode("failed to get device bar 1 memory info", code) 154 | } 155 | bar1total := bytesToMegabytes(bar1.Bar1Total) 156 | 157 | pci, code := nvml.Device.GetPciInfo(device) 158 | if code != nvml.SUCCESS { 159 | return nil, decode("failed to get device pci info", code) 160 | } 161 | 162 | linkWidth, code := nvml.DeviceGetMaxPcieLinkWidth(device) 163 | if code != nvml.SUCCESS { 164 | if code == nvml.ERROR_NOT_SUPPORTED { 165 | linkWidth = 0 166 | } else { 167 | return nil, decode("failed to get pcie link width", code) 168 | } 169 | } 170 | 171 | linkGeneration, code := nvml.DeviceGetMaxPcieLinkGeneration(device) 172 | if code != nvml.SUCCESS { 173 | if code == nvml.ERROR_NOT_SUPPORTED { 174 | linkGeneration = 0 175 | } else { 176 | return nil, decode("failed to get pcie link generation", code) 177 | } 178 | } 179 | 180 | // https://en.wikipedia.org/wiki/PCI_Express 181 | var bandwidth uint 182 | switch linkGeneration { 183 | case 6: 184 | bandwidth = uint(linkWidth) * (4 << 10) 185 | case 5: 186 | bandwidth = uint(linkWidth) * (3 << 10) 187 | case 4: 188 | bandwidth = uint(linkWidth) * (2 << 10) 189 | case 3: 190 | bandwidth = uint(linkWidth) * (1 << 10) 191 | } 192 | 193 | busID := buildID(pci.BusId) 194 | 195 | coreClock, code := nvml.DeviceGetClockInfo(device, nvml.CLOCK_GRAPHICS) 196 | if code != nvml.SUCCESS { 197 | return nil, decode("failed to get device core clock", code) 198 | } 199 | coreClockU := uint(coreClock) 200 | 201 | memClock, code := nvml.DeviceGetClockInfo(device, nvml.CLOCK_MEM) 202 | if code != nvml.SUCCESS { 203 | return nil, decode("failed to get device mem clock", code) 204 | } 205 | memClockU := uint(memClock) 206 | 207 | mode, code := nvml.DeviceGetDisplayMode(device) 208 | if code != nvml.SUCCESS { 209 | return nil, decode("failed to get device display mode", code) 210 | } 211 | 212 | persistence, code := nvml.DeviceGetPersistenceMode(device) 213 | if code != nvml.SUCCESS { 214 | return nil, decode("failed to get device persistence mode", code) 215 | } 216 | 217 | return &DeviceInfo{ 218 | UUID: uuid, 219 | Name: &name, 220 | MemoryMiB: &memoryTotal, 221 | PowerW: &powerU, 222 | BAR1MiB: &bar1total, 223 | PCIBandwidthMBPerS: &bandwidth, 224 | PCIBusID: busID, 225 | CoresClockMHz: &coreClockU, 226 | MemoryClockMHz: &memClockU, 227 | DisplayState: fmt.Sprintf("%v", mode), 228 | PersistenceMode: fmt.Sprintf("%v", persistence), 229 | }, nil 230 | } 231 | 232 | func buildID(id [32]int8) string { 233 | b := make([]byte, len(id)) 234 | for i := 0; i < len(id); i++ { 235 | b[i] = byte(id[i]) 236 | } 237 | return string(b) 238 | } 239 | 240 | // DeviceInfoAndStatusByUUID returns DeviceInfo and DeviceStatus for index GPU in system device list. 241 | func (n *nvmlDriver) DeviceInfoAndStatusByUUID(uuid string) (*DeviceInfo, *DeviceStatus, error) { 242 | di, err := n.DeviceInfoByUUID(uuid) 243 | if err != nil { 244 | return nil, nil, err 245 | } 246 | 247 | device, code := nvml.DeviceGetHandleByUUID(uuid) 248 | if code != nvml.SUCCESS { 249 | return nil, nil, decode("failed to get device info", code) 250 | } 251 | 252 | mem, code := nvml.DeviceGetMemoryInfo(device) 253 | if code != nvml.SUCCESS { 254 | return nil, nil, decode("failed to get device memory utilization", code) 255 | } 256 | memUsedU := bytesToMegabytes(mem.Used) 257 | 258 | bar, code := nvml.DeviceGetBAR1MemoryInfo(device) 259 | if code != nvml.SUCCESS { 260 | return nil, nil, decode("failed to get device bar1 memory info", code) 261 | } 262 | barUsed := bytesToMegabytes(bar.Bar1Used) 263 | 264 | isMig := false 265 | _, code = nvml.DeviceGetDeviceHandleFromMigDeviceHandle(device) 266 | if code == nvml.ERROR_NOT_FOUND || code == nvml.ERROR_INVALID_ARGUMENT { 267 | // Device is not a MIG device. 268 | } else if code != nvml.SUCCESS { 269 | return nil, nil, decode("failed to get device parent device handle", code) 270 | } else { 271 | isMig = true 272 | } 273 | 274 | // MIG devices don't have temperature, power usage or utilization properties 275 | // so just nil them out. 276 | utzGPU, utzMem, utzEncU, utzDecU := uint(0), uint(0), uint(0), uint(0) 277 | powerU, tempU := uint(0), uint(0) 278 | if !isMig { 279 | utz, code := nvml.DeviceGetUtilizationRates(device) 280 | if code != nvml.SUCCESS { 281 | return nil, nil, decode("failed to get device utilization", code) 282 | } 283 | utzGPU = uint(utz.Gpu) 284 | utzMem = uint(utz.Memory) 285 | 286 | utzEnc, _, code := nvml.DeviceGetEncoderUtilization(device) 287 | if code != nvml.SUCCESS { 288 | return nil, nil, decode("failed to get device encoder utilization", code) 289 | } 290 | utzEncU = uint(utzEnc) 291 | 292 | utzDec, _, code := nvml.Device.GetDecoderUtilization(device) 293 | if code != nvml.SUCCESS { 294 | return nil, nil, decode("failed to get device decoder utilization", code) 295 | } 296 | utzDecU = uint(utzDec) 297 | 298 | temp, code := nvml.DeviceGetTemperature(device, nvml.TEMPERATURE_GPU) 299 | if code != nvml.SUCCESS { 300 | if code == nvml.ERROR_NOT_SUPPORTED { 301 | temp = 0 302 | } else { 303 | return nil, nil, decode("failed to get device temperature", code) 304 | } 305 | } 306 | tempU = uint(temp) 307 | 308 | power, code := nvml.DeviceGetPowerUsage(device) 309 | if code != nvml.SUCCESS { 310 | if code == nvml.ERROR_NOT_SUPPORTED { 311 | power = 0 312 | } else { 313 | return nil, nil, decode("failed to get device power usage", code) 314 | } 315 | } 316 | powerU = uint(power) 317 | } 318 | 319 | ecc, code := nvml.DeviceGetDetailedEccErrors(device, nvml.MEMORY_ERROR_TYPE_CORRECTED, nvml.VOLATILE_ECC) 320 | if code != nvml.SUCCESS { 321 | if code == nvml.ERROR_NOT_SUPPORTED { 322 | ecc = nvml.EccErrorCounts{} 323 | } else { 324 | return nil, nil, decode("failed to get device ecc error counts", code) 325 | } 326 | } 327 | 328 | return di, &DeviceStatus{ 329 | TemperatureC: &tempU, 330 | GPUUtilization: &utzGPU, 331 | MemoryUtilization: &utzMem, 332 | EncoderUtilization: &utzEncU, 333 | DecoderUtilization: &utzDecU, 334 | UsedMemoryMiB: &memUsedU, 335 | PowerUsageW: &powerU, 336 | BAR1UsedMiB: &barUsed, 337 | ECCErrorsDevice: &ecc.DeviceMemory, 338 | ECCErrorsL1Cache: &ecc.L1Cache, 339 | ECCErrorsL2Cache: &ecc.L2Cache, 340 | ECCErrorsRegisterFile: &ecc.RegisterFile, 341 | }, nil 342 | } 343 | -------------------------------------------------------------------------------- /nvml/shared.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvml 5 | 6 | import "errors" 7 | 8 | var ( 9 | // UnavailableLib is returned when the nvml library could not be loaded. 10 | UnavailableLib = errors.New("could not load NVML library") 11 | ) 12 | 13 | type mode int 14 | 15 | const ( 16 | normal mode = iota 17 | parent 18 | mig 19 | ) 20 | 21 | // nvmlDriver implements NvmlDriver 22 | // Users are required to call Initialize method before using any other methods 23 | type nvmlDriver struct{} 24 | 25 | // NvmlDriver represents set of methods to query nvml library 26 | type NvmlDriver interface { 27 | Initialize() error 28 | Shutdown() error 29 | SystemDriverVersion() (string, error) 30 | ListDeviceUUIDs() (map[string]mode, error) 31 | DeviceInfoByUUID(string) (*DeviceInfo, error) 32 | DeviceInfoAndStatusByUUID(string) (*DeviceInfo, *DeviceStatus, error) 33 | } 34 | 35 | // DeviceInfo represents nvml device data 36 | // this struct is returned by NvmlDriver DeviceInfoByUUID and 37 | // DeviceInfoAndStatusByUUID methods 38 | type DeviceInfo struct { 39 | // The following fields are guaranteed to be retrieved from nvml 40 | UUID string 41 | PCIBusID string 42 | DisplayState string 43 | PersistenceMode string 44 | 45 | // The following fields can be nil after call to nvml, because nvml was 46 | // not able to retrieve this fields for specific nvidia card 47 | Name *string 48 | MemoryMiB *uint64 49 | PowerW *uint 50 | BAR1MiB *uint64 51 | PCIBandwidthMBPerS *uint 52 | CoresClockMHz *uint 53 | MemoryClockMHz *uint 54 | } 55 | 56 | // DeviceStatus represents nvml device status 57 | // this struct is returned by NvmlDriver DeviceInfoAndStatusByUUID method 58 | type DeviceStatus struct { 59 | // The following fields can be nil after call to nvml, because nvml was 60 | // not able to retrieve this fields for specific nvidia card 61 | PowerUsageW *uint 62 | TemperatureC *uint 63 | GPUUtilization *uint // % 64 | MemoryUtilization *uint // % 65 | EncoderUtilization *uint // % 66 | DecoderUtilization *uint // % 67 | BAR1UsedMiB *uint64 68 | UsedMemoryMiB *uint64 69 | ECCErrorsL1Cache *uint64 70 | ECCErrorsL2Cache *uint64 71 | ECCErrorsDevice *uint64 72 | ECCErrorsRegisterFile *uint64 73 | } 74 | -------------------------------------------------------------------------------- /scripts/version.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) HashiCorp, Inc. 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | 6 | version_file=$1 7 | version_metadata_file=$2 8 | version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' <"${version_file}") 9 | prerelease=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' <"${version_file}") 10 | metadata=$(awk '$1 == "VersionMetadata" && $2 == "=" { gsub(/"/, "", $3); print $3 }' <"${version_metadata_file}") 11 | 12 | if [ -n "$metadata" ] && [ -n "$prerelease" ]; then 13 | echo "${version}-${prerelease}+${metadata}" 14 | elif [ -n "$metadata" ]; then 15 | echo "${version}+${metadata}" 16 | elif [ -n "$prerelease" ]; then 17 | echo "${version}-${prerelease}" 18 | else 19 | echo "${version}" 20 | fi 21 | -------------------------------------------------------------------------------- /stats.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package nvidia 5 | 6 | import ( 7 | "context" 8 | "time" 9 | 10 | "github.com/hashicorp/nomad-device-nvidia/nvml" 11 | "github.com/hashicorp/nomad/helper/pointer" 12 | "github.com/hashicorp/nomad/plugins/device" 13 | "github.com/hashicorp/nomad/plugins/shared/structs" 14 | ) 15 | 16 | const ( 17 | // Attribute names for reporting stats output 18 | PowerUsageAttr = "Power usage" 19 | PowerUsageUnit = "W" 20 | PowerUsageDesc = "Power usage for this GPU in watts and " + 21 | "its associated circuitry (e.g. memory) / Maximum GPU Power" 22 | GPUUtilizationAttr = "GPU utilization" 23 | GPUUtilizationUnit = "%" 24 | GPUUtilizationDesc = "Percent of time over the past sample period " + 25 | "during which one or more kernels were executing on the GPU." 26 | MemoryUtilizationAttr = "Memory utilization" 27 | MemoryUtilizationUnit = "%" 28 | MemoryUtilizationDesc = "Percentage of bandwidth used during the past sample period" 29 | EncoderUtilizationAttr = "Encoder utilization" 30 | EncoderUtilizationUnit = "%" 31 | EncoderUtilizationDesc = "Percent of time over the past sample period " + 32 | "during which GPU Encoder was used" 33 | DecoderUtilizationAttr = "Decoder utilization" 34 | DecoderUtilizationUnit = "%" 35 | DecoderUtilizationDesc = "Percent of time over the past sample period " + 36 | "during which GPU Decoder was used" 37 | TemperatureAttr = "Temperature" 38 | TemperatureUnit = "C" // Celsius degrees 39 | TemperatureDesc = "Temperature of the Unit" 40 | MemoryStateAttr = "Memory state" 41 | MemoryStateUnit = "MiB" // Mebibytes 42 | MemoryStateDesc = "UsedMemory / TotalMemory" 43 | BAR1StateAttr = "BAR1 buffer state" 44 | BAR1StateUnit = "MiB" // Mebibytes 45 | BAR1StateDesc = "UsedBAR1 / TotalBAR1" 46 | ECCErrorsL1CacheAttr = "ECC L1 errors" 47 | ECCErrorsL1CacheUnit = "#" // number of errors 48 | ECCErrorsL1CacheDesc = "Requested L1Cache error counter for the device" 49 | ECCErrorsL2CacheAttr = "ECC L2 errors" 50 | ECCErrorsL2CacheUnit = "#" // number of errors 51 | ECCErrorsL2CacheDesc = "Requested L2Cache error counter for the device" 52 | ECCErrorsDeviceAttr = "ECC memory errors" 53 | ECCErrorsDeviceUnit = "#" // number of errors 54 | ECCErrorsDeviceDesc = "Requested memory error counter for the device" 55 | ) 56 | 57 | // stats is the long running goroutine that streams device statistics 58 | func (d *NvidiaDevice) stats(ctx context.Context, stats chan<- *device.StatsResponse, interval time.Duration) { 59 | defer close(stats) 60 | 61 | if d.initErr != nil { 62 | if d.initErr.Error() != nvml.UnavailableLib.Error() { 63 | d.logger.Error("exiting stats due to problems with NVML loading", "error", d.initErr) 64 | stats <- device.NewStatsError(d.initErr) 65 | } 66 | 67 | return 68 | } 69 | 70 | // Create a timer that will fire immediately for the first detection 71 | ticker := time.NewTimer(0) 72 | 73 | for { 74 | select { 75 | case <-ctx.Done(): 76 | return 77 | case <-ticker.C: 78 | ticker.Reset(interval) 79 | } 80 | 81 | d.writeStatsToChannel(stats, time.Now()) 82 | } 83 | } 84 | 85 | // filterStatsByID accepts list of StatsData and set of IDs 86 | // this function would return entries from StatsData with IDs found in the set 87 | func filterStatsByID(stats []*nvml.StatsData, ids map[string]struct{}) []*nvml.StatsData { 88 | var filteredStats []*nvml.StatsData 89 | for _, statsItem := range stats { 90 | if _, ok := ids[statsItem.UUID]; ok { 91 | filteredStats = append(filteredStats, statsItem) 92 | } 93 | } 94 | return filteredStats 95 | } 96 | 97 | // writeStatsToChannel collects StatsData from NVML backend, groups StatsData 98 | // by DeviceName attribute, populates DeviceGroupStats structure for every group 99 | // and sends data over provided channel 100 | func (d *NvidiaDevice) writeStatsToChannel(stats chan<- *device.StatsResponse, timestamp time.Time) { 101 | statsData, err := d.nvmlClient.GetStatsData() 102 | if err != nil { 103 | d.logger.Error("failed to get nvidia stats", "error", err) 104 | stats <- &device.StatsResponse{ 105 | Error: err, 106 | } 107 | return 108 | } 109 | 110 | // filter only stats from devices that are stored in NvidiaDevice struct 111 | d.deviceLock.RLock() 112 | statsData = filterStatsByID(statsData, d.devices) 113 | d.deviceLock.RUnlock() 114 | 115 | // group stats by DeviceName struct field 116 | statsListByDeviceName := make(map[string][]*nvml.StatsData) 117 | for _, statsItem := range statsData { 118 | deviceName := statsItem.DeviceName 119 | if deviceName == nil { 120 | // nvml driver was not able to detect device name. This kind 121 | // of devices are placed to single group with 'notAvailable' name 122 | notAvailableCopy := notAvailable 123 | deviceName = ¬AvailableCopy 124 | } 125 | 126 | statsListByDeviceName[*deviceName] = append(statsListByDeviceName[*deviceName], statsItem) 127 | } 128 | 129 | // place data device.DeviceGroupStats struct for every group of stats 130 | deviceGroupsStats := make([]*device.DeviceGroupStats, 0, len(statsListByDeviceName)) 131 | for groupName, groupStats := range statsListByDeviceName { 132 | deviceGroupsStats = append(deviceGroupsStats, statsForGroup(groupName, groupStats, timestamp)) 133 | } 134 | 135 | stats <- &device.StatsResponse{ 136 | Groups: deviceGroupsStats, 137 | } 138 | } 139 | 140 | func newNotAvailableDeviceStats(unit, desc string) *structs.StatValue { 141 | return &structs.StatValue{Unit: unit, Desc: desc, StringVal: pointer.Of(notAvailable)} 142 | } 143 | 144 | // statsForGroup is a helper function that populates device.DeviceGroupStats 145 | // for given groupName with groupStats list 146 | func statsForGroup(groupName string, groupStats []*nvml.StatsData, timestamp time.Time) *device.DeviceGroupStats { 147 | instanceStats := make(map[string]*device.DeviceStats) 148 | for _, statsItem := range groupStats { 149 | instanceStats[statsItem.UUID] = statsForItem(statsItem, timestamp) 150 | } 151 | 152 | return &device.DeviceGroupStats{ 153 | Vendor: vendor, 154 | Type: deviceType, 155 | Name: groupName, 156 | InstanceStats: instanceStats, 157 | } 158 | } 159 | 160 | // statsForItem is a helper function that populates device.DeviceStats for given 161 | // nvml.StatsData 162 | func statsForItem(statsItem *nvml.StatsData, timestamp time.Time) *device.DeviceStats { 163 | // nvml.StatsData holds pointers to values that can be nil 164 | // In case they are nil return stats with 'notAvailable' constant 165 | var ( 166 | powerUsageStat *structs.StatValue 167 | GPUUtilizationStat *structs.StatValue 168 | memoryUtilizationStat *structs.StatValue 169 | encoderUtilizationStat *structs.StatValue 170 | decoderUtilizationStat *structs.StatValue 171 | temperatureStat *structs.StatValue 172 | memoryStateStat *structs.StatValue 173 | BAR1StateStat *structs.StatValue 174 | ECCErrorsL1CacheStat *structs.StatValue 175 | ECCErrorsL2CacheStat *structs.StatValue 176 | ECCErrorsDeviceStat *structs.StatValue 177 | ) 178 | 179 | if statsItem.PowerUsageW == nil || statsItem.PowerW == nil { 180 | powerUsageStat = newNotAvailableDeviceStats(PowerUsageUnit, PowerUsageDesc) 181 | } else { 182 | powerUsageStat = &structs.StatValue{ 183 | Unit: PowerUsageUnit, 184 | Desc: PowerUsageDesc, 185 | IntNumeratorVal: pointer.Of(int64(*statsItem.PowerUsageW)), 186 | IntDenominatorVal: uintToInt64Ptr(statsItem.PowerW), 187 | } 188 | } 189 | 190 | if statsItem.GPUUtilization == nil { 191 | GPUUtilizationStat = newNotAvailableDeviceStats(GPUUtilizationUnit, GPUUtilizationDesc) 192 | } else { 193 | GPUUtilizationStat = &structs.StatValue{ 194 | Unit: GPUUtilizationUnit, 195 | Desc: GPUUtilizationDesc, 196 | IntNumeratorVal: uintToInt64Ptr(statsItem.GPUUtilization), 197 | } 198 | } 199 | 200 | if statsItem.MemoryUtilization == nil { 201 | memoryUtilizationStat = newNotAvailableDeviceStats(MemoryUtilizationUnit, MemoryUtilizationDesc) 202 | } else { 203 | memoryUtilizationStat = &structs.StatValue{ 204 | Unit: MemoryUtilizationUnit, 205 | Desc: MemoryUtilizationDesc, 206 | IntNumeratorVal: uintToInt64Ptr(statsItem.MemoryUtilization), 207 | } 208 | } 209 | 210 | if statsItem.EncoderUtilization == nil { 211 | encoderUtilizationStat = newNotAvailableDeviceStats(EncoderUtilizationUnit, EncoderUtilizationDesc) 212 | } else { 213 | encoderUtilizationStat = &structs.StatValue{ 214 | Unit: EncoderUtilizationUnit, 215 | Desc: EncoderUtilizationDesc, 216 | IntNumeratorVal: uintToInt64Ptr(statsItem.EncoderUtilization), 217 | } 218 | } 219 | 220 | if statsItem.DecoderUtilization == nil { 221 | decoderUtilizationStat = newNotAvailableDeviceStats(DecoderUtilizationUnit, DecoderUtilizationDesc) 222 | } else { 223 | decoderUtilizationStat = &structs.StatValue{ 224 | Unit: DecoderUtilizationUnit, 225 | Desc: DecoderUtilizationDesc, 226 | IntNumeratorVal: uintToInt64Ptr(statsItem.DecoderUtilization), 227 | } 228 | } 229 | 230 | if statsItem.TemperatureC == nil { 231 | temperatureStat = newNotAvailableDeviceStats(TemperatureUnit, TemperatureDesc) 232 | } else { 233 | temperatureStat = &structs.StatValue{ 234 | Unit: TemperatureUnit, 235 | Desc: TemperatureDesc, 236 | IntNumeratorVal: uintToInt64Ptr(statsItem.TemperatureC), 237 | } 238 | } 239 | 240 | if statsItem.UsedMemoryMiB == nil || statsItem.MemoryMiB == nil { 241 | memoryStateStat = newNotAvailableDeviceStats(MemoryStateUnit, MemoryStateDesc) 242 | } else { 243 | memoryStateStat = &structs.StatValue{ 244 | Unit: MemoryStateUnit, 245 | Desc: MemoryStateDesc, 246 | IntNumeratorVal: uint64ToInt64Ptr(statsItem.UsedMemoryMiB), 247 | IntDenominatorVal: uint64ToInt64Ptr(statsItem.MemoryMiB), 248 | } 249 | } 250 | 251 | if statsItem.BAR1UsedMiB == nil || statsItem.BAR1MiB == nil { 252 | BAR1StateStat = newNotAvailableDeviceStats(BAR1StateUnit, BAR1StateDesc) 253 | } else { 254 | BAR1StateStat = &structs.StatValue{ 255 | Unit: BAR1StateUnit, 256 | Desc: BAR1StateDesc, 257 | IntNumeratorVal: uint64ToInt64Ptr(statsItem.BAR1UsedMiB), 258 | IntDenominatorVal: uint64ToInt64Ptr(statsItem.BAR1MiB), 259 | } 260 | } 261 | 262 | if statsItem.ECCErrorsL1Cache == nil { 263 | ECCErrorsL1CacheStat = newNotAvailableDeviceStats(ECCErrorsL1CacheUnit, ECCErrorsL1CacheDesc) 264 | } else { 265 | ECCErrorsL1CacheStat = &structs.StatValue{ 266 | Unit: ECCErrorsL1CacheUnit, 267 | Desc: ECCErrorsL1CacheDesc, 268 | IntNumeratorVal: uint64ToInt64Ptr(statsItem.ECCErrorsL1Cache), 269 | } 270 | } 271 | 272 | if statsItem.ECCErrorsL2Cache == nil { 273 | ECCErrorsL2CacheStat = newNotAvailableDeviceStats(ECCErrorsL2CacheUnit, ECCErrorsL2CacheDesc) 274 | } else { 275 | ECCErrorsL2CacheStat = &structs.StatValue{ 276 | Unit: ECCErrorsL2CacheUnit, 277 | Desc: ECCErrorsL2CacheDesc, 278 | IntNumeratorVal: uint64ToInt64Ptr(statsItem.ECCErrorsL2Cache), 279 | } 280 | } 281 | 282 | if statsItem.ECCErrorsDevice == nil { 283 | ECCErrorsDeviceStat = newNotAvailableDeviceStats(ECCErrorsDeviceUnit, ECCErrorsDeviceDesc) 284 | } else { 285 | ECCErrorsDeviceStat = &structs.StatValue{ 286 | Unit: ECCErrorsDeviceUnit, 287 | Desc: ECCErrorsDeviceDesc, 288 | IntNumeratorVal: uint64ToInt64Ptr(statsItem.ECCErrorsDevice), 289 | } 290 | } 291 | return &device.DeviceStats{ 292 | Summary: memoryStateStat, 293 | Stats: &structs.StatObject{ 294 | Attributes: map[string]*structs.StatValue{ 295 | PowerUsageAttr: powerUsageStat, 296 | GPUUtilizationAttr: GPUUtilizationStat, 297 | MemoryUtilizationAttr: memoryUtilizationStat, 298 | EncoderUtilizationAttr: encoderUtilizationStat, 299 | DecoderUtilizationAttr: decoderUtilizationStat, 300 | TemperatureAttr: temperatureStat, 301 | MemoryStateAttr: memoryStateStat, 302 | BAR1StateAttr: BAR1StateStat, 303 | ECCErrorsL1CacheAttr: ECCErrorsL1CacheStat, 304 | ECCErrorsL2CacheAttr: ECCErrorsL2CacheStat, 305 | ECCErrorsDeviceAttr: ECCErrorsDeviceStat, 306 | }, 307 | }, 308 | Timestamp: timestamp, 309 | } 310 | } 311 | 312 | func uintToInt64Ptr(u *uint) *int64 { 313 | if u == nil { 314 | return nil 315 | } 316 | 317 | v := int64(*u) 318 | return &v 319 | } 320 | 321 | func uint64ToInt64Ptr(u *uint64) *int64 { 322 | if u == nil { 323 | return nil 324 | } 325 | 326 | v := int64(*u) 327 | return &v 328 | } 329 | -------------------------------------------------------------------------------- /version/generate.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) HashiCorp, Inc. 3 | # SPDX-License-Identifier: MPL-2.0 4 | 5 | version_file=$1 6 | version_metadata_file=$2 7 | version=$(awk '$1 == "Version" && $2 == "=" { gsub(/"/, "", $3); print $3 }' <"${version_file}") 8 | prerelease=$(awk '$1 == "VersionPrerelease" && $2 == "=" { gsub(/"/, "", $3); print $3 }' <"${version_file}") 9 | metadata=$(awk '$1 == "VersionMetadata" && $2 == "=" { gsub(/"/, "", $3); print $3 }' <"${version_metadata_file}") 10 | 11 | if [ -n "$metadata" ] && [ -n "$prerelease" ]; then 12 | echo "${version}-${prerelease}+${metadata}" 13 | elif [ -n "$metadata" ]; then 14 | echo "${version}+${metadata}" 15 | elif [ -n "$prerelease" ]; then 16 | echo "${version}-${prerelease}" 17 | else 18 | echo "${version}" 19 | fi 20 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) HashiCorp, Inc. 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package version 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | ) 10 | 11 | var ( 12 | // The git commit that was compiled. These will be filled in by the compiler. 13 | GitCommit string 14 | GitDescribe string 15 | 16 | // The main version number that is being run at the moment. 17 | // 18 | // Version must conform to the format expected by 19 | // github.com/hashicorp/go-version for tests to work. 20 | Version = "1.2.0" 21 | 22 | // A pre-release marker for the version. If this is "" (empty string) 23 | // then it means that it is a final release. Otherwise, this is a pre-release 24 | // such as "dev" (in development), "beta.1", "rc1.1", etc. 25 | VersionPrerelease = "dev" 26 | 27 | // VersionMetadata is metadata further describing the build type. 28 | VersionMetadata = "" 29 | ) 30 | 31 | // GetHumanVersion composes the parts of the version in a way that's suitable 32 | // for displaying to humans. 33 | func GetHumanVersion() string { 34 | version := Version 35 | if GitDescribe != "" { 36 | version = GitDescribe 37 | } 38 | 39 | release := VersionPrerelease 40 | if GitDescribe == "" && release == "" { 41 | release = "dev" 42 | } 43 | 44 | if release != "" { 45 | if !strings.HasSuffix(version, "-"+release) { 46 | // if we tagged a prerelease version then the release is in the version already 47 | version += fmt.Sprintf("-%s", release) 48 | } 49 | if GitCommit != "" { 50 | version += fmt.Sprintf(" (%s)", GitCommit) 51 | } 52 | } 53 | 54 | // Strip off any single quotes added by the git information. 55 | return strings.ReplaceAll(version, "'", "") 56 | } 57 | --------------------------------------------------------------------------------