├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── semantic.yml └── workflows │ ├── codeql.yaml │ ├── create-release.yml │ ├── dependency-review.yml │ └── scorecards.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .pipelines ├── nightly.yml ├── pr.yml └── templates │ ├── cleanup-template.yml │ ├── cluster-health-template.yml │ ├── e2e-kind-template.yml │ ├── e2e-upgrade-template.yml │ ├── kind-debug-template.yml │ ├── manifest-template.yml │ ├── scan-images-template.yml │ ├── soak-test-template.yml │ └── unit-tests-template.yml ├── AUTHORS ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── cmd └── server │ └── main.go ├── developers.md ├── docs ├── manual-install.md ├── metrics.md ├── rotation.md └── testing.md ├── go.mod ├── go.sum ├── pkg ├── auth │ ├── auth.go │ └── auth_test.go ├── config │ └── azure_config.go ├── consts │ └── consts.go ├── metrics │ ├── exporter.go │ ├── exporter_test.go │ ├── prometheus_exporter.go │ └── stats_reporter.go ├── plugin │ ├── healthz.go │ ├── healthz_test.go │ ├── keyvault.go │ ├── keyvault_test.go │ ├── kms_v2_server.go │ ├── kms_v2_server_test.go │ ├── mock_keyvault │ │ └── keyvault_mock.go │ ├── server.go │ └── server_test.go ├── utils │ ├── grpc.go │ ├── grpc_test.go │ ├── sanitize.go │ └── sanitize_test.go └── version │ ├── version.go │ └── version_test.go ├── scripts ├── connect-registry.sh ├── setup-kind-cluster.sh ├── setup-kmsv2-kind-cluster.sh └── setup-local-registry.sh ├── tests ├── client │ └── client_test.go └── e2e │ ├── azure.json │ ├── encryption-config.yaml │ ├── helpers.bash │ ├── kind-config.yaml │ ├── kms.yaml │ ├── kmsv2-encryption-config.yaml │ ├── test.bats │ └── testkmsv2.bats └── tools ├── go.mod ├── go.sum └── tools.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help KMS Plugin for Key Vault improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 12 | **Steps To Reproduce** 13 | 14 | **Expected behavior** 15 | 16 | **KMS Plugin for Key Vault version** 17 | 18 | **Kubernetes version** 19 | 20 | **Additional context** 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for KMS Plugin for Key Vault 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the request** 11 | 12 | **Explain why KMS Plugin for Key Vault needs it** 13 | 14 | **Describe the solution you'd like** 15 | 16 | **Describe alternatives you've considered** 17 | 18 | **Additional context** 19 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | **Reason for Change**: 4 | 5 | 6 | 7 | **Issue Fixed**: 8 | 9 | 10 | **Notes for Reviewers**: 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | commit-message: 8 | prefix: "chore" 9 | ignore: 10 | - dependency-name: "*" 11 | update-types: 12 | - "version-update:semver-major" 13 | - "version-update:semver-minor" 14 | 15 | - package-ecosystem: github-actions 16 | directory: / 17 | schedule: 18 | interval: daily 19 | commit-message: 20 | prefix: "chore" 21 | 22 | - package-ecosystem: docker 23 | directory: / 24 | schedule: 25 | interval: daily 26 | commit-message: 27 | prefix: "chore" 28 | 29 | - package-ecosystem: gomod 30 | directory: /tools 31 | schedule: 32 | interval: daily 33 | commit-message: 34 | prefix: "chore" 35 | -------------------------------------------------------------------------------- /.github/semantic.yml: -------------------------------------------------------------------------------- 1 | titleOnly: true 2 | types: 3 | - chore 4 | - ci 5 | - docs 6 | - feat 7 | - fix 8 | - perf 9 | - refactor 10 | - release 11 | - revert 12 | - security 13 | - test 14 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | schedule: 11 | - cron: "0 15 * * 1" # Mondays at 7:00 AM PST 12 | 13 | permissions: read-all 14 | 15 | jobs: 16 | analyze: 17 | name: Analyze 18 | runs-on: ubuntu-latest 19 | permissions: 20 | security-events: write 21 | 22 | steps: 23 | - name: Harden Runner 24 | uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 25 | with: 26 | egress-policy: audit 27 | 28 | - name: Checkout repository 29 | uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab 30 | 31 | - name: Initialize CodeQL 32 | uses: github/codeql-action/init@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 33 | with: 34 | languages: go 35 | 36 | - name: Autobuild 37 | uses: github/codeql-action/autobuild@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 38 | 39 | - name: Perform CodeQL Analysis 40 | uses: github/codeql-action/analyze@b2c19fb9a2a485599ccf4ed5d65527d94bc57226 41 | -------------------------------------------------------------------------------- /.github/workflows/create-release.yml: -------------------------------------------------------------------------------- 1 | name: create_release 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | 7 | permissions: 8 | contents: write 9 | 10 | jobs: 11 | create-release: 12 | runs-on: ubuntu-22.04 13 | steps: 14 | - name: Harden Runner 15 | uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 16 | with: 17 | egress-policy: audit 18 | 19 | - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 20 | with: 21 | submodules: true 22 | fetch-depth: 0 23 | 24 | - name: Goreleaser 25 | uses: goreleaser/goreleaser-action@336e29918d653399e599bfca99fadc1d7ffbc9f7 # v4.3.0 26 | with: 27 | version: "~> v2" 28 | args: release --clean --fail-fast --timeout 60m --verbose 29 | env: 30 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 31 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | # Dependency Review Action 2 | # 3 | # This Action will scan dependency manifest files that change as part of a Pull Request, 4 | # surfacing known-vulnerable versions of the packages declared or updated in the PR. 5 | # Once installed, if the workflow run is marked as required, 6 | # PRs introducing known-vulnerable packages will be blocked from merging. 7 | # 8 | # Source repository: https://github.com/actions/dependency-review-action 9 | name: 'Dependency Review' 10 | on: [pull_request] 11 | 12 | permissions: 13 | contents: read 14 | 15 | jobs: 16 | dependency-review: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Harden Runner 20 | uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 21 | with: 22 | egress-policy: audit 23 | 24 | - name: 'Checkout Repository' 25 | uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 26 | - name: 'Dependency Review' 27 | uses: actions/dependency-review-action@0efb1d1d84fc9633afcdaad14c485cbbc90ef46c # v2.5.1 28 | -------------------------------------------------------------------------------- /.github/workflows/scorecards.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. They are provided 2 | # by a third-party and are governed by separate terms of service, privacy 3 | # policy, and support documentation. 4 | 5 | name: Scorecard supply-chain security 6 | on: 7 | # For Branch-Protection check. Only the default branch is supported. See 8 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection 9 | branch_protection_rule: 10 | # To guarantee Maintained check is occasionally updated. See 11 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained 12 | schedule: 13 | - cron: '20 7 * * 2' 14 | push: 15 | branches: ["master"] 16 | 17 | # Declare default permissions as read only. 18 | permissions: read-all 19 | 20 | jobs: 21 | analysis: 22 | name: Scorecard analysis 23 | runs-on: ubuntu-latest 24 | permissions: 25 | # Needed to upload the results to code-scanning dashboard. 26 | security-events: write 27 | # Needed to publish results and get a badge (see publish_results below). 28 | id-token: write 29 | contents: read 30 | actions: read 31 | 32 | steps: 33 | - name: Harden Runner 34 | uses: step-security/harden-runner@6b3083af2869dc3314a0257a42f4af696cc79ba3 # v2.3.1 35 | with: 36 | egress-policy: audit 37 | 38 | - name: "Checkout code" 39 | uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 40 | with: 41 | persist-credentials: false 42 | 43 | - name: "Run analysis" 44 | uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 45 | with: 46 | results_file: results.sarif 47 | results_format: sarif 48 | # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: 49 | # - you want to enable the Branch-Protection check on a *public* repository, or 50 | # - you are installing Scorecards on a *private* repository 51 | # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat. 52 | # repo_token: ${{ secrets.SCORECARD_TOKEN }} 53 | 54 | # Public repositories: 55 | # - Publish results to OpenSSF REST API for easy access by consumers 56 | # - Allows the repository to include the Scorecard badge. 57 | # - See https://github.com/ossf/scorecard-action#publishing-results. 58 | # For private repositories: 59 | # - `publish_results` will always be set to `false`, regardless 60 | # of the value entered here. 61 | publish_results: true 62 | 63 | # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF 64 | # format to the repository Actions tab. 65 | - name: "Upload artifact" 66 | uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 67 | with: 68 | name: SARIF file 69 | path: results.sarif 70 | retention-days: 5 71 | 72 | # Upload the results to GitHub's code scanning dashboard. 73 | - name: "Upload to code-scanning" 74 | uses: github/codeql-action/upload-sarif@8662eabe0e9f338a07350b7fd050732745f93848 # v2.3.1 75 | with: 76 | sarif_file: results.sarif 77 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | setenv.sh 16 | kubernetes-kms 17 | vendor 18 | *.env 19 | 20 | # Vscode files 21 | .vscode 22 | 23 | # OSX trash 24 | .DS_Store 25 | 26 | .idea/ 27 | _output/ 28 | 29 | # e2e output 30 | tests/e2e/generated_manifests/* 31 | 32 | # Go tools 33 | .tools/ 34 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | 4 | linters-settings: 5 | gocritic: 6 | enabled-tags: 7 | - performance 8 | lll: 9 | line-length: 200 10 | misspell: 11 | locale: US 12 | staticcheck: 13 | go: "1.23" 14 | 15 | linters: 16 | disable-all: true 17 | enable: 18 | - errcheck 19 | - exportloopref 20 | - forcetypeassert 21 | - goconst 22 | - gocritic 23 | - gocyclo 24 | - godot 25 | - gofmt 26 | - gofumpt 27 | - goimports 28 | - gosec 29 | - gosimple 30 | - govet 31 | - ineffassign 32 | - misspell 33 | - nakedret 34 | - prealloc 35 | - revive 36 | - staticcheck 37 | - typecheck 38 | - unused 39 | - whitespace 40 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # refer to https://goreleaser.com for more options 2 | version: 2 3 | builds: 4 | - skip: true 5 | release: 6 | prerelease: auto 7 | header: | 8 | ## {{.Tag}} - {{ time "2006-01-02" }} 9 | changelog: 10 | disable: false 11 | groups: 12 | - title: Bug Fixes 🐞 13 | regexp: ^.*fix[(\\w)]*:+.*$ 14 | - title: Build 🏭 15 | regexp: ^.*build[(\\w)]*:+.*$ 16 | - title: Code Refactoring 💎 17 | regexp: ^.*refactor[(\\w)]*:+.*$ 18 | - title: Code Style 🎶 19 | regexp: ^.*style[(\\w)]*:+.*$ 20 | - title: Continuous Integration 💜 21 | regexp: ^.*ci[(\\w)]*:+.*$ 22 | - title: Documentation 📘 23 | regexp: ^.*docs[(\\w)]*:+.*$ 24 | - title: Features 🌈 25 | regexp: ^.*feat[(\\w)]*:+.*$ 26 | - title: Maintenance 🔧 27 | regexp: ^.*chore[(\\w)]*:+.*$ 28 | - title: Performance Improvements 🚀 29 | regexp: ^.*perf[(\\w)]*:+.*$ 30 | - title: Revert Change ◀️ 31 | regexp: ^.*revert[(\\w)]*:+.*$ 32 | - title: Security Fix 🛡️ 33 | regexp: ^.*security[(\\w)]*:+.*$ 34 | - title: Testing 💚 35 | regexp: ^.*test[(\\w)]*:+.*$ 36 | -------------------------------------------------------------------------------- /.pipelines/nightly.yml: -------------------------------------------------------------------------------- 1 | trigger: none 2 | 3 | schedules: 4 | - cron: "0 0 * * *" 5 | always: true 6 | displayName: "Nightly Build & Test" 7 | branches: 8 | include: 9 | - master 10 | 11 | pool: staging-pool-amd64-mariner-2 12 | 13 | jobs: 14 | - template: templates/unit-tests-template.yml 15 | - template: templates/soak-test-template.yml 16 | - template: templates/e2e-upgrade-template.yml 17 | -------------------------------------------------------------------------------- /.pipelines/pr.yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | branches: 3 | include: 4 | - master 5 | 6 | pr: 7 | branches: 8 | include: 9 | - master 10 | paths: 11 | exclude: 12 | - docs/* 13 | - README.md 14 | - .github/* 15 | 16 | pool: staging-pool-amd64-mariner-2 17 | 18 | jobs: 19 | - template: templates/unit-tests-template.yml 20 | - template: templates/e2e-kind-template.yml 21 | -------------------------------------------------------------------------------- /.pipelines/templates/cleanup-template.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - script: | 3 | kubectl logs -l component=azure-kms-provider -n kube-system --tail -1 4 | kubectl get pods -o wide -A 5 | displayName: "Get logs" 6 | 7 | - script: make e2e-delete-kind 8 | displayName: "Delete cluster" 9 | -------------------------------------------------------------------------------- /.pipelines/templates/cluster-health-template.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - script: | 3 | kubectl wait --for=condition=ready node --all 4 | kubectl wait pod -n kube-system --for=condition=Ready --all 5 | kubectl get nodes -owide 6 | displayName: "Check cluster health" 7 | -------------------------------------------------------------------------------- /.pipelines/templates/e2e-kind-template.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: 3 | timeoutInMinutes: 15 4 | cancelTimeoutInMinutes: 5 5 | workspace: 6 | clean: all 7 | variables: 8 | - name: REGISTRY_NAME 9 | value: kind-registry 10 | - name: REGISTRY_PORT 11 | value: 5000 12 | - name: KUBERNETES_VERSION 13 | value: v1.32.3 14 | - name: KIND_CLUSTER_NAME 15 | value: kms 16 | - name: KIND_NETWORK 17 | value: kind 18 | # contains the following environment variables: 19 | # - AZURE_TENANT_ID 20 | # - KEYVAULT_NAME 21 | # - KEY_NAME 22 | # - KEY_VERSION 23 | # - USER_ASSIGNED_IDENTITY_ID 24 | - group: kubernetes-kms 25 | strategy: 26 | matrix: 27 | kmsv1_kind_v1_30_10: 28 | KUBERNETES_VERSION: v1.30.10 29 | kmsv1_kind_v1_31_6: 30 | KUBERNETES_VERSION: v1.31.6 31 | kmsv1_kind_v1_32_3: 32 | KUBERNETES_VERSION: v1.32.3 33 | steps: 34 | - task: GoTool@0 35 | inputs: 36 | version: 1.23.8 37 | - script: make e2e-install-prerequisites 38 | displayName: "Install e2e test prerequisites" 39 | - script: | 40 | make e2e-setup-kind 41 | displayName: "Setup kind cluster with azure kms plugin" 42 | env: 43 | REGISTRY_NAME: $(REGISTRY_NAME) 44 | REGISTRY_PORT: $(REGISTRY_PORT) 45 | KUBERNETES_VERSION: $(KUBERNETES_VERSION) 46 | KIND_CLUSTER_NAME: $(KIND_CLUSTER_NAME) 47 | KIND_NETWORK: $(KIND_NETWORK) 48 | - template: cluster-health-template.yml 49 | - template: kind-debug-template.yml 50 | - script: make e2e-test 51 | displayName: "Run e2e tests for KMS v1" 52 | - template: cleanup-template.yml 53 | - job: 54 | timeoutInMinutes: 15 55 | cancelTimeoutInMinutes: 5 56 | workspace: 57 | clean: all 58 | variables: 59 | - name: REGISTRY_NAME 60 | value: kind-registry 61 | - name: REGISTRY_PORT 62 | value: 5000 63 | - name: KUBERNETES_VERSION 64 | value: v1.32.3 65 | - name: KIND_CLUSTER_NAME 66 | value: kms 67 | - name: KIND_NETWORK 68 | value: kind 69 | # contains the following environment variables: 70 | # - AZURE_TENANT_ID 71 | # - KEYVAULT_NAME 72 | # - KEY_NAME 73 | # - KEY_VERSION 74 | # - USER_ASSIGNED_IDENTITY_ID 75 | - group: kubernetes-kms 76 | strategy: 77 | matrix: 78 | kmsv2_kind_v1_30_10: 79 | KUBERNETES_VERSION: v1.30.10 80 | kmsv2_kind_v1_31_6: 81 | KUBERNETES_VERSION: v1.31.6 82 | kmsv2_kind_v1_32_3: 83 | KUBERNETES_VERSION: v1.32.3 84 | steps: 85 | - task: GoTool@0 86 | inputs: 87 | version: 1.23.8 88 | - script: make e2e-install-prerequisites 89 | displayName: "Install e2e test prerequisites" 90 | - script: | 91 | make e2e-kmsv2-setup-kind 92 | displayName: "Setup kind cluster with azure kms plugin" 93 | env: 94 | REGISTRY_NAME: $(REGISTRY_NAME) 95 | REGISTRY_PORT: $(REGISTRY_PORT) 96 | KUBERNETES_VERSION: $(KUBERNETES_VERSION) 97 | KIND_CLUSTER_NAME: $(KIND_CLUSTER_NAME) 98 | KIND_NETWORK: $(KIND_NETWORK) 99 | - template: cluster-health-template.yml 100 | - template: kind-debug-template.yml 101 | - script: make e2e-kmsv2-test 102 | displayName: "Run e2e tests for KMS v2" 103 | - template: cleanup-template.yml 104 | -------------------------------------------------------------------------------- /.pipelines/templates/e2e-upgrade-template.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: e2e_upgrade_tests 3 | timeoutInMinutes: 10 4 | cancelTimeoutInMinutes: 5 5 | workspace: 6 | clean: all 7 | variables: 8 | - name: REGISTRY_NAME 9 | value: kind-registry 10 | - name: REGISTRY_PORT 11 | value: 5000 12 | - name: KUBERNETES_VERSION 13 | value: v1.23.5 14 | - name: KIND_CLUSTER_NAME 15 | value: kms 16 | - name: KIND_NETWORK 17 | value: kind 18 | # contains the following environment variables: 19 | # - AZURE_TENANT_ID 20 | # - KEYVAULT_NAME 21 | # - KEY_NAME 22 | # - KEY_VERSION 23 | # - USER_ASSIGNED_IDENTITY_ID 24 | - group: kubernetes-kms 25 | 26 | steps: 27 | - script: make e2e-install-prerequisites 28 | displayName: "Install e2e test prerequisites" 29 | 30 | - script: | 31 | . scripts/setup-local-registry.sh 32 | displayName: "Setup local registry" 33 | env: 34 | REGISTRY_NAME: $(REGISTRY_NAME) 35 | REGISTRY_PORT: $(REGISTRY_PORT) 36 | 37 | - script: | 38 | version=$(git tag -l --sort=v:refname | tail -n 1) 39 | echo "##vso[task.setvariable variable=LATEST_KMS_VERSION]$version" 40 | 41 | echo "Latest released kms version - $version" 42 | displayName: "Get latest released version" 43 | 44 | - template: manifest-template.yml 45 | parameters: 46 | registry: mcr.microsoft.com/oss/v2/azure/kms 47 | imageName: keyvault 48 | imageVersion: $(LATEST_KMS_VERSION) 49 | 50 | - script: | 51 | . scripts/setup-kind-cluster.sh & 52 | . scripts/connect-registry.sh & 53 | wait 54 | displayName: "Setup kind cluster with azure kms plugin" 55 | env: 56 | REGISTRY_NAME: $(REGISTRY_NAME) 57 | REGISTRY_PORT: $(REGISTRY_PORT) 58 | KUBERNETES_VERSION: $(KUBERNETES_VERSION) 59 | KIND_CLUSTER_NAME: $(KIND_CLUSTER_NAME) 60 | KIND_NETWORK: $(KIND_NETWORK) 61 | 62 | - template: cluster-health-template.yml 63 | - template: kind-debug-template.yml 64 | 65 | - script: make e2e-test 66 | displayName: "Run e2e tests" 67 | 68 | - script: | 69 | echo "##vso[task.setvariable variable=LOCAL_IMAGE_VERSION]$(git rev-parse --short HEAD)" 70 | displayName: "Update Image version" 71 | 72 | # This stage will upgrade kms plugin. The path (./tests/e2e/generated_manifests) is mounted in kind cluster. 73 | # Any changes in the host will automatically be reflected in /etc/kubernetes/manifests mount path and that static pod is restarted with new changes. 74 | # manifest-template updates these files with registry, imageName and version to desired upgrade values. 75 | - template: manifest-template.yml 76 | parameters: 77 | registry: localhost:$(REGISTRY_PORT) 78 | imageName: keyvault 79 | imageVersion: e2e-$(LOCAL_IMAGE_VERSION) 80 | 81 | - script: | 82 | # wait for the kind network to exist 83 | echo "waiting for upgraded kms pod to be Running" 84 | for i in $(seq 1 25); do 85 | image=$(kubectl get pods -n kube-system azure-kms-provider-kms-control-plane -o jsonpath="{.spec.containers[*].image}") 86 | phase=$(kubectl get pods -n kube-system azure-kms-provider-kms-control-plane -o jsonpath="{.status.phase}") 87 | echo "image - $image phase - $phase" 88 | if [ "${image}" == "${REGISTRY}/${IMAGE_NAME}:e2e-${LOCAL_IMAGE_VERSION}" ] && [ "${phase}" == "Running" ]; then 89 | break 90 | else 91 | sleep 5 92 | fi 93 | done 94 | # Give additional 5s for plugin to start. Remove this once https://github.com/Azure/kubernetes-kms/issues/113 is fixed. 95 | sleep 5 96 | displayName: "Wait for kms upgrade" 97 | 98 | - template: cluster-health-template.yml 99 | - template: kind-debug-template.yml 100 | 101 | - script: make e2e-test 102 | displayName: "Run e2e tests" 103 | 104 | - template: cleanup-template.yml 105 | -------------------------------------------------------------------------------- /.pipelines/templates/kind-debug-template.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - script: | 3 | docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kubernetes-kms.yaml" 4 | docker exec kms-control-plane bash -c "cat /etc/kubernetes/manifests/kube-apiserver.yaml" 5 | docker exec kms-control-plane bash -c "cat /etc/kubernetes/encryption-config.yaml" 6 | docker exec kms-control-plane bash -c "journalctl -u kubelet > kubelet.log && cat kubelet.log" 7 | docker exec kms-control-plane bash -c "cd /var/log/containers ; cat *" 8 | docker network ls 9 | displayName: "Debug logs" 10 | condition: failed() 11 | -------------------------------------------------------------------------------- /.pipelines/templates/manifest-template.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | - name: registry 3 | type: string 4 | - name: imageName 5 | type: string 6 | - name: imageVersion 7 | type: string 8 | 9 | steps: 10 | - script: | 11 | export REGISTRY=${{ parameters.registry }} 12 | export IMAGE_NAME=${{ parameters.imageName }} 13 | export IMAGE_VERSION=${{ parameters.imageVersion }} 14 | 15 | make e2e-generate-manifests 16 | 17 | echo "##vso[task.setvariable variable=REGISTRY]${{ parameters.registry }}" 18 | echo "##vso[task.setvariable variable=IMAGE_NAME]${{ parameters.imageName }}" 19 | displayName: "Generate Manifests" 20 | -------------------------------------------------------------------------------- /.pipelines/templates/scan-images-template.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - script: | 3 | export REGISTRY="e2e" 4 | export IMAGE_VERSION="test" 5 | export OUTPUT_TYPE="type=docker" 6 | make docker-init-buildx docker-build 7 | 8 | wget https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION:-0.32.0}/trivy_${TRIVY_VERSION:-0.32.0}_Linux-64bit.tar.gz 9 | tar zxvf trivy_${TRIVY_VERSION:-0.32.0}_Linux-64bit.tar.gz 10 | 11 | # show all vulnerabilities in the logs 12 | ./trivy image "${REGISTRY}/keyvault:${IMAGE_VERSION}" 13 | ./trivy image --exit-code 1 --ignore-unfixed --severity MEDIUM,HIGH,CRITICAL "${REGISTRY}/keyvault:${IMAGE_VERSION}" || exit 1 14 | displayName: "Scan images for vulnerability" 15 | -------------------------------------------------------------------------------- /.pipelines/templates/soak-test-template.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: soak_test_aks_engine 3 | timeoutInMinutes: 10 4 | cancelTimeoutInMinutes: 5 5 | 6 | workspace: 7 | clean: all 8 | 9 | variables: 10 | - group: kubernetes-kms-soak-aks-engine 11 | 12 | steps: 13 | - script: make install-soak-prerequisites 14 | displayName: "Install e2e soak test prerequisites" 15 | 16 | - task: DownloadSecureFile@1 17 | name: kubeconfig 18 | inputs: 19 | secureFile: kubeconfig 20 | displayName: "Download KUBECONFIG" 21 | 22 | - script: | 23 | export KUBECONFIG=$(kubeconfig.secureFilePath) 24 | echo "##vso[task.setvariable variable=KUBECONFIG]${KUBECONFIG}" 25 | displayName: "Set KUBECONFIG" 26 | 27 | - template: cluster-health-template.yml 28 | 29 | - script: IS_SOAK_TEST=true make e2e-test 30 | displayName: "Run e2e tests" 31 | -------------------------------------------------------------------------------- /.pipelines/templates/unit-tests-template.yml: -------------------------------------------------------------------------------- 1 | jobs: 2 | - job: unit_tests 3 | timeoutInMinutes: 10 4 | cancelTimeoutInMinutes: 5 5 | workspace: 6 | clean: all 7 | variables: 8 | # contains the following environment variables: 9 | # - AZURE_TENANT_ID 10 | # - KEYVAULT_NAME 11 | # - KEY_NAME 12 | # - KEY_VERSION 13 | # - USER_ASSIGNED_IDENTITY_ID 14 | - group: kubernetes-kms 15 | 16 | steps: 17 | - script: make lint 18 | displayName: Run lint 19 | - script: make unit-test 20 | displayName: Run unit tests 21 | - script: make build 22 | displayName: Build 23 | - script: | 24 | sudo ./_output/kubernetes-kms --version 25 | displayName: Check binary version 26 | - script: | 27 | sudo mkdir /etc/kubernetes 28 | echo -e '{\n "tenantId": "'$AZURE_TENANT_ID'",\n "useManagedIdentityExtension": true,\n "userAssignedIdentityID": "'$USER_ASSIGNED_IDENTITY_ID'",\n}' | sudo tee --append /etc/kubernetes/azure.json > /dev/null 29 | sudo chown root:root /etc/kubernetes/azure.json && sudo chmod 600 /etc/kubernetes/azure.json 30 | displayName: Setup azure.json on host 31 | - script: | 32 | sudo ./_output/kubernetes-kms --keyvault-name $KEYVAULT_NAME --key-name $KEY_NAME --key-version $KEY_VERSION --listen-addr "unix:///opt/azurekms.sock" > /dev/null & 33 | echo Waiting 2 seconds for the server to start 34 | sleep 2 35 | sudo make integration-test 36 | displayName: Run integration tests 37 | - template: scan-images-template.yml 38 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Rita Zhang 2 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Ref: https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/creating-a-repository-on-github/about-code-owners 2 | 3 | * @aramase @enj 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This code of conduct outlines expectations for participation in Microsoft-managed open source communities, as well as steps for reporting unacceptable behavior. We are committed to providing a welcoming and inspiring community for all. People violating this code of conduct may be banned from the community. 4 | 5 | Our open source communities strive to: 6 | 7 | - **Be friendly and patient:** Remember you might not be communicating in someone else's primary spoken or programming language, and others may not have your level of understanding. 8 | - **Be welcoming:** Our communities welcome and support people of all backgrounds and identities. This includes, but is not limited to members of any race, ethnicity, culture, national origin, color, immigration status, social and economic class, educational level, sex, sexual orientation, gender identity and expression, age, size, family status, political belief, religion, and mental and physical ability. 9 | - **Be respectful:** We are a world-wide community of professionals, and we conduct ourselves professionally. Disagreement is no excuse for poor behavior and poor manners. Disrespectful and unacceptable behavior includes, but is not limited to: 10 | Violent threats or language. 11 | Discriminatory or derogatory jokes and language. 12 | Posting sexually explicit or violent material. 13 | Posting, or threatening to post, people's personally identifying information ("doxing"). 14 | Insults, especially those using discriminatory terms or slurs. 15 | Behavior that could be perceived as sexual attention. 16 | Advocating for or encouraging any of the above behaviors. 17 | - **Understand disagreements:** Disagreements, both social and technical, are useful learning opportunities. Seek to understand the other viewpoints and resolve differences constructively. 18 | - This code is not exhaustive or complete. It serves to capture our common understanding of a productive, collaborative environment. We expect the code to be followed in spirit as much as in the letter. 19 | 20 | ## Scope 21 | 22 | This code of conduct applies to all repos and communities for Microsoft-managed open source projects regardless of whether or not the repo explicitly calls out its use of this code. The code also applies in public spaces when an individual is representing a project or its community. Examples include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 23 | 24 | Note: Some Microsoft-managed communities have codes of conduct that pre-date this document and issue resolution process. While communities are not required to change their code, they are expected to use the resolution process outlined here. The review team will coordinate with the communities involved to address your concerns. 25 | 26 | ## Reporting Code of Conduct Issues 27 | 28 | We encourage all communities to resolve issues on their own whenever possible. This builds a broader and deeper understanding and ultimately a healthier interaction. In the event that an issue cannot be resolved locally, please feel free to report your concerns by contacting opencode@microsoft.com. Your report will be handled in accordance with the issue resolution process described in the [Code of Conduct FAQ]. 29 | 30 | In your report please include: 31 | 32 | - Your contact information. 33 | - Names (real, usernames or pseudonyms) of any individuals involved. If there are additional witnesses, please include them as well. 34 | - Your account of what occurred, and if you believe the incident is ongoing. If there is a publicly available record (e.g. a mailing list archive or a public chat log), please include a link or attachment. 35 | - Any additional information that may be helpful. 36 | 37 | All reports will be reviewed by a multi-person team and will result in a response that is deemed necessary and appropriate to the circumstances. Where additional perspectives are needed, the team may seek insight from others with relevant expertise or experience. The confidentiality of the person reporting the incident will be kept at all times. Involved parties are never part of the review team. 38 | 39 | Anyone asked to stop unacceptable behavior is expected to comply immediately. If an individual engages in unacceptable behavior, the review team may take any action they deem appropriate, including a permanent ban from the community. 40 | 41 | *This code of conduct is based on the [template] established by the [TODO Group] and used by numerous other large communities (e.g., [Facebook], [Yahoo], [Twitter], [GitHub]) and the Scope section from the [Contributor Covenant version 1.4].* 42 | 43 | [Code of Conduct FAQ]: https://opensource.microsoft.com/codeofconduct/faq/ 44 | [template]: http://todogroup.org/opencodeofconduct 45 | [TODO Group]: http://todogroup.org/ 46 | [Facebook]: https://code.facebook.com/pages/876921332402685/open-source-code-of-conduct 47 | [Yahoo]: https://yahoo.github.io/codeofconduct 48 | [Twitter]: https://engineering.twitter.com/opensource/code-of-conduct 49 | [GitHub]: http://todogroup.org/opencodeofconduct/#opensource@github.com 50 | [Contributor Covenant version 1.4]: http://contributor-covenant.org/version/1/4/ 51 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 4 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 5 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 6 | 7 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 8 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 9 | provided by the bot. You will only need to do this once across all repos using our CLA. 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/oss/go/microsoft/golang:1.23.8-bookworm@sha256:df6c0a931c3646afea9d9858a40985a613f692467da696ef8ffc4d1996d7a6bb AS builder 2 | 3 | WORKDIR /workspace 4 | # Copy the Go Modules manifests 5 | COPY go.mod go.mod 6 | COPY go.sum go.sum 7 | # cache deps before building and copying source so that we don't need to re-download as much 8 | # and so that source changes don't invalidate our downloaded layer 9 | RUN go mod download 10 | 11 | # Copy the go source 12 | COPY cmd/server/main.go main.go 13 | COPY pkg/ pkg/ 14 | 15 | ARG TARGETARCH 16 | ARG TARGETPLATFORM 17 | ARG LDFLAGS 18 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -a -ldflags "${LDFLAGS:--X github.com/Azure/kubernetes-kms/pkg/version.BuildVersion=latest}" -o _output/kubernetes-kms main.go 19 | 20 | # Use distroless as minimal base image to package the manager binary 21 | # Refer to https://github.com/GoogleContainerTools/distroless for more details 22 | FROM --platform=${TARGETPLATFORM:-linux/amd64} mcr.microsoft.com/cbl-mariner/distroless/minimal:2.0-nonroot.20250402@sha256:c5e349966c9a8ffe5af65970300d2b6899592da1714490b46561f5d86a0ab1e0 23 | WORKDIR / 24 | COPY --from=builder /workspace/_output/kubernetes-kms . 25 | 26 | ENTRYPOINT [ "/kubernetes-kms" ] 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ORG_PATH=github.com/Azure 2 | PROJECT_NAME := kubernetes-kms 3 | REPO_PATH="$(ORG_PATH)/$(PROJECT_NAME)" 4 | 5 | REGISTRY_NAME ?= upstreamk8sci 6 | REPO_PREFIX ?= oss/azure/kms 7 | REGISTRY ?= $(REGISTRY_NAME).azurecr.io/$(REPO_PREFIX) 8 | LOCAL_REGISTRY_NAME ?= kind-registry 9 | LOCAL_REGISTRY_PORT ?= 5000 10 | IMAGE_NAME ?= keyvault 11 | IMAGE_VERSION ?= v0.7.0 12 | IMAGE_TAG := $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) 13 | CGO_ENABLED_FLAG := 0 14 | 15 | # build variables 16 | BUILD_VERSION_VAR := $(REPO_PATH)/pkg/version.BuildVersion 17 | BUILD_DATE_VAR := $(REPO_PATH)/pkg/version.BuildDate 18 | BUILD_DATE := $$(date +%Y-%m-%d-%H:%M) 19 | GIT_VAR := $(REPO_PATH)/pkg/version.GitCommit 20 | GIT_HASH := $$(git rev-parse --short HEAD) 21 | LDFLAGS ?= "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(IMAGE_VERSION) -X $(GIT_VAR)=$(GIT_HASH)" 22 | 23 | GO_FILES=$(shell go list ./... | grep -v /test/e2e) 24 | TOOLS_MOD_DIR := ./tools 25 | TOOLS_DIR := $(abspath ./.tools) 26 | 27 | # docker env var 28 | DOCKER_BUILDKIT = 1 29 | export DOCKER_BUILDKIT 30 | 31 | # Testing var 32 | KIND_VERSION ?= 0.27.0 33 | KUBERNETES_VERSION ?= v1.32.3 34 | BATS_VERSION ?= 1.4.1 35 | 36 | ## -------------------------------------- 37 | ## Linting 38 | ## -------------------------------------- 39 | 40 | $(TOOLS_DIR)/golangci-lint: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go 41 | cd $(TOOLS_MOD_DIR) && \ 42 | go build -o $(TOOLS_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint 43 | 44 | .PHONY: lint 45 | lint: $(TOOLS_DIR)/golangci-lint 46 | $(TOOLS_DIR)/golangci-lint run --timeout=5m -v 47 | 48 | ## -------------------------------------- 49 | ## Images 50 | ## -------------------------------------- 51 | 52 | ALL_LINUX_ARCH ?= amd64 arm64 53 | # Output type of docker buildx build 54 | OUTPUT_TYPE ?= type=registry 55 | 56 | BUILDX_BUILDER_NAME ?= img-builder 57 | QEMU_VERSION ?= 5.2.0-2 58 | # The architecture of the image 59 | ARCH ?= amd64 60 | 61 | .PHONY: build 62 | build: 63 | go build -a -ldflags $(LDFLAGS) -o _output/kubernetes-kms ./cmd/server/ 64 | 65 | .PHONY: docker-init-buildx 66 | docker-init-buildx: 67 | @if ! docker buildx ls | grep $(BUILDX_BUILDER_NAME); then \ 68 | docker run --rm --privileged mirror.gcr.io/multiarch/qemu-user-static:$(QEMU_VERSION) --reset -p yes; \ 69 | docker buildx create --name $(BUILDX_BUILDER_NAME) --use; \ 70 | docker buildx inspect $(BUILDX_BUILDER_NAME) --bootstrap; \ 71 | fi 72 | 73 | .PHONY: docker-build 74 | docker-build: 75 | docker buildx build \ 76 | --build-arg LDFLAGS=$(LDFLAGS) \ 77 | --no-cache \ 78 | --platform="linux/$(ARCH)" \ 79 | --output=$(OUTPUT_TYPE) \ 80 | -t $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)-linux-$(ARCH) . \ 81 | --progress=plain; \ 82 | 83 | @if [ "$(ARCH)" = "amd64" ] && [ "$(OUTPUT_TYPE)" = "type=docker" ]; then \ 84 | docker tag $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)-linux-$(ARCH) $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION); \ 85 | fi 86 | 87 | .PHONY: docker-build-all 88 | docker-build-all: 89 | @for arch in $(ALL_LINUX_ARCH); do \ 90 | $(MAKE) ARCH=$${arch} docker-build; \ 91 | done 92 | 93 | .PHONY: docker-push-manifest 94 | docker-push-manifest: 95 | docker manifest create --amend $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) $(foreach arch,$(ALL_LINUX_ARCH),$(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)-linux-$(arch)); \ 96 | for arch in $(ALL_LINUX_ARCH); do \ 97 | docker manifest annotate --os linux --arch $${arch} $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION) $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)-linux-$${arch}; \ 98 | done; \ 99 | docker manifest push --purge $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION); \ 100 | 101 | ## -------------------------------------- 102 | ## Testing 103 | ## -------------------------------------- 104 | 105 | .PHONY: integration-test 106 | integration-test: 107 | go test -v -count=1 -failfast github.com/Azure/kubernetes-kms/tests/client 108 | 109 | .PHONY: unit-test 110 | unit-test: 111 | go test -race -v -count=1 -failfast `go list ./... | grep -v client` 112 | 113 | 114 | ## -------------------------------------- 115 | ## E2E Testing 116 | ## -------------------------------------- 117 | e2e-install-prerequisites: 118 | # Download and install kind 119 | curl -L https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-linux-amd64 --output kind && chmod +x kind && sudo mv kind /usr/local/bin/ 120 | # Download and install kubectl 121 | curl -LO https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl && chmod +x ./kubectl && sudo mv kubectl /usr/local/bin/ 122 | # Download and install bats 123 | curl -sSLO https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz && tar -zxvf v${BATS_VERSION}.tar.gz && sudo bash bats-core-${BATS_VERSION}/install.sh /usr/local 124 | 125 | .PHONY: install-soak-prerequisites 126 | install-soak-prerequisites: e2e-install-prerequisites 127 | # Download and install node-shell 128 | curl -LO https://github.com/kvaps/kubectl-node-shell/raw/master/kubectl-node_shell && chmod +x ./kubectl-node_shell && sudo mv ./kubectl-node_shell /usr/local/bin/kubectl-node_shell 129 | 130 | e2e-setup-kind: setup-local-registry 131 | ./scripts/setup-kind-cluster.sh & 132 | ./scripts/connect-registry.sh & 133 | sleep 90s 134 | 135 | e2e-kmsv2-setup-kind: setup-local-registry 136 | ./scripts/setup-kmsv2-kind-cluster.sh & 137 | ./scripts/connect-registry.sh & 138 | sleep 90s 139 | 140 | .PHONY: setup-local-registry 141 | setup-local-registry: 142 | ./scripts/setup-local-registry.sh 143 | 144 | e2e-generate-manifests: 145 | @mkdir -p tests/e2e/generated_manifests 146 | envsubst < tests/e2e/azure.json > tests/e2e/generated_manifests/azure.json 147 | envsubst < tests/e2e/kms.yaml > tests/e2e/generated_manifests/kms.yaml 148 | 149 | e2e-delete-kind: 150 | # delete kind e2e cluster created for tests 151 | kind delete cluster --name kms 152 | 153 | e2e-test: 154 | # Run test suite with kind cluster 155 | bats -t tests/e2e/test.bats 156 | 157 | e2e-kmsv2-test: 158 | # Run test suite with kind cluster 159 | bats -t tests/e2e/testkmsv2.bats 160 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KMS Plugin for Key Vault 2 | 3 | [![Build Status](https://dev.azure.com/AzureContainerUpstream/Kubernetes%20KMS/_apis/build/status/Kubernetes%20KMS%20CI?branchName=master)](https://dev.azure.com/AzureContainerUpstream/Kubernetes%20KMS/_build/latest?definitionId=442&branchName=master) 4 | [![Go Report Card](https://goreportcard.com/badge/Azure/kubernetes-kms)](https://goreportcard.com/report/Azure/kubernetes-kms) 5 | ![GitHub go.mod Go version](https://img.shields.io/github/go-mod/go-version/Azure/kubernetes-kms) 6 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/Azure/kubernetes-kms) 7 | [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/Azure/kubernetes-kms/badge)](https://api.securityscorecards.dev/projects/github.com/Azure/kubernetes-kms) 8 | 9 | Enables encryption at rest of your Kubernetes data in etcd using Azure Key Vault. 10 | 11 | From the Kubernetes documentation on [Encrypting Secret Data at Rest]: 12 | 13 | > _[KMS Plugin for Key Vault is]_ the recommended choice for using a third party tool for key management. Simplifies key rotation, with a new data encryption key (DEK) generated for each encryption, and key encryption key (KEK) rotation controlled by the user. 14 | 15 | ⚠️ **NOTE**: Currently, KMS plugin for Key Vault does not support key rotation. If you create a new key version in KMS, decryption will fail since it won't match the key used for encryption when the cluster was created. 16 | 17 | 💡 **NOTE**: To integrate your application secrets from a key management system outside of Kubernetes, use [Azure Key Vault Provider for Secrets Store CSI Driver]. 18 | 19 | ## Features 20 | 21 | - Use a key in Key Vault for etcd encryption 22 | - Use a key in Key Vault protected by a Hardware Security Module (HSM) 23 | - Bring your own keys 24 | - Store secrets, keys, and certs in etcd, but manage them as part of Kubernetes 25 | 26 | ## Getting Started 27 | 28 | ### Prerequisites 29 | 30 | 💡 Make sure you have a Kubernetes cluster version 1.10 or later, the minimum version that is supported by KMS Plugin for Key Vault. 31 | 32 | ### Azure Kubernetes Service (AKS) 33 | 34 | Azure Kubernetes Service ([AKS]) creates managed, supported Kubernetes clusters on Azure. 35 | 36 | To enable encryption at rest for Kubernetes resources in etcd, check out the KMS plugin for Key Vault on AKS feature in this [doc](https://docs.microsoft.com/en-us/azure/aks/use-kms-etcd-encryption). 37 | 38 | ### Setting up KMS Plugin manually 39 | 40 | Refer to [doc](docs/manual-install.md) for steps to setup the KMS Key Vault plugin on an existing cluster. 41 | 42 | ## Verifying that Data is Encrypted 43 | 44 | Now that Azure KMS provider is running in your cluster and the encryption configuration is setup, it will encrypt the data in etcd. Let's verify that is working: 45 | 46 | 1. Create a new secret: 47 | 48 | ```bash 49 | kubectl create secret generic secret1 -n default --from-literal=mykey=mydata 50 | ``` 51 | 52 | 2. Using `etcdctl`, read the secret from etcd: 53 | 54 | ```bash 55 | sudo ETCDCTL_API=3 etcdctl --cacert=/etc/kubernetes/certs/ca.crt --cert=/etc/kubernetes/certs/etcdclient.crt --key=/etc/kubernetes/certs/etcdclient.key get /registry/secrets/default/secret1 56 | ``` 57 | 58 | 3. Check that the stored secret is prefixed with `k8s:enc:kms:v1:azurekmsprovider` when KMSv1 is used for encryption, or with `k8s:enc:kms:v2:azurekmsprovider` when KMSv2 is used. This prefix indicates that the data has been encrypted by the Azure KMS provider. 59 | 60 | 4. Verify the secret is decrypted correctly when retrieved via the Kubernetes API: 61 | 62 | ```bash 63 | kubectl get secrets secret1 -o yaml 64 | ``` 65 | 66 | The output should match `mykey: bXlkYXRh`, which is the encoded data of `mydata`. 67 | 68 | ## Rotation 69 | 70 | Refer to [doc](docs/rotation.md) for steps to rotate the KMS Key on an existing cluster. 71 | 72 | ## Metrics 73 | Refer to [doc](docs/metrics.md) for details on the metrics exposed by the KMS Key Vault plugin. 74 | 75 | ## Contributing 76 | 77 | The KMS Plugin for Key Vault project welcomes contributions and suggestions. Please see [CONTRIBUTING](CONTRIBUTING.md) for details. 78 | 79 | ## Roadmap 80 | You can view the public roadmap for the KMS plugin for Azure KeyVault on the GitHub Project [here](https://github.com/orgs/Azure/projects/440). Note that all target dates are aspirational and subject to change. 81 | 82 | ## Release 83 | 84 | Currently, this project releases monthly to patch security vulnerabilities, and bi-monthly for new features. We target the **first week** of the month for release. 85 | 86 | ## Code of conduct 87 | 88 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information, see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 89 | 90 | ## Support 91 | 92 | KMS Plugin for Key Vault is an open source project that is [**not** covered by the Microsoft Azure support policy](https://support.microsoft.com/en-us/help/2941892/support-for-linux-and-open-source-technology-in-azure). [Please search open issues here](https://github.com/Azure/kubernetes-kms/issues), and if your issue isn't already represented please [open a new one](https://github.com/Azure/kubernetes-kms/issues/new/choose). The project maintainers will respond to the best of their abilities. 93 | 94 | [aks]: https://azure.microsoft.com/services/kubernetes-service/ 95 | [encrypting secret data at rest]: https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#providers 96 | [azure key vault provider for secrets store csi driver]: https://github.com/Azure/secrets-store-csi-driver-provider-azure 97 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /cmd/server/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package main 7 | 8 | import ( 9 | "context" 10 | "flag" 11 | "fmt" 12 | "math" 13 | "net" 14 | "net/url" 15 | "os" 16 | "os/signal" 17 | "strconv" 18 | "syscall" 19 | "time" 20 | 21 | "github.com/Azure/kubernetes-kms/pkg/config" 22 | "github.com/Azure/kubernetes-kms/pkg/metrics" 23 | "github.com/Azure/kubernetes-kms/pkg/plugin" 24 | "github.com/Azure/kubernetes-kms/pkg/utils" 25 | "github.com/Azure/kubernetes-kms/pkg/version" 26 | 27 | "google.golang.org/grpc" 28 | "k8s.io/klog/v2" 29 | kmsv1 "k8s.io/kms/apis/v1beta1" 30 | kmsv2 "k8s.io/kms/apis/v2" 31 | "monis.app/mlog" 32 | ) 33 | 34 | var ( 35 | listenAddr = flag.String("listen-addr", "unix:///opt/azurekms.socket", "gRPC listen address") 36 | keyvaultName = flag.String("keyvault-name", "", "Azure Key Vault name") 37 | keyName = flag.String("key-name", "", "Azure Key Vault KMS key name") 38 | keyVersion = flag.String("key-version", "", "Azure Key Vault KMS key version") 39 | managedHSM = flag.Bool("managed-hsm", false, "Azure Key Vault Managed HSM. Refer to https://docs.microsoft.com/en-us/azure/key-vault/managed-hsm/overview for more details.") 40 | logFormatJSON = flag.Bool("log-format-json", false, "set log formatter to json") 41 | logLevel = flag.Uint("v", 0, "In order of increasing verbosity: 0=warning/error, 2=info, 4=debug, 6=trace, 10=all") 42 | // TODO remove this flag in future release. 43 | _ = flag.String("configFilePath", "/etc/kubernetes/azure.json", "[DEPRECATED] Path for Azure Cloud Provider config file") 44 | configFilePath = flag.String("config-file-path", "/etc/kubernetes/azure.json", "Path for Azure Cloud Provider config file") 45 | versionInfo = flag.Bool("version", false, "Prints the version information") 46 | 47 | healthzPort = flag.Uint("healthz-port", 8787, "port for health check") 48 | healthzPath = flag.String("healthz-path", "/healthz", "path for health check") 49 | healthzTimeout = flag.Duration("healthz-timeout", 20*time.Second, "RPC timeout for health check") 50 | metricsBackend = flag.String("metrics-backend", "prometheus", "Backend used for metrics") 51 | metricsAddress = flag.String("metrics-addr", "8095", "The address the metric endpoint binds to") 52 | 53 | proxyMode = flag.Bool("proxy-mode", false, "Proxy mode") 54 | proxyAddress = flag.String("proxy-address", "", "proxy address") 55 | proxyPort = flag.Int("proxy-port", 7788, "port for proxy") 56 | ) 57 | 58 | func main() { 59 | if err := setupKMSPlugin(); err != nil { 60 | mlog.Fatal(err) 61 | } 62 | } 63 | 64 | func setupKMSPlugin() error { 65 | defer mlog.Setup()() // set up log flushing and attempt to flush on exit 66 | flag.Parse() 67 | ctx := withShutdownSignal(context.Background()) 68 | 69 | logFormat := mlog.FormatText 70 | if *logFormatJSON { 71 | logFormat = mlog.FormatJSON 72 | } 73 | 74 | if *logLevel > math.MaxUint8 { 75 | return fmt.Errorf("invalid log level: %d", *logLevel) 76 | } 77 | 78 | if err := mlog.ValidateAndSetKlogLevelAndFormatGlobally(ctx, klog.Level(uint8(*logLevel)), logFormat); err != nil { 79 | return fmt.Errorf("invalid --log-level set: %w", err) 80 | } 81 | 82 | if *versionInfo { 83 | if err := version.PrintVersion(); err != nil { 84 | return fmt.Errorf("failed to print version: %w", err) 85 | } 86 | return nil 87 | } 88 | 89 | // initialize metrics exporter 90 | err := metrics.InitMetricsExporter(*metricsBackend, *metricsAddress) 91 | if err != nil { 92 | return fmt.Errorf("failed to initialize metrics exporter: %w", err) 93 | } 94 | 95 | mlog.Always("Starting KeyManagementServiceServer service", "version", version.BuildVersion, "buildDate", version.BuildDate) 96 | 97 | pluginConfig := &plugin.Config{ 98 | KeyVaultName: *keyvaultName, 99 | KeyName: *keyName, 100 | KeyVersion: *keyVersion, 101 | ManagedHSM: *managedHSM, 102 | ProxyMode: *proxyMode, 103 | ProxyAddress: *proxyAddress, 104 | ProxyPort: *proxyPort, 105 | ConfigFilePath: *configFilePath, 106 | } 107 | 108 | azureConfig, err := config.GetAzureConfig(pluginConfig.ConfigFilePath) 109 | if err != nil { 110 | return fmt.Errorf("failed to get azure config: %w", err) 111 | } 112 | 113 | kvClient, err := plugin.NewKeyVaultClient( 114 | azureConfig, 115 | pluginConfig.KeyVaultName, 116 | pluginConfig.KeyName, 117 | pluginConfig.KeyVersion, 118 | pluginConfig.ProxyMode, 119 | pluginConfig.ProxyAddress, 120 | pluginConfig.ProxyPort, 121 | pluginConfig.ManagedHSM, 122 | ) 123 | if err != nil { 124 | return fmt.Errorf("failed to create key vault client: %w", err) 125 | } 126 | 127 | // Initialize and run the GRPC server 128 | proto, addr, err := utils.ParseEndpoint(*listenAddr) 129 | if err != nil { 130 | return fmt.Errorf("failed to parse endpoint: %w", err) 131 | } 132 | if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { 133 | return fmt.Errorf("failed to remove socket file %s: %w", addr, err) 134 | } 135 | 136 | listener, err := net.Listen(proto, addr) 137 | if err != nil { 138 | return fmt.Errorf("failed to listen addr: %s, proto: %s: %w", addr, proto, err) 139 | } 140 | 141 | opts := []grpc.ServerOption{ 142 | grpc.UnaryInterceptor(utils.UnaryServerInterceptor), 143 | } 144 | 145 | s := grpc.NewServer(opts...) 146 | 147 | // register kms v1 server 148 | kmsV1Server, err := plugin.NewKMSv1Server(kvClient) 149 | if err != nil { 150 | return fmt.Errorf("failed to create server: %w", err) 151 | } 152 | kmsv1.RegisterKeyManagementServiceServer(s, kmsV1Server) 153 | 154 | // register kms v2 server 155 | kmsV2Server, err := plugin.NewKMSv2Server(kvClient) 156 | if err != nil { 157 | return fmt.Errorf("failed to create kms V2 server: %w", err) 158 | } 159 | kmsv2.RegisterKeyManagementServiceServer(s, kmsV2Server) 160 | 161 | mlog.Always("Listening for connections", "addr", listener.Addr().String()) 162 | go func() { 163 | if err := s.Serve(listener); err != nil { 164 | mlog.Fatal(fmt.Errorf("failed to serve kms server: %w", err)) 165 | } 166 | }() 167 | 168 | // Health check for kms v1 and v2 169 | healthz := &plugin.HealthZ{ 170 | KMSv1Server: kmsV1Server, 171 | KMSv2Server: kmsV2Server, 172 | HealthCheckURL: &url.URL{ 173 | Host: net.JoinHostPort("", strconv.FormatUint(uint64(*healthzPort), 10)), 174 | Path: *healthzPath, 175 | }, 176 | UnixSocketPath: listener.Addr().String(), 177 | RPCTimeout: *healthzTimeout, 178 | } 179 | go healthz.Serve() 180 | 181 | <-ctx.Done() 182 | // gracefully stop the grpc server 183 | mlog.Always("terminating the server") 184 | s.GracefulStop() 185 | 186 | return nil 187 | } 188 | 189 | // withShutdownSignal returns a copy of the parent context that will close if 190 | // the process receives termination signals. 191 | func withShutdownSignal(ctx context.Context) context.Context { 192 | signalChan := make(chan os.Signal, 1) 193 | signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, os.Interrupt) 194 | 195 | nctx, cancel := context.WithCancel(ctx) 196 | 197 | go func() { 198 | <-signalChan 199 | mlog.Always("received shutdown signal") 200 | cancel() 201 | }() 202 | return nctx 203 | } 204 | -------------------------------------------------------------------------------- /developers.md: -------------------------------------------------------------------------------- 1 | # Developers Guide 2 | 3 | This guide explains how to set up your environment for developing the Azure kubernetes kms service. 4 | 5 | ## Prerequisites 6 | 7 | - Go 1.9.0 or later 8 | - dep 9 | - kubectl 1.9 or later 10 | - An Azure account (needed for creating Azure key vault) 11 | - Git 12 | - make 13 | 14 | ### Structure of the Code 15 | 16 | The code for the kubernetes-kms project is organized as follows: 17 | 18 | - The built binary is located in root `./kubernetes-kms` 19 | - The `test/` directory contains `client.go`, which creates a connection against the grpc unix service at `/opt/azurekms.socket` then executes client-side API calls against the `KeyManagementService` service. This is used by the CI/CD pipeline. 20 | 21 | Go dependencies are managed with [dep](https://github.com/golang/dep) and stored in the 22 | `vendor/` directory. 23 | 24 | 25 | ### Git Conventions 26 | 27 | We use Git for our version control system. The `master` branch is the 28 | home of the current development candidate. Releases are tagged. 29 | 30 | We accept changes to the code via GitHub Pull Requests (PRs). One 31 | workflow for doing this is as follows: 32 | 33 | 1. Use `go get` to clone this repository: `go get github.com/Azure/kubernetes-kms` 34 | 2. Fork that repository into your GitHub account 35 | 3. Add your repository as a remote for `$GOPATH/github.com/Azure/kubernetes-kms` 36 | 4. Create a new working branch (`git checkout -b feat/my-feature`) and 37 | do your work on that branch. 38 | 5. When you are ready for us to review, push your branch to GitHub, and 39 | then open a new pull request with us. 40 | 41 | ### Build the Code 42 | 43 | We use `make` and `Makefile` to build the binary and the Docker image. To start the build process: 44 | 45 | 1. Run `make build` to build the binary `/kubernetes-kms` for your OS 46 | 47 | ### Run the Code Locally 48 | 49 | To test your code locally: 50 | 51 | 1. On a linux machine, you can run `sudo ./kubernetes-kms --configFilePath ` to create the gRPC unix domain socket running at `/opt/azurekms.socket`. This will start the gRPC server. 52 | 2. Create an Azure resource group, a Key Vault, and update the key vault's access policy with: 53 | 54 | ```bash 55 | az group create -n mykeyvaultrg -l eastus 56 | az keyvault create -n k8skv -g mykeyvaultrg 57 | az keyvault set-policy -n k8skv --key-permissions create decrypt encrypt get list --spn 58 | ``` 59 | If you do not have a service principal, please refer to this [doc](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli?view=azure-cli-latest). 60 | 61 | 3. Populate a `azure.json` file locally. The gRPC server will look for this file in the path provided by `configFilePath`. By default, `configFilePath` is set to `etc/kubernetes/azure.json`. 62 | 63 | ```json 64 | { 65 | "tenantId": "", 66 | "subscriptionId": "", 67 | "aadClientId": "", 68 | "aadClientSecret": "", 69 | "resourceGroup": "mykeyvaultrg", 70 | "location": "eastus", 71 | "providerVaultName": "k8skv", 72 | "providerKeyName": "mykey" 73 | } 74 | ``` 75 | 4. Test with the gRPC client, run `sudo GOPATH=[YOUR GOPATH] GOCACHE=off go test tests/client/client_test.go`. 76 | 5. Test racing condition with the gRPC client, run `sudo GOPATH=[YOUR GOPATH] go test test/client/client_test.go & sudo GOPATH=[YOUR GOPATH] go test test/client/client_test.go &`. 77 | 78 | ### Build image 79 | 1. Run `make build-image` to build the binary `/kubernetes-kms` for linux and Docker image `mcr.microsoft.com/k8s/kms/keyvault:latest` 80 | -------------------------------------------------------------------------------- /docs/manual-install.md: -------------------------------------------------------------------------------- 1 | # 🛠 Manual Configurations # 2 | 3 | This guide demonstrates steps required to enable the KMS Plugin for Key Vault in an existing cluster. 4 | 5 | ### 1. Create a Keyvault 6 | 7 | If you're bringing your own keys, skip this step. 8 | 9 | ```bash 10 | KEYVAULT_NAME=k8skv 11 | RG=mykubernetesrg 12 | LOC=eastus 13 | 14 | # create resource group that'll contain the keyvault instance 15 | az group create -n $RG -l $LOC 16 | # create keyvault 17 | az keyvault create -n $KV_NAME -g $RG 18 | # create key that will be used for encryption 19 | az keyvault key create -n k8s --vault-name $KV_NAME --kty RSA --size 2048 20 | ``` 21 | 22 | ### 2. Give the cluster identity permissions to access the keys in keyvault 23 | 24 | The KMS Plugin uses the cluster service principal or managed identity to access the keyvault instance. 25 | 26 | #### More on authentication methods 27 | 28 | [`/etc/kubernetes/azure.json`](https://kubernetes-sigs.github.io/cloud-provider-azure/install/configs/) is a well-known JSON file in each node that provides the details about which method KMS Plugin uses for access to Keyvault: 29 | 30 | | Authentication method | `/etc/kubernetes/azure.json` fields used | 31 | | -------------------------------- | ------------------------------------------------------------------------------------------- | 32 | | System-assigned managed identity | `useManagedIdentityExtension: true` and `userAssignedIdentityID:""` | 33 | | User-assigned managed identity | `useManagedIdentityExtension: true` and `userAssignedIdentityID:""` | 34 | | Service principal (default) | `aadClientID: ""` and `aadClientSecret: ""` | 35 | 36 | #### Obtaining the ID of the cluster managed identity/service principal 37 | 38 | After your cluster is provisioned, depending on your cluster identity configuration, run one of the following commands to retrieve the **ID** of your managed identity or service principal, which will be used for role assignment to access Keyvault: 39 | 40 | | Cluster configuration | Command | 41 | | ---------------------------------- | -------------------------------------------------------------------------------------------------------------- | 42 | | AKS cluster with service principal | `az aks show -g -n --query servicePrincipalProfile.clientId -otsv` | 43 | | AKS cluster with managed identity | `az aks show -g -n --query identityProfile.kubeletidentity.clientId -otsv` | 44 | 45 | Assign the following permissions: 46 | 47 | ```bash 48 | az keyvault set-policy -n $KEYVAULT_NAME --key-permissions decrypt encrypt --spn 49 | ``` 50 | 51 | ### 3. Deploy the KMS Plugin 52 | 53 | For all Kubernetes control plane nodes, add the static pod manifest to `/etc/kubernetes/manifests` 54 | 55 | ```yaml 56 | apiVersion: v1 57 | kind: Pod 58 | metadata: 59 | name: azure-kms-provider 60 | namespace: kube-system 61 | labels: 62 | tier: control-plane 63 | component: azure-kms-provider 64 | spec: 65 | priorityClassName: system-node-critical 66 | hostNetwork: true 67 | containers: 68 | - name: azure-kms-provider 69 | image: mcr.microsoft.com/oss/v2/azure/kms/keyvault:v0.8.0 70 | imagePullPolicy: IfNotPresent 71 | args: 72 | - --listen-addr=unix:///opt/azurekms.socket # [OPTIONAL] gRPC listen address. Default is unix:///opt/azurekms.socket 73 | - --keyvault-name=${KV_NAME} # [REQUIRED] Name of the keyvault. Must match criteria specified at https://docs.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name 74 | - --key-name=${KEY_NAME} # [REQUIRED] Name of the keyvault key used for encrypt/decrypt 75 | - --key-version=${KEY_VERSION} # [REQUIRED] Version of the key to use 76 | - --log-format-json=false # [OPTIONAL] Set log formatter to json. Default is false. 77 | - --healthz-port=8787 # [OPTIONAL] port for health check. Default is 8787 78 | - --healthz-path=/healthz # [OPTIONAL] path for health check. Default is /healthz 79 | - --healthz-timeout=20s # [OPTIONAL] RPC timeout for health check. Default is 20s 80 | - --managed-hsm=false # [OPTIONAL] Use Azure Key Vault managed HSM. Default is false. 81 | - -v=1 82 | securityContext: 83 | allowPrivilegeEscalation: false 84 | capabilities: 85 | drop: 86 | - ALL 87 | readOnlyRootFilesystem: true 88 | runAsUser: 0 89 | ports: 90 | - containerPort: 8787 # Must match the value defined in --healthz-port 91 | protocol: TCP 92 | livenessProbe: 93 | httpGet: 94 | path: /healthz # Must match the value defined in --healthz-path 95 | port: 8787 # Must match the value defined in --healthz-port 96 | failureThreshold: 2 97 | periodSeconds: 10 98 | resources: 99 | requests: 100 | cpu: 100m 101 | memory: 128Mi 102 | limits: 103 | cpu: 4 104 | memory: 2Gi 105 | volumeMounts: 106 | - name: etc-kubernetes 107 | mountPath: /etc/kubernetes 108 | - name: etc-ssl 109 | mountPath: /etc/ssl 110 | readOnly: true 111 | - name: sock 112 | mountPath: /opt 113 | volumes: 114 | - name: etc-kubernetes 115 | hostPath: 116 | path: /etc/kubernetes 117 | - name: etc-ssl 118 | hostPath: 119 | path: /etc/ssl 120 | - name: sock 121 | hostPath: 122 | path: /opt 123 | ``` 124 | 125 | View logs from the kms pod: 126 | 127 | ```bash 128 | kubectl logs -l component=azure-kms-provider -n kube-system 129 | 130 | I0219 17:35:33.608840 1 main.go:60] "Starting KeyManagementServiceServer service" version="v0.0.11" buildDate="2021-02-19-17:33" 131 | I0219 17:35:33.609090 1 azure_config.go:27] populating AzureConfig from /etc/kubernetes/azure.json 132 | I0219 17:35:33.609420 1 auth.go:66] "azure: using client_id+client_secret to retrieve access token" clientID="9a7a##### REDACTED #####bb26" clientSecret="23T.##### REDACTED #####vw-r" 133 | I0219 17:35:33.609568 1 keyvault.go:66] "using kms key for encrypt/decrypt" vaultName="k8skmskv" keyName="key1" keyVersion="5cdf48ea6bb9456ebf637e1130b7751a" 134 | I0219 17:35:33.609897 1 main.go:86] Listening for connections on address: /opt/azurekms.socket 135 | ... 136 | ``` 137 | 138 | ### 4. Create encryption configuration 139 | 140 | Create a new encryption configuration file `/etc/kubernetes/manifests/encryptionconfig.yaml` using the appropriate properties for the `kms` provider: 141 | 142 | ```yaml 143 | kind: EncryptionConfiguration 144 | apiVersion: apiserver.config.k8s.io/v1 145 | resources: 146 | - resources: # List of kubernetes resources that will be encrypted in etcd using the KMS plugin 147 | - secrets 148 | providers: 149 | - kms: 150 | name: azurekmsprovider 151 | endpoint: unix:///opt/azurekms.socket # This endpoint must match the value defined in --listen-addr for the KMS plugin 152 | cachesize: 1000 153 | - identity: {} 154 | ``` 155 | 156 | The encryption configuration file needs to be accessible by all the api servers. 157 | 158 | ### 5. Modify `/etc/kubernetes/kube-apiserver.yaml` 159 | 160 | Add the following flag: 161 | 162 | ```yaml 163 | --encryption-provider-config=/etc/kubernetes/encryptionconfig.yaml 164 | ``` 165 | 166 | Mount `/opt` to access the socket: 167 | 168 | ```yaml 169 | ... 170 | volumeMounts: 171 | - name: "sock" 172 | mountPath: "/opt" 173 | ... 174 | volumes: 175 | - name: "sock" 176 | hostPath: 177 | path: "/opt" 178 | ``` 179 | 180 | ### 6. Restart your API server 181 | -------------------------------------------------------------------------------- /docs/metrics.md: -------------------------------------------------------------------------------- 1 | # Metrics provided by KMS plugin for Key Vault 2 | 3 | This project uses [opentelemetry](https://opentelemetry.io/) for reporting metrics. Please refer to it's status [here](https://github.com/open-telemetry/opentelemetry-go#project-status). Prometheus is the only exporter that's currently supported. 4 | 5 | ## List of metrics provided by the kms plugin 6 | 7 | | Metric | Description | Tags | 8 | | ------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | 9 | | kms_request | Distribution of how long it took for an operation | `status=success OR error`

`operation=encrypt OR decrypt OR grpc_encrypt OR grpc_decrypt`

`error_message` | 10 | 11 | 12 | ### Sample Metrics output 13 | 14 | ```shell 15 | # HELP kms_request Distribution of how long it took for an operation 16 | # TYPE kms_request histogram 17 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.1"} 18 18 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.13492828476735633"} 18 19 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.18205642030260802"} 18 20 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.24564560522315804"} 18 21 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.3314454017339986"} 18 22 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.4472135954999578"} 18 23 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.6034176336545162"} 18 24 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.8141810630738084"} 18 25 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.0985605433061172"} 18 26 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.4822688982138947"} 18 27 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.9999999999999991"} 18 28 | kms_request_bucket{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="+Inf"} 18 29 | kms_request_sum{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 1.010053082 30 | kms_request_count{operation="decrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 18 31 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.1"} 19 32 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.13492828476735633"} 19 33 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.18205642030260802"} 19 34 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.24564560522315804"} 19 35 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.3314454017339986"} 19 36 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.4472135954999578"} 19 37 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.6034176336545162"} 19 38 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.8141810630738084"} 19 39 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.0985605433061172"} 19 40 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.4822688982138947"} 19 41 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.9999999999999991"} 19 42 | kms_request_bucket{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="+Inf"} 19 43 | kms_request_sum{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 1.021080768 44 | kms_request_count{operation="encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 19 45 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.1"} 1 46 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.13492828476735633"} 1 47 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.18205642030260802"} 1 48 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.24564560522315804"} 1 49 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.3314454017339986"} 1 50 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.4472135954999578"} 1 51 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.6034176336545162"} 1 52 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.8141810630738084"} 1 53 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.0985605433061172"} 1 54 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.4822688982138947"} 1 55 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.9999999999999991"} 1 56 | kms_request_bucket{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="+Inf"} 1 57 | kms_request_sum{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 0.053279316 58 | kms_request_count{operation="grpc_encrypt",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 1 59 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.1"} 0 60 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.13492828476735633"} 11 61 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.18205642030260802"} 13 62 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.24564560522315804"} 13 63 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.3314454017339986"} 13 64 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.4472135954999578"} 13 65 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.6034176336545162"} 14 66 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.8141810630738084"} 14 67 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.0985605433061172"} 14 68 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.4822688982138947"} 14 69 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.9999999999999991"} 14 70 | kms_request_bucket{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="+Inf"} 14 71 | kms_request_sum{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 2.1240865880000004 72 | kms_request_count{operation="grpc_status",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 14 73 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.1"} 9 74 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.13492828476735633"} 9 75 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.18205642030260802"} 9 76 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.24564560522315804"} 9 77 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.3314454017339986"} 9 78 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.4472135954999578"} 9 79 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.6034176336545162"} 9 80 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="0.8141810630738084"} 9 81 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.0985605433061172"} 9 82 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.4822688982138947"} 9 83 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="1.9999999999999991"} 9 84 | kms_request_bucket{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success",le="+Inf"} 9 85 | kms_request_sum{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 0.0007254060000000001 86 | kms_request_count{operation="grpc_version",otel_scope_name="keyvaultkms",otel_scope_version="",status="success"} 9 87 | ``` 88 | -------------------------------------------------------------------------------- /docs/rotation.md: -------------------------------------------------------------------------------- 1 | # Rotating KMS key 2 | 3 | This guide demonstrates steps required to update your cluster to use a new KMS key for encryption. 4 | 5 | > NOTE: Ensure to read the Kubernetes documentation on [Rotating a decryption key](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/#rotating-a-decryption-key) before proceeding with the guide. 6 | 7 | ### 1. Generate a new key or rotate the existing key 8 | 9 | * If this is a new key in a different keyvault, then give the cluster identity permissions to access the keys in keyvault. Refer to [doc](./manual-install.md#2-give-the-cluster-identity-permissions-to-access-the-keys-in-keyvault) for details. 10 | * If this is a new version of the same key that's already being used, then proceed to the next step. 11 | 12 | ### 2. Deploy another instance of KMS plugin with new key 13 | 14 | To rotate the encrypt/decrypt key in the cluster, you'll need to run 2 kms plugin pods simultaneously listening on different unix sockets before making the transition. 15 | 16 | For all Kubernetes control plane nodes, add the static pod manifest to `/etc/kubernetes/manifests` 17 | 18 | ```yaml 19 | apiVersion: v1 20 | kind: Pod 21 | metadata: 22 | name: azure-kms-provider-2 23 | namespace: kube-system 24 | labels: 25 | tier: control-plane 26 | component: azure-kms-provider 27 | spec: 28 | priorityClassName: system-node-critical 29 | hostNetwork: true 30 | containers: 31 | - name: azure-kms-provider 32 | image: mcr.microsoft.com/oss/v2/azure/kms/keyvault:v0.8.0 33 | imagePullPolicy: IfNotPresent 34 | args: 35 | - --listen-addr=unix:///opt/azurekms2.socket # unix:///opt/azurekms.socket is used by the primary kms plugin pod. So use a different listen address here for the new kms plugin pod. 36 | - --keyvault-name=${KV_NAME} # [REQUIRED] Name of the keyvault 37 | - --key-name=${KEY_NAME} # [REQUIRED] Name of the keyvault key used for encrypt/decrypt 38 | - --key-version=${KEY_VERSION} # [REQUIRED] Version of the key to use 39 | - --log-format-json=false # [OPTIONAL] Set log formatter to json. Default is false. 40 | - --healthz-port=8788 # The port used here should be different than the one used by the primary kms plugin pod. 41 | - --healthz-path=/healthz # [OPTIONAL] path for health check. Default is /healthz 42 | - --healthz-timeout=20s # [OPTIONAL] RPC timeout for health check. Default is 20s 43 | - --managed-hsm=false # [OPTIONAL] Use Azure Key Vault managed HSM. Default is false. 44 | - -v=5 45 | securityContext: 46 | allowPrivilegeEscalation: false 47 | capabilities: 48 | drop: 49 | - ALL 50 | readOnlyRootFilesystem: true 51 | runAsUser: 0 52 | ports: 53 | - containerPort: 8788 # Must match the value defined in --healthz-port 54 | protocol: TCP 55 | livenessProbe: 56 | httpGet: 57 | path: /healthz # Must match the value defined in --healthz-path 58 | port: 8788 # Must match the value defined in --healthz-port 59 | failureThreshold: 2 60 | periodSeconds: 10 61 | resources: 62 | requests: 63 | cpu: 100m 64 | memory: 128Mi 65 | limits: 66 | cpu: "4" 67 | memory: 2Gi 68 | volumeMounts: 69 | - name: etc-kubernetes 70 | mountPath: /etc/kubernetes 71 | - name: etc-ssl 72 | mountPath: /etc/ssl 73 | readOnly: true 74 | - name: sock 75 | mountPath: /opt 76 | volumes: 77 | - name: etc-kubernetes 78 | hostPath: 79 | path: /etc/kubernetes 80 | - name: etc-ssl 81 | hostPath: 82 | path: /etc/ssl 83 | - name: sock 84 | hostPath: 85 | path: /opt 86 | nodeSelector: 87 | kubernetes.io/os: linux 88 | ``` 89 | 90 | View logs from the kms pod: 91 | 92 | ```bash 93 | kubectl logs -l component=azure-kms-provider -n kube-system 94 | 95 | I0219 17:35:33.608840 1 main.go:60] "Starting KeyManagementServiceServer service" version="v0.0.11" buildDate="2021-02-19-17:33" 96 | I0219 17:35:33.609090 1 azure_config.go:27] populating AzureConfig from /etc/kubernetes/azure.json 97 | I0219 17:35:33.609420 1 auth.go:66] "azure: using client_id+client_secret to retrieve access token" clientID="9a7a##### REDACTED #####bb26" clientSecret="23T.##### REDACTED #####vw-r" 98 | I0219 17:35:33.609568 1 keyvault.go:66] "using kms key for encrypt/decrypt" vaultName="k8skmskv" keyName="key1" keyVersion="5cdf48ea6bb9456ebf637e1130b7751a" 99 | I0219 17:35:33.609897 1 main.go:86] Listening for connections on address: /opt/azurekms2.socket 100 | ... 101 | ``` 102 | 103 | ### 3. Add the new provider to encryption configuration in `/etc/kubernetes/manifests/encryptionconfig.yaml` 104 | 105 | ```yaml 106 | kind: EncryptionConfiguration 107 | apiVersion: apiserver.config.k8s.io/v1 108 | resources: 109 | - resources: # List of kubernetes resources that will be encrypted in etcd using the KMS plugin 110 | - secrets 111 | providers: 112 | - kms: 113 | name: azurekmsprovider 114 | endpoint: unix:///opt/azurekms.socket # This endpoint must match the value defined in --listen-addr for the KMS plugin using old key 115 | cachesize: 1000 116 | - kms: 117 | name: azurekmsprovider2 118 | endpoint: unix:///opt/azurekms2.socket # This endpoint must match the value defined in --listen-addr for the KMS plugin using new key 119 | cachesize: 1000 120 | ``` 121 | 122 | ### 4. Restart all `kube-apiserver` 123 | 124 | * Proceed to the next step if using a single `kube-apiserver` 125 | * If using multiple control plane nodes, restart the `kube-apiserver` to ensure each server can still decrypt using the new key in the encryption config. 126 | * To validate the decryption still works, run `kubectl get secret -o yaml` with one of the existing secrets to confirm the data is returned and is valid. 127 | 128 | ### 5. Switch the order of provider in the encryption config 129 | 130 | ```yaml 131 | kind: EncryptionConfiguration 132 | apiVersion: apiserver.config.k8s.io/v1 133 | resources: 134 | - resources: # List of kubernetes resources that will be encrypted in etcd using the KMS plugin 135 | - secrets 136 | providers: 137 | # kms provider with new key 138 | - kms: 139 | name: azurekmsprovider2 140 | endpoint: unix:///opt/azurekms2.socket # This endpoint must match the value defined in --listen-addr for the KMS plugin using new key 141 | cachesize: 1000 142 | # kms provider with old key 143 | - kms: 144 | name: azurekmsprovider 145 | endpoint: unix:///opt/azurekms.socket # This endpoint must match the value defined in --listen-addr for the KMS plugin using old key 146 | cachesize: 1000 147 | ``` 148 | 149 | ### 6. Restart all `kube-apiserver` again 150 | 151 | Refer to [step 4](#4-restart-all-kube-apiserver) to again restart the `kube-apiserver` for the encryption config changes to take effect. 152 | 153 | ### 7. Decrypt and re-encrypt existing secrets with new key 154 | 155 | Since secrets are encrypted on write, performing an update on a secret will encrypt that content. 156 | 157 | Run `kubectl get secrets --all-namespaces -o json | kubectl replace -f -` to encrypt all existing secrets with the new key. 158 | 159 | > NOTE: For larger clusters, you may wish to subdivide the secrets by namespace or script an update. 160 | 161 | #### How does this work? 162 | 163 | The first provider in the encryption configuration is used for new encrypt calls. For decrypt, all existing kms providers in encryption configuration will be tried until one of the decrypt call succeeds. 164 | 165 | ### 8. Remove the old provider from encryption configuration 166 | 167 | Now that all the secrets have been re-encrypted with the new key, we can safely remove the old kms provider from the encryption configuration. 168 | 169 | ```yaml 170 | kind: EncryptionConfiguration 171 | apiVersion: apiserver.config.k8s.io/v1 172 | resources: 173 | - resources: # List of kubernetes resources that will be encrypted in etcd using the KMS plugin 174 | - secrets 175 | providers: 176 | # kms provider with new key 177 | - kms: 178 | name: azurekmsprovider2 179 | endpoint: unix:///opt/azurekms2.socket # This endpoint must match the value defined in --listen-addr for the KMS plugin using new key 180 | cachesize: 1000 181 | ``` 182 | -------------------------------------------------------------------------------- /docs/testing.md: -------------------------------------------------------------------------------- 1 | # End-to-end testing for KMS Plugin for Keyvault 2 | 3 | ## Prerequisites 4 | 5 | To run tests locally, following components are required: 6 | 7 | 1. [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) 8 | 1. [bats](https://bats-core.readthedocs.io/en/latest/installation.html) 9 | 1. [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) 10 | 11 | To install the prerequisites, run the following command: 12 | 13 | ```bash 14 | make e2e-install-prerequisites 15 | ``` 16 | 17 | The E2E test suite extracts runtime configurations through environment variables. Below is a list of environment variables to set before running the E2E test suite. 18 | | Variable | Description | 19 | | ------------------- | --------------------------------------------------------------------------------------------------- | 20 | | AZURE_CLIENT_ID | The client ID of your service principal that has `encrypt, decrypt` access to the keyvault key. | 21 | | AZURE_CLIENT_SECRET | The client secret of your service principal that has `encrypt, decrypt` access to the keyvault key. | 22 | | AZURE_TENANT_ID | The Azure tenant ID. | 23 | | KEYVAULT_NAME | The Azure Keyvault name. | 24 | | KEY_NAME | The name of Keyvault key that will be used by the kms plugin. | 25 | | KEY_VERSION | The version of Keyvault key that will be used by the kms plugin. | 26 | 27 | ## Running the tests 28 | 29 | The e2e tests are run against a [kind](https://kind.sigs.k8s.io/) cluster that's created as part of the test script. The script also creates a local docker registry that's used for test images. 30 | 31 | 1. Setup cluster, registry and build image: 32 | 33 | ```bash 34 | make e2e-setup-kind 35 | ``` 36 | 37 | - This creates the local registry 38 | - Builds a kms plugin image with the latest changes and pushes to local registry 39 | - Creates a kind cluster with connectivity to local registry and kms plugin enabled with custom image 40 | 41 | 1. Run the end-to-end tests: 42 | 43 | ```bash 44 | make e2e-test 45 | ``` 46 | 47 | 1. To delete the kind cluster after running tests: 48 | 49 | ```bash 50 | make e2e-delete-kind 51 | ``` 52 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Azure/kubernetes-kms 2 | 3 | go 1.23.8 4 | 5 | require ( 6 | github.com/Azure/azure-sdk-for-go v68.0.0+incompatible 7 | github.com/Azure/go-autorest/autorest v0.11.28 8 | github.com/Azure/go-autorest/autorest/adal v0.9.23 9 | go.opentelemetry.io/otel v1.15.1 10 | go.opentelemetry.io/otel/exporters/prometheus v0.38.1 11 | go.opentelemetry.io/otel/metric v0.38.1 12 | golang.org/x/crypto v0.37.0 13 | golang.org/x/net v0.39.0 14 | google.golang.org/grpc v1.58.3 15 | gopkg.in/yaml.v3 v3.0.1 16 | k8s.io/apimachinery v0.27.1 17 | k8s.io/klog/v2 v2.100.1 18 | k8s.io/kms v0.27.1 19 | monis.app/mlog v0.0.4 20 | ) 21 | 22 | require ( 23 | github.com/Azure/go-autorest v14.2.0+incompatible // indirect 24 | github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect 25 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 26 | github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect 27 | github.com/Azure/go-autorest/logger v0.2.1 // indirect 28 | github.com/Azure/go-autorest/tracing v0.6.0 // indirect 29 | github.com/beorn7/perks v1.0.1 // indirect 30 | github.com/blang/semver/v4 v4.0.0 // indirect 31 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 32 | github.com/davecgh/go-spew v1.1.1 // indirect 33 | github.com/go-logr/logr v1.2.4 // indirect 34 | github.com/go-logr/stdr v1.2.2 // indirect 35 | github.com/go-logr/zapr v1.2.3 // indirect 36 | github.com/gogo/protobuf v1.3.2 // indirect 37 | github.com/golang-jwt/jwt/v4 v4.5.2 // indirect 38 | github.com/golang/protobuf v1.5.3 // indirect 39 | github.com/google/gofuzz v1.2.0 // indirect 40 | github.com/google/uuid v1.3.0 // indirect; indirectgit 41 | github.com/inconshreveable/mousetrap v1.0.1 // indirect 42 | github.com/json-iterator/go v1.1.12 // indirect 43 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 44 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 45 | github.com/modern-go/reflect2 v1.0.2 // indirect 46 | github.com/pmezard/go-difflib v1.0.0 // indirect 47 | github.com/prometheus/client_golang v1.15.0 48 | github.com/prometheus/client_model v0.3.0 // indirect 49 | github.com/prometheus/common v0.42.0 // indirect 50 | github.com/prometheus/procfs v0.9.0 // indirect 51 | github.com/spf13/cobra v1.6.1 // indirect 52 | github.com/spf13/pflag v1.0.5 // indirect 53 | github.com/stretchr/testify v1.8.2 // indirect 54 | go.opentelemetry.io/otel/sdk v1.15.1 // indirect 55 | go.opentelemetry.io/otel/sdk/metric v0.38.1 56 | go.opentelemetry.io/otel/trace v1.15.1 // indirect 57 | go.uber.org/atomic v1.10.0 // indirect 58 | go.uber.org/multierr v1.8.0 // indirect 59 | go.uber.org/zap v1.24.0 // indirect 60 | golang.org/x/sys v0.32.0 // indirect 61 | golang.org/x/text v0.24.0 // indirect 62 | google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect 63 | google.golang.org/protobuf v1.33.0 // indirect 64 | gopkg.in/inf.v0 v0.9.1 // indirect 65 | gopkg.in/yaml.v2 v2.4.0 // indirect 66 | k8s.io/component-base v0.27.1 // indirect 67 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 68 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 69 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 70 | ) 71 | -------------------------------------------------------------------------------- /pkg/auth/auth.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package auth 7 | 8 | import ( 9 | "crypto/rsa" 10 | "crypto/x509" 11 | "fmt" 12 | "net/http" 13 | "os" 14 | "regexp" 15 | 16 | "github.com/Azure/kubernetes-kms/pkg/config" 17 | "github.com/Azure/kubernetes-kms/pkg/consts" 18 | 19 | "github.com/Azure/go-autorest/autorest" 20 | "github.com/Azure/go-autorest/autorest/adal" 21 | "github.com/Azure/go-autorest/autorest/azure" 22 | "golang.org/x/crypto/pkcs12" 23 | "monis.app/mlog" 24 | ) 25 | 26 | // GetKeyvaultToken() returns token for Keyvault endpoint. 27 | func GetKeyvaultToken(config *config.AzureConfig, env *azure.Environment, resource string, proxyMode bool) (authorizer autorest.Authorizer, err error) { 28 | servicePrincipalToken, err := GetServicePrincipalToken(config, env.ActiveDirectoryEndpoint, resource, proxyMode) 29 | if err != nil { 30 | return nil, err 31 | } 32 | authorizer = autorest.NewBearerAuthorizer(servicePrincipalToken) 33 | return authorizer, nil 34 | } 35 | 36 | // GetServicePrincipalToken creates a new service principal token based on the configuration. 37 | func GetServicePrincipalToken(config *config.AzureConfig, aadEndpoint, resource string, proxyMode bool) (adal.OAuthTokenProvider, error) { 38 | oauthConfig, err := adal.NewOAuthConfig(aadEndpoint, config.TenantID) 39 | if err != nil { 40 | return nil, fmt.Errorf("failed to create OAuth config, error: %v", err) 41 | } 42 | 43 | if config.UseManagedIdentityExtension { 44 | mlog.Info("using managed identity extension to retrieve access token") 45 | msiEndpoint, err := adal.GetMSIVMEndpoint() 46 | if err != nil { 47 | return nil, fmt.Errorf("failed to get managed service identity endpoint, error: %v", err) 48 | } 49 | // using user-assigned managed identity to access keyvault 50 | if len(config.UserAssignedIdentityID) > 0 { 51 | mlog.Info("using User-assigned managed identity to retrieve access token", "clientID", redactClientCredentials(config.UserAssignedIdentityID)) 52 | return adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, 53 | resource, 54 | config.UserAssignedIdentityID) 55 | } 56 | mlog.Info("using system-assigned managed identity to retrieve access token") 57 | // using system-assigned managed identity to access keyvault 58 | return adal.NewServicePrincipalTokenFromMSI( 59 | msiEndpoint, 60 | resource) 61 | } 62 | 63 | if len(config.ClientSecret) > 0 && len(config.ClientID) > 0 { 64 | mlog.Info("azure: using client_id+client_secret to retrieve access token", 65 | "clientID", redactClientCredentials(config.ClientID), "clientSecret", redactClientCredentials(config.ClientSecret)) 66 | 67 | spt, err := adal.NewServicePrincipalToken( 68 | *oauthConfig, 69 | config.ClientID, 70 | config.ClientSecret, 71 | resource) 72 | if err != nil { 73 | return nil, err 74 | } 75 | if proxyMode { 76 | return addTargetTypeHeader(spt), nil 77 | } 78 | return spt, nil 79 | } 80 | 81 | if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 { 82 | mlog.Info("using jwt client_assertion (client_cert+client_private_key) to retrieve access token") 83 | certData, err := os.ReadFile(config.AADClientCertPath) 84 | if err != nil { 85 | return nil, fmt.Errorf("failed to read client certificate from file %s, error: %v", config.AADClientCertPath, err) 86 | } 87 | certificate, privateKey, err := decodePkcs12(certData, config.AADClientCertPassword) 88 | if err != nil { 89 | return nil, fmt.Errorf("failed to decode the client certificate, error: %v", err) 90 | } 91 | spt, err := adal.NewServicePrincipalTokenFromCertificate( 92 | *oauthConfig, 93 | config.ClientID, 94 | certificate, 95 | privateKey, 96 | resource) 97 | if err != nil { 98 | return nil, err 99 | } 100 | if proxyMode { 101 | return addTargetTypeHeader(spt), nil 102 | } 103 | return spt, nil 104 | } 105 | 106 | return nil, fmt.Errorf("no credentials provided for accessing keyvault") 107 | } 108 | 109 | // ParseAzureEnvironment returns azure environment by name. 110 | func ParseAzureEnvironment(cloudName string) (*azure.Environment, error) { 111 | var env azure.Environment 112 | var err error 113 | if cloudName == "" { 114 | env = azure.PublicCloud 115 | } else { 116 | env, err = azure.EnvironmentFromName(cloudName) 117 | } 118 | return &env, err 119 | } 120 | 121 | // decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and 122 | // the private RSA key. 123 | func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) { 124 | privateKey, certificate, err := pkcs12.Decode(pkcs, password) 125 | if err != nil { 126 | return nil, nil, fmt.Errorf("decoding the PKCS#12 client certificate: %v", err) 127 | } 128 | rsaPrivateKey, isRsaKey := privateKey.(*rsa.PrivateKey) 129 | if !isRsaKey { 130 | return nil, nil, fmt.Errorf("PKCS#12 certificate must contain a RSA private key") 131 | } 132 | 133 | return certificate, rsaPrivateKey, nil 134 | } 135 | 136 | // redactClientCredentials applies regex to a sensitive string and return the redacted value. 137 | func redactClientCredentials(sensitiveString string) string { 138 | r := regexp.MustCompile(`^(\S{4})(\S|\s)*(\S{4})$`) 139 | return r.ReplaceAllString(sensitiveString, "$1##### REDACTED #####$3") 140 | } 141 | 142 | // addTargetTypeHeader adds the target header if proxy mode is enabled. 143 | func addTargetTypeHeader(spt *adal.ServicePrincipalToken) *adal.ServicePrincipalToken { 144 | spt.SetSender(autorest.CreateSender( 145 | (func() autorest.SendDecorator { 146 | return func(s autorest.Sender) autorest.Sender { 147 | return autorest.SenderFunc(func(r *http.Request) (*http.Response, error) { 148 | r.Header.Set(consts.RequestHeaderTargetType, consts.TargetTypeAzureActiveDirectory) 149 | return s.Do(r) 150 | }) 151 | } 152 | })())) 153 | return spt 154 | } 155 | -------------------------------------------------------------------------------- /pkg/auth/auth_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package auth 7 | 8 | import ( 9 | "reflect" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/Azure/kubernetes-kms/pkg/config" 14 | 15 | "github.com/Azure/go-autorest/autorest/adal" 16 | "github.com/Azure/go-autorest/autorest/azure" 17 | ) 18 | 19 | func TestParseAzureEnvironment(t *testing.T) { 20 | envNamesArray := []string{"AZURECHINACLOUD", "AZUREGERMANCLOUD", "AZUREPUBLICCLOUD", "AZUREUSGOVERNMENTCLOUD", ""} 21 | for _, envName := range envNamesArray { 22 | azureEnv, err := ParseAzureEnvironment(envName) 23 | if err != nil { 24 | t.Fatalf("expected no error, got %v", err) 25 | } 26 | if strings.EqualFold(envName, "") && !strings.EqualFold(azureEnv.Name, "AZUREPUBLICCLOUD") { 27 | t.Fatalf("string doesn't match, expected AZUREPUBLICCLOUD, got %s", azureEnv.Name) 28 | } else if !strings.EqualFold(envName, "") && !strings.EqualFold(envName, azureEnv.Name) { 29 | t.Fatalf("string doesn't match, expected %s, got %s", envName, azureEnv.Name) 30 | } 31 | } 32 | 33 | wrongEnvName := "AZUREWRONGCLOUD" 34 | _, err := ParseAzureEnvironment(wrongEnvName) 35 | if err == nil { 36 | t.Fatalf("expected error for wrong azure environment name") 37 | } 38 | } 39 | 40 | func TestRedactClientCredentials(t *testing.T) { 41 | tests := []struct { 42 | name string 43 | clientID string 44 | expected string 45 | }{ 46 | { 47 | name: "should redact client id", 48 | clientID: "aabc0000-a83v-9h4m-000j-2c0a66b0c1f9", 49 | expected: "aabc##### REDACTED #####c1f9", 50 | }, 51 | } 52 | 53 | for _, test := range tests { 54 | t.Run(test.name, func(t *testing.T) { 55 | actual := redactClientCredentials(test.clientID) 56 | if actual != test.expected { 57 | t.Fatalf("expected: %s, got %s", test.expected, actual) 58 | } 59 | }) 60 | } 61 | } 62 | 63 | func TestGetServicePrincipalTokenFromMSIWithUserAssignedID(t *testing.T) { 64 | tests := []struct { 65 | name string 66 | config *config.AzureConfig 67 | proxyMode bool // The proxy mode doesn't matter if user-assigned managed identity is used to get service principal token 68 | }{ 69 | { 70 | name: "using user-assigned managed identity to access keyvault", 71 | config: &config.AzureConfig{ 72 | UseManagedIdentityExtension: true, 73 | UserAssignedIdentityID: "clientID", 74 | TenantID: "TenantID", 75 | ClientID: "AADClientID", 76 | ClientSecret: "AADClientSecret", 77 | }, 78 | proxyMode: false, 79 | }, 80 | // The Azure service principal is ignored when 81 | // UseManagedIdentityExtension is set to true 82 | { 83 | name: "using user-assigned managed identity over service principal if set to true", 84 | config: &config.AzureConfig{ 85 | UseManagedIdentityExtension: true, 86 | UserAssignedIdentityID: "clientID", 87 | }, 88 | proxyMode: true, 89 | }, 90 | } 91 | 92 | for _, test := range tests { 93 | t.Run(test.name, func(t *testing.T) { 94 | token, err := GetServicePrincipalToken(test.config, "https://login.microsoftonline.com/", "https://vault.azure.net", test.proxyMode) 95 | if err != nil { 96 | t.Fatalf("expected err to be nil, got: %v", err) 97 | } 98 | msiEndpoint, err := adal.GetMSIVMEndpoint() 99 | if err != nil { 100 | t.Fatalf("expected err to be nil, got: %v", err) 101 | } 102 | spt, err := adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, "https://vault.azure.net", "clientID") 103 | if err != nil { 104 | t.Fatalf("expected err to be nil, got: %v", err) 105 | } 106 | if !reflect.DeepEqual(token, spt) { 107 | t.Fatalf("expected: %v, got: %v", spt, token) 108 | } 109 | }) 110 | } 111 | } 112 | 113 | func TestGetServicePrincipalTokenFromMSI(t *testing.T) { 114 | tests := []struct { 115 | name string 116 | config *config.AzureConfig 117 | proxyMode bool // The proxy mode doesn't matter if MSI is used to get service principal token 118 | }{ 119 | { 120 | name: "using system-assigned managed identity to access keyvault", 121 | config: &config.AzureConfig{ 122 | UseManagedIdentityExtension: true, 123 | }, 124 | proxyMode: false, 125 | }, 126 | // The Azure service principal is ignored when 127 | // UseManagedIdentityExtension is set to true 128 | { 129 | name: "using system-assigned managed identity over service principal if set to true", 130 | config: &config.AzureConfig{ 131 | UseManagedIdentityExtension: true, 132 | TenantID: "TenantID", 133 | ClientID: "AADClientID", 134 | ClientSecret: "AADClientSecret", 135 | }, 136 | proxyMode: true, 137 | }, 138 | } 139 | 140 | for _, test := range tests { 141 | t.Run(test.name, func(t *testing.T) { 142 | token, err := GetServicePrincipalToken(test.config, "https://login.microsoftonline.com/", "https://vault.azure.net", test.proxyMode) 143 | if err != nil { 144 | t.Fatalf("expected err to be nil, got: %v", err) 145 | } 146 | msiEndpoint, err := adal.GetMSIVMEndpoint() 147 | if err != nil { 148 | t.Fatalf("expected err to be nil, got: %v", err) 149 | } 150 | spt, err := adal.NewServicePrincipalTokenFromMSI(msiEndpoint, "https://vault.azure.net") 151 | if err != nil { 152 | t.Fatalf("expected err to be nil, got: %v", err) 153 | } 154 | if !reflect.DeepEqual(token, spt) { 155 | t.Fatalf("expected: %v, got: %v", spt, token) 156 | } 157 | }) 158 | } 159 | } 160 | 161 | func TestGetServicePrincipalToken(t *testing.T) { 162 | tests := []struct { 163 | name string 164 | config *config.AzureConfig 165 | }{ 166 | { 167 | name: "using service-principal credentials to access keyvault", 168 | config: &config.AzureConfig{ 169 | TenantID: "TenantID", 170 | ClientID: "AADClientID", 171 | ClientSecret: "AADClientSecret", 172 | }, 173 | }, 174 | } 175 | 176 | for _, test := range tests { 177 | t.Run(test.name, func(t *testing.T) { 178 | token, err := GetServicePrincipalToken(test.config, "https://login.microsoftonline.com/", "https://vault.azure.net", false) 179 | if err != nil { 180 | t.Fatalf("expected err to be nil, got: %v", err) 181 | } 182 | env := &azure.PublicCloud 183 | 184 | oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, test.config.TenantID) 185 | if err != nil { 186 | t.Fatalf("expected err to be nil, got: %v", err) 187 | } 188 | spt, err := adal.NewServicePrincipalToken(*oauthConfig, test.config.ClientID, test.config.ClientSecret, "https://vault.azure.net") 189 | if err != nil { 190 | t.Fatalf("expected err to be nil, got: %v", err) 191 | } 192 | if !reflect.DeepEqual(token, spt) { 193 | t.Fatalf("expected: %+v, got: %+v", spt, token) 194 | } 195 | }) 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /pkg/config/azure_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "gopkg.in/yaml.v3" 8 | "monis.app/mlog" 9 | ) 10 | 11 | // AzureConfig is representing /etc/kubernetes/azure.json. 12 | type AzureConfig struct { 13 | Cloud string `json:"cloud" yaml:"cloud"` 14 | TenantID string `json:"tenantId" yaml:"tenantId"` 15 | ClientID string `json:"aadClientId" yaml:"aadClientId"` 16 | ClientSecret string `json:"aadClientSecret" yaml:"aadClientSecret"` 17 | UseManagedIdentityExtension bool `json:"useManagedIdentityExtension,omitempty" yaml:"useManagedIdentityExtension,omitempty"` 18 | UserAssignedIdentityID string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"` 19 | AADClientCertPath string `json:"aadClientCertPath" yaml:"aadClientCertPath"` 20 | AADClientCertPassword string `json:"aadClientCertPassword" yaml:"aadClientCertPassword"` 21 | } 22 | 23 | // GetAzureConfig returns configs in the azure.json cloud provider file. 24 | func GetAzureConfig(configFile string) (config *AzureConfig, err error) { 25 | cfg := AzureConfig{} 26 | 27 | mlog.Trace("populating AzureConfig from config file", "configFile", configFile) 28 | bytes, err := os.ReadFile(configFile) 29 | if err != nil { 30 | return nil, fmt.Errorf("failed to load config file %s, error: %+v", configFile, err) 31 | } 32 | if err = yaml.Unmarshal(bytes, &cfg); err != nil { 33 | return nil, fmt.Errorf("failed to unmarshal azure.json, error: %+v", err) 34 | } 35 | return &cfg, nil 36 | } 37 | -------------------------------------------------------------------------------- /pkg/consts/consts.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package consts 7 | 8 | const ( 9 | // In proxy mode, the header is added into the requests from kms-plugin. 10 | // The proxy will check the header and forward the request to different destinations. 11 | // e.g. When the value of the header "x-azure-proxy-target" is "KeyVault", the request 12 | // is forwared to Azure Key Vault by the proxy. 13 | RequestHeaderTargetType = "x-azure-proxy-target" 14 | TargetTypeAzureActiveDirectory = "AzureActiveDirectory" 15 | TargetTypeKeyVault = "KeyVault" 16 | ) 17 | -------------------------------------------------------------------------------- /pkg/metrics/exporter.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "monis.app/mlog" 8 | ) 9 | 10 | const ( 11 | prometheusExporter = "prometheus" 12 | ) 13 | 14 | // InitMetricsExporter initializes new exporter. 15 | func InitMetricsExporter(metricsBackend, metricsAddress string) error { 16 | exporter := strings.ToLower(metricsBackend) 17 | mlog.Always("metrics backend", "exporter", exporter) 18 | 19 | switch exporter { 20 | // Prometheus is the only exporter supported for now 21 | case prometheusExporter: 22 | return initPrometheusExporter(metricsAddress) 23 | default: 24 | return fmt.Errorf("unsupported metrics backend %v", metricsBackend) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /pkg/metrics/exporter_test.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | ) 7 | 8 | func TestInitMetricsExporter(t *testing.T) { 9 | testCases := []struct { 10 | name string 11 | metricsBackend string 12 | metricsAddress string 13 | expectedError bool 14 | }{ 15 | { 16 | name: "With_Prometheus_Backend", 17 | metricsBackend: "prometheus", 18 | metricsAddress: "8095", 19 | expectedError: false, 20 | }, 21 | { 22 | name: "With_Non_Prometheus_Backend", 23 | metricsBackend: "nonprometheus", 24 | expectedError: true, 25 | }, 26 | { 27 | name: "With_Uppercase_Backend_Name", 28 | metricsBackend: "Prometheus", 29 | metricsAddress: "8096", 30 | expectedError: false, 31 | }, 32 | } 33 | 34 | for _, testCase := range testCases { 35 | t.Run(testCase.name, func(t *testing.T) { 36 | err := InitMetricsExporter(testCase.metricsBackend, testCase.metricsAddress) 37 | 38 | if testCase.expectedError && err == nil || !testCase.expectedError && err != nil { 39 | t.Fatalf("expected error: %v, found: %v", testCase.expectedError, err) 40 | } 41 | 42 | // Reset handler to test /metrics repeatedly. 43 | http.DefaultServeMux = new(http.ServeMux) 44 | }) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/metrics/prometheus_exporter.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | 8 | cgprometheus "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promhttp" 10 | "go.opentelemetry.io/otel/exporters/prometheus" 11 | "go.opentelemetry.io/otel/metric/global" 12 | "go.opentelemetry.io/otel/sdk/metric" 13 | "go.opentelemetry.io/otel/sdk/metric/aggregation" 14 | "monis.app/mlog" 15 | ) 16 | 17 | const ( 18 | metricsEndpoint = "metrics" 19 | ) 20 | 21 | func initPrometheusExporter(metricsAddress string) error { 22 | exporter, err := prometheus.New() 23 | if err != nil { 24 | return err 25 | } 26 | 27 | meterProvider := metric.NewMeterProvider( 28 | metric.WithReader(exporter), 29 | metric.WithView( 30 | metric.NewView( 31 | metric.Instrument{ 32 | Kind: metric.InstrumentKindHistogram, 33 | }, 34 | metric.Stream{ 35 | Aggregation: aggregation.ExplicitBucketHistogram{ 36 | // Use custom buckets to avoid the default buckets which are too small for our use case. 37 | // Start 100ms with last bucket being [~4m, +Inf) 38 | Boundaries: cgprometheus.ExponentialBucketsRange(0.1, 2, 11), 39 | }, 40 | }, 41 | ), 42 | ), 43 | ) 44 | global.SetMeterProvider(meterProvider) 45 | 46 | http.HandleFunc(fmt.Sprintf("/%s", metricsEndpoint), promhttp.Handler().ServeHTTP) 47 | go func() { 48 | server := &http.Server{ 49 | Addr: fmt.Sprintf(":%s", metricsAddress), 50 | ReadHeaderTimeout: 5 * time.Second, 51 | } 52 | if err := server.ListenAndServe(); err != nil { 53 | mlog.Fatal(err, "failed to register prometheus endpoint", "metricsAddress", metricsAddress) 54 | } 55 | }() 56 | mlog.Always("Prometheus metrics server running", "address", metricsAddress) 57 | 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /pkg/metrics/stats_reporter.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | 6 | "go.opentelemetry.io/otel/attribute" 7 | "go.opentelemetry.io/otel/metric" 8 | "go.opentelemetry.io/otel/metric/global" 9 | ) 10 | 11 | const ( 12 | instrumentationName = "keyvaultkms" 13 | errorMessageKey = "error_message" 14 | statusTypeKey = "status" 15 | operationTypeKey = "operation" 16 | kmsRequestMetricName = "kms_request" 17 | // ErrorStatusTypeValue sets status tag to "error". 18 | ErrorStatusTypeValue = "error" 19 | // SuccessStatusTypeValue sets status tag to "success". 20 | SuccessStatusTypeValue = "success" 21 | // EncryptOperationTypeValue sets operation tag to "encrypt". 22 | EncryptOperationTypeValue = "encrypt" 23 | // DecryptOperationTypeValue sets operation tag to "decrypt". 24 | DecryptOperationTypeValue = "decrypt" 25 | // GrpcOperationTypeValue sets operation tag to "grpc". 26 | GrpcOperationTypeValue = "grpc" 27 | ) 28 | 29 | type reporter struct { 30 | histogram metric.Float64Histogram 31 | } 32 | 33 | // StatsReporter reports metrics. 34 | type StatsReporter interface { 35 | ReportRequest(ctx context.Context, operationType, status string, duration float64, errors ...string) 36 | } 37 | 38 | // NewStatsReporter instantiates otel reporter. 39 | func NewStatsReporter() (StatsReporter, error) { 40 | meter := global.Meter(instrumentationName) 41 | 42 | metricCounter, err := meter.Float64Histogram( 43 | kmsRequestMetricName, 44 | metric.WithDescription("Distribution of how long it took for an operation"), 45 | ) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | return &reporter{ 51 | histogram: metricCounter, 52 | }, nil 53 | } 54 | 55 | func (r *reporter) ReportRequest(ctx context.Context, operationType, status string, duration float64, errors ...string) { 56 | labels := []attribute.KeyValue{ 57 | attribute.String(operationTypeKey, operationType), 58 | attribute.String(statusTypeKey, status), 59 | } 60 | 61 | // Add errors 62 | if (status == ErrorStatusTypeValue) && len(errors) > 0 { 63 | for _, err := range errors { 64 | labels = append(labels, attribute.String(errorMessageKey, err)) 65 | } 66 | } 67 | 68 | r.histogram.Record(ctx, duration, metric.WithAttributes(labels...)) 69 | } 70 | -------------------------------------------------------------------------------- /pkg/plugin/healthz.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "net" 12 | "net/http" 13 | "net/url" 14 | "time" 15 | 16 | "github.com/Azure/kubernetes-kms/pkg/version" 17 | 18 | "google.golang.org/grpc" 19 | "google.golang.org/grpc/credentials/insecure" 20 | "k8s.io/apimachinery/pkg/util/uuid" 21 | kmsv1 "k8s.io/kms/apis/v1beta1" 22 | kmsv2 "k8s.io/kms/apis/v2" 23 | "monis.app/mlog" 24 | ) 25 | 26 | const ( 27 | healthCheckPlainText = "healthcheck" 28 | ) 29 | 30 | // HealthZ is the health check server for the KMS plugin. 31 | type HealthZ struct { 32 | KMSv1Server *KeyManagementServiceServer 33 | KMSv2Server *KeyManagementServiceV2Server 34 | HealthCheckURL *url.URL 35 | UnixSocketPath string 36 | RPCTimeout time.Duration 37 | } 38 | 39 | // Serve creates the http handler for serving health requests. 40 | func (h *HealthZ) Serve() { 41 | serveMux := http.NewServeMux() 42 | serveMux.HandleFunc(h.HealthCheckURL.EscapedPath(), h.ServeHTTP) 43 | server := &http.Server{ 44 | Addr: h.HealthCheckURL.Host, 45 | ReadHeaderTimeout: 5 * time.Second, 46 | Handler: serveMux, 47 | } 48 | if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 49 | mlog.Fatal(err, "failed to start health check server", "url", h.HealthCheckURL.String()) 50 | } 51 | } 52 | 53 | func (h *HealthZ) ServeHTTP(w http.ResponseWriter, _ *http.Request) { 54 | mlog.Trace("Started health check") 55 | ctx, cancel := context.WithTimeout(context.Background(), h.RPCTimeout) 56 | defer cancel() 57 | 58 | conn, err := h.dialUnixSocket() 59 | if err != nil { 60 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 61 | return 62 | } 63 | defer conn.Close() 64 | 65 | // create the kms client for v1 66 | kmsClient := kmsv1.NewKeyManagementServiceClient(conn) 67 | 68 | // create the kms client for v2 69 | kmsV2Client := kmsv2.NewKeyManagementServiceClient(conn) 70 | 71 | // check version response against KMS-Plugin's gRPC endpoint. 72 | err = h.checkRPC(ctx, kmsClient, kmsV2Client) 73 | if err != nil { 74 | http.Error(w, err.Error(), http.StatusServiceUnavailable) 75 | return 76 | } 77 | 78 | // Both encryption and decryption calls are made for each version, 79 | // resulting in a total of 4 calls to the keyvault. 80 | // Additionally, a health check is performed every 10 seconds. 81 | 82 | // v1 checks 83 | // check the configured keyvault, key, key version and permissions are still 84 | // valid to encrypt and decrypt with test data. 85 | enc, err := h.KMSv1Server.Encrypt(ctx, &kmsv1.EncryptRequest{Plain: []byte(healthCheckPlainText)}) 86 | if err != nil { 87 | http.Error(w, err.Error(), http.StatusInternalServerError) 88 | return 89 | } 90 | dec, err := h.KMSv1Server.Decrypt(ctx, &kmsv1.DecryptRequest{Cipher: enc.Cipher}) 91 | if err != nil { 92 | http.Error(w, err.Error(), http.StatusInternalServerError) 93 | return 94 | } 95 | if string(dec.Plain) != healthCheckPlainText { 96 | http.Error(w, "plain text mismatch after decryption", http.StatusInternalServerError) 97 | return 98 | } 99 | 100 | // v2 checks. 101 | // appending a string to UUID allows us to differentiate the UUIDs generated by us from those generated by the API server. 102 | uid := "local-healthz-check-" + string(uuid.NewUUID()) 103 | 104 | v2EncryptResponse, err := h.KMSv2Server.Encrypt( 105 | ctx, 106 | &kmsv2.EncryptRequest{ 107 | Plaintext: []byte(healthCheckPlainText), 108 | Uid: uid, 109 | }, 110 | ) 111 | if err != nil { 112 | http.Error(w, err.Error(), http.StatusInternalServerError) 113 | return 114 | } 115 | 116 | v2DecryptResponse, err := h.KMSv2Server.Decrypt(ctx, &kmsv2.DecryptRequest{ 117 | Ciphertext: v2EncryptResponse.Ciphertext, 118 | KeyId: v2EncryptResponse.KeyId, 119 | Uid: uid, // passing the same uid to track roundtrip encrypt/decrypt calls 120 | Annotations: v2EncryptResponse.Annotations, 121 | }) 122 | if err != nil { 123 | http.Error(w, err.Error(), http.StatusInternalServerError) 124 | return 125 | } 126 | 127 | if string(v2DecryptResponse.Plaintext) != healthCheckPlainText { 128 | http.Error(w, "plain text mismatch after decryption with KMSv2", http.StatusInternalServerError) 129 | return 130 | } 131 | 132 | w.WriteHeader(http.StatusOK) 133 | if _, err = w.Write([]byte("ok")); err != nil { 134 | http.Error(w, err.Error(), http.StatusInternalServerError) 135 | return 136 | } 137 | mlog.Trace("Completed health check") 138 | } 139 | 140 | // checkRPC initiates a grpc request to validate the socket is responding 141 | // sends a KMS VersionRequest and checks if the VersionResponse is valid. 142 | func (h *HealthZ) checkRPC( 143 | ctx context.Context, 144 | kmsV1Client kmsv1.KeyManagementServiceClient, 145 | kmsV2Client kmsv2.KeyManagementServiceClient, 146 | ) error { 147 | v, err := kmsV1Client.Version(ctx, &kmsv1.VersionRequest{}) 148 | if err != nil { 149 | return err 150 | } 151 | if v.Version != version.KMSv1APIVersion || v.RuntimeName != version.Runtime || v.RuntimeVersion != version.BuildVersion { 152 | return fmt.Errorf("failed to get correct version response") 153 | } 154 | 155 | v2Status, err := kmsV2Client.Status(ctx, &kmsv2.StatusRequest{}) 156 | if err != nil { 157 | return err 158 | } 159 | if v2Status.Version != version.KMSv2APIVersion { 160 | return fmt.Errorf( 161 | "failed to get correct version response for v2 expected: %s, got: %s", 162 | version.KMSv2APIVersion, 163 | v2Status.Version, 164 | ) 165 | } 166 | 167 | return nil 168 | } 169 | 170 | func (h *HealthZ) dialUnixSocket() (*grpc.ClientConn, error) { 171 | return grpc.Dial( 172 | h.UnixSocketPath, 173 | grpc.WithTransportCredentials(insecure.NewCredentials()), 174 | grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) { 175 | return (&net.Dialer{}).DialContext(ctx, "unix", target) 176 | }), 177 | ) 178 | } 179 | -------------------------------------------------------------------------------- /pkg/plugin/healthz_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "io" 12 | "net" 13 | "net/http" 14 | "net/http/httptest" 15 | "net/url" 16 | "os" 17 | "testing" 18 | "time" 19 | 20 | "github.com/Azure/kubernetes-kms/pkg/metrics" 21 | mockkeyvault "github.com/Azure/kubernetes-kms/pkg/plugin/mock_keyvault" 22 | 23 | "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault" 24 | "google.golang.org/grpc" 25 | kmsv1 "k8s.io/kms/apis/v1beta1" 26 | kmsv2 "k8s.io/kms/apis/v2" 27 | "monis.app/mlog" 28 | ) 29 | 30 | func TestServe(t *testing.T) { 31 | tests := []struct { 32 | desc string 33 | setEncryptResponse string 34 | setDecryptResponse string 35 | setEncryptError error 36 | setDecryptError error 37 | expectedHTTPStatusCode int 38 | }{ 39 | { 40 | desc: "failed to encrypt in health check", 41 | setEncryptResponse: "", 42 | setEncryptError: fmt.Errorf("failed to encrypt"), 43 | expectedHTTPStatusCode: http.StatusServiceUnavailable, 44 | }, 45 | { 46 | desc: "failed to decrypt in health check", 47 | setEncryptResponse: "", 48 | setEncryptError: nil, 49 | setDecryptResponse: "", 50 | setDecryptError: fmt.Errorf("failed to decrypt"), 51 | expectedHTTPStatusCode: http.StatusServiceUnavailable, 52 | }, 53 | { 54 | desc: "encrypt-decrypt mismatch", 55 | setEncryptResponse: "bar", 56 | setEncryptError: nil, 57 | setDecryptResponse: "foo", 58 | setDecryptError: nil, 59 | expectedHTTPStatusCode: http.StatusServiceUnavailable, 60 | }, 61 | { 62 | desc: "successful health check", 63 | setEncryptResponse: "bar", 64 | setDecryptResponse: "healthcheck", 65 | expectedHTTPStatusCode: http.StatusOK, 66 | }, 67 | } 68 | 69 | for _, test := range tests { 70 | t.Run(test.desc, func(t *testing.T) { 71 | socketPath := fmt.Sprintf("%s/kms.sock", getTempTestDir(t)) 72 | defer os.Remove(socketPath) 73 | 74 | fakeKMSServer, fakeKMSV2Server, mockKVClient, err := setupFakeKMSServer(socketPath) 75 | if err != nil { 76 | t.Fatalf("failed to create fake kms server, err: %+v", err) 77 | } 78 | 79 | mockKVClient.SetEncryptResponse([]byte(test.setEncryptResponse), test.setEncryptError) 80 | mockKVClient.SetDecryptResponse([]byte(test.setDecryptResponse), test.setDecryptError) 81 | 82 | healthz := &HealthZ{ 83 | KMSv1Server: fakeKMSServer, 84 | KMSv2Server: fakeKMSV2Server, 85 | UnixSocketPath: socketPath, 86 | RPCTimeout: 20 * time.Second, 87 | HealthCheckURL: &url.URL{ 88 | Scheme: "http", 89 | Host: net.JoinHostPort("localhost", "8080"), 90 | Path: "/healthz", 91 | }, 92 | } 93 | 94 | server := httptest.NewServer(healthz) 95 | defer server.Close() 96 | 97 | respCode, body := doHealthCheck(t, server.URL) 98 | if respCode != test.expectedHTTPStatusCode { 99 | t.Fatalf("expected status code: %v, got: %v", test.expectedHTTPStatusCode, respCode) 100 | } 101 | if test.expectedHTTPStatusCode == http.StatusOK && string(body) != "ok" { 102 | t.Fatalf("expected response body to be 'ok', got: %s", string(body)) 103 | } 104 | }) 105 | } 106 | } 107 | 108 | func TestCheckRPC(t *testing.T) { 109 | socketPath := fmt.Sprintf("%s/kms.sock", getTempTestDir(t)) 110 | defer os.Remove(socketPath) 111 | 112 | fakeKMSV1Server, fakeKMSV2Server, mockKVClient, err := setupFakeKMSServer(socketPath) 113 | if err != nil { 114 | t.Fatalf("failed to create fake kms server, err: %+v", err) 115 | } 116 | healthz := &HealthZ{ 117 | KMSv1Server: fakeKMSV1Server, 118 | KMSv2Server: fakeKMSV2Server, 119 | UnixSocketPath: socketPath, 120 | } 121 | mockKVClient.SetEncryptResponse([]byte(healthCheckPlainText), nil) 122 | mockKVClient.SetDecryptResponse([]byte(healthCheckPlainText), nil) 123 | 124 | conn, err := healthz.dialUnixSocket() 125 | if err != nil { 126 | t.Fatalf("failed to create connection, err: %+v", err) 127 | } 128 | 129 | err = healthz.checkRPC( 130 | context.TODO(), 131 | kmsv1.NewKeyManagementServiceClient(conn), 132 | kmsv2.NewKeyManagementServiceClient(conn), 133 | ) 134 | if err != nil { 135 | t.Fatalf("expected err to be nil, got: %+v", err) 136 | } 137 | } 138 | 139 | func getTempTestDir(t *testing.T) string { 140 | tmpDir, err := os.MkdirTemp("", "ut") 141 | if err != nil { 142 | t.Fatalf("expected err to be nil, got: %+v", err) 143 | } 144 | return tmpDir 145 | } 146 | 147 | func setupFakeKMSServer(socketPath string) ( 148 | *KeyManagementServiceServer, 149 | *KeyManagementServiceV2Server, 150 | *mockkeyvault.KeyVaultClient, 151 | error, 152 | ) { 153 | listener, err := net.Listen("unix", socketPath) 154 | if err != nil { 155 | return nil, nil, nil, err 156 | } 157 | 158 | statsReporter, err := metrics.NewStatsReporter() 159 | if err != nil { 160 | return nil, nil, nil, err 161 | } 162 | 163 | kvClient := &mockkeyvault.KeyVaultClient{ 164 | KeyID: "mock-key-id", 165 | Algorithm: keyvault.RSA15, 166 | } 167 | fakeKMSV1Server := &KeyManagementServiceServer{ 168 | kvClient: kvClient, 169 | reporter: statsReporter, 170 | } 171 | 172 | fakeKMSV2Server := &KeyManagementServiceV2Server{ 173 | kvClient: kvClient, 174 | reporter: statsReporter, 175 | } 176 | 177 | s := grpc.NewServer() 178 | kmsv1.RegisterKeyManagementServiceServer(s, fakeKMSV1Server) 179 | kmsv2.RegisterKeyManagementServiceServer(s, fakeKMSV2Server) 180 | go func() { 181 | if err := s.Serve(listener); err != nil { 182 | mlog.Fatal(err, "failed to serve fake kms server") 183 | } 184 | }() 185 | 186 | return fakeKMSV1Server, fakeKMSV2Server, kvClient, nil 187 | } 188 | 189 | func doHealthCheck(t *testing.T, url string) (int, []byte) { 190 | req, err := http.NewRequest(http.MethodGet, url, nil) 191 | if err != nil { 192 | t.Fatalf("failed to create new http request, err: %+v", err) 193 | } 194 | resp, err := http.DefaultClient.Do(req) 195 | if err != nil { 196 | t.Fatalf("failed to invoke http request, err: %+v", err) 197 | } 198 | defer resp.Body.Close() 199 | body, err := io.ReadAll(resp.Body) 200 | if err != nil { 201 | t.Fatalf("failed to read response body, err: %+v", err) 202 | } 203 | return resp.StatusCode, body 204 | } 205 | -------------------------------------------------------------------------------- /pkg/plugin/keyvault.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "context" 10 | "crypto/sha256" 11 | "encoding/base64" 12 | "fmt" 13 | "net/url" 14 | "path" 15 | "regexp" 16 | "strings" 17 | 18 | "github.com/Azure/kubernetes-kms/pkg/auth" 19 | "github.com/Azure/kubernetes-kms/pkg/config" 20 | "github.com/Azure/kubernetes-kms/pkg/consts" 21 | "github.com/Azure/kubernetes-kms/pkg/utils" 22 | "github.com/Azure/kubernetes-kms/pkg/version" 23 | 24 | kv "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault" 25 | "github.com/Azure/go-autorest/autorest" 26 | "github.com/Azure/go-autorest/autorest/azure" 27 | "k8s.io/kms/pkg/service" 28 | "monis.app/mlog" 29 | ) 30 | 31 | // encryptionResponseVersion is validated prior to decryption. 32 | // This is helpful in case we want to change anything about the data we send in the future. 33 | var encryptionResponseVersion = "1" 34 | 35 | const ( 36 | dateAnnotationKey = "date.azure.akv.io" 37 | requestIDAnnotationKey = "x-ms-request-id.azure.akv.io" 38 | keyvaultRegionAnnotationKey = "x-ms-keyvault-region.azure.akv.io" 39 | versionAnnotationKey = "version.azure.akv.io" 40 | algorithmAnnotationKey = "algorithm.azure.akv.io" 41 | dateAnnotationValue = "Date" 42 | requestIDAnnotationValue = "X-Ms-Request-Id" 43 | keyvaultRegionAnnotationValue = "X-Ms-Keyvault-Region" 44 | ) 45 | 46 | // Client interface for interacting with Keyvault. 47 | type Client interface { 48 | Encrypt( 49 | ctx context.Context, 50 | plain []byte, 51 | encryptionAlgorithm kv.JSONWebKeyEncryptionAlgorithm, 52 | ) (*service.EncryptResponse, error) 53 | Decrypt( 54 | ctx context.Context, 55 | cipher []byte, 56 | encryptionAlgorithm kv.JSONWebKeyEncryptionAlgorithm, 57 | apiVersion string, 58 | annotations map[string][]byte, 59 | decryptRequestKeyID string, 60 | ) ([]byte, error) 61 | GetUserAgent() string 62 | GetVaultURL() string 63 | } 64 | 65 | // KeyVaultClient is a client for interacting with Keyvault. 66 | type KeyVaultClient struct { 67 | baseClient kv.BaseClient 68 | config *config.AzureConfig 69 | vaultName string 70 | keyName string 71 | keyVersion string 72 | vaultURL string 73 | keyIDHash string 74 | azureEnvironment *azure.Environment 75 | } 76 | 77 | // NewKeyVaultClient returns a new key vault client to use for kms operations. 78 | func NewKeyVaultClient( 79 | config *config.AzureConfig, 80 | vaultName, keyName, keyVersion string, 81 | proxyMode bool, 82 | proxyAddress string, 83 | proxyPort int, 84 | managedHSM bool, 85 | ) (Client, error) { 86 | // Sanitize vaultName, keyName, keyVersion. (https://github.com/Azure/kubernetes-kms/issues/85) 87 | vaultName = utils.SanitizeString(vaultName) 88 | keyName = utils.SanitizeString(keyName) 89 | keyVersion = utils.SanitizeString(keyVersion) 90 | 91 | // this should be the case for bring your own key, clusters bootstrapped with 92 | // aks-engine or aks and standalone kms plugin deployments 93 | if len(vaultName) == 0 || len(keyName) == 0 || len(keyVersion) == 0 { 94 | return nil, fmt.Errorf("key vault name, key name and key version are required") 95 | } 96 | kvClient := kv.New() 97 | err := kvClient.AddToUserAgent(version.GetUserAgent()) 98 | if err != nil { 99 | return nil, fmt.Errorf("failed to add user agent to keyvault client, error: %+v", err) 100 | } 101 | env, err := auth.ParseAzureEnvironment(config.Cloud) 102 | if err != nil { 103 | return nil, fmt.Errorf("failed to parse cloud environment: %s, error: %+v", config.Cloud, err) 104 | } 105 | if proxyMode { 106 | env.ActiveDirectoryEndpoint = fmt.Sprintf("http://%s:%d/", proxyAddress, proxyPort) 107 | } 108 | 109 | vaultResourceURL := getVaultResourceIdentifier(managedHSM, env) 110 | if vaultResourceURL == azure.NotAvailable { 111 | return nil, fmt.Errorf("keyvault resource identifier not available for cloud: %s", env.Name) 112 | } 113 | token, err := auth.GetKeyvaultToken(config, env, vaultResourceURL, proxyMode) 114 | if err != nil { 115 | return nil, fmt.Errorf("failed to get key vault token, error: %+v", err) 116 | } 117 | kvClient.Authorizer = token 118 | 119 | vaultURL, err := getVaultURL(vaultName, managedHSM, env) 120 | if err != nil { 121 | return nil, fmt.Errorf("failed to get vault url, error: %+v", err) 122 | } 123 | 124 | keyIDHash, err := getKeyIDHash(*vaultURL, keyName, keyVersion) 125 | if err != nil { 126 | return nil, fmt.Errorf("failed to get key id hash, error: %w", err) 127 | } 128 | 129 | if proxyMode { 130 | kvClient.RequestInspector = autorest.WithHeader(consts.RequestHeaderTargetType, consts.TargetTypeKeyVault) 131 | vaultURL = getProxiedVaultURL(vaultURL, proxyAddress, proxyPort) 132 | } 133 | 134 | mlog.Always("using kms key for encrypt/decrypt", "vaultURL", *vaultURL, "keyName", keyName, "keyVersion", keyVersion) 135 | 136 | client := &KeyVaultClient{ 137 | baseClient: kvClient, 138 | config: config, 139 | vaultName: vaultName, 140 | keyName: keyName, 141 | keyVersion: keyVersion, 142 | vaultURL: *vaultURL, 143 | azureEnvironment: env, 144 | keyIDHash: keyIDHash, 145 | } 146 | return client, nil 147 | } 148 | 149 | // Encrypt encrypts the given plain text using the keyvault key. 150 | func (kvc *KeyVaultClient) Encrypt( 151 | ctx context.Context, 152 | plain []byte, 153 | encryptionAlgorithm kv.JSONWebKeyEncryptionAlgorithm, 154 | ) (*service.EncryptResponse, error) { 155 | value := base64.RawURLEncoding.EncodeToString(plain) 156 | 157 | params := kv.KeyOperationsParameters{ 158 | Algorithm: encryptionAlgorithm, 159 | Value: &value, 160 | } 161 | result, err := kvc.baseClient.Encrypt(ctx, kvc.vaultURL, kvc.keyName, kvc.keyVersion, params) 162 | if err != nil { 163 | return nil, fmt.Errorf("failed to encrypt, error: %+v", err) 164 | } 165 | 166 | if kvc.keyIDHash != fmt.Sprintf("%x", sha256.Sum256([]byte(*result.Kid))) { 167 | return nil, fmt.Errorf( 168 | "key id initialized does not match with the key id from encryption result, expected: %s, got: %s", 169 | kvc.keyIDHash, 170 | *result.Kid, 171 | ) 172 | } 173 | 174 | annotations := map[string][]byte{ 175 | dateAnnotationKey: []byte(result.Header.Get(dateAnnotationValue)), 176 | requestIDAnnotationKey: []byte(result.Header.Get(requestIDAnnotationValue)), 177 | keyvaultRegionAnnotationKey: []byte(result.Header.Get(keyvaultRegionAnnotationValue)), 178 | versionAnnotationKey: []byte(encryptionResponseVersion), 179 | algorithmAnnotationKey: []byte(encryptionAlgorithm), 180 | } 181 | 182 | return &service.EncryptResponse{ 183 | Ciphertext: []byte(*result.Result), 184 | KeyID: kvc.keyIDHash, 185 | Annotations: annotations, 186 | }, nil 187 | } 188 | 189 | // Decrypt decrypts the given cipher text using the keyvault key. 190 | func (kvc *KeyVaultClient) Decrypt( 191 | ctx context.Context, 192 | cipher []byte, 193 | encryptionAlgorithm kv.JSONWebKeyEncryptionAlgorithm, 194 | apiVersion string, 195 | annotations map[string][]byte, 196 | decryptRequestKeyID string, 197 | ) ([]byte, error) { 198 | if apiVersion == version.KMSv2APIVersion { 199 | err := kvc.validateAnnotations(annotations, decryptRequestKeyID, encryptionAlgorithm) 200 | if err != nil { 201 | return nil, err 202 | } 203 | } 204 | 205 | value := string(cipher) 206 | params := kv.KeyOperationsParameters{ 207 | Algorithm: encryptionAlgorithm, 208 | Value: &value, 209 | } 210 | 211 | result, err := kvc.baseClient.Decrypt(ctx, kvc.vaultURL, kvc.keyName, kvc.keyVersion, params) 212 | if err != nil { 213 | return nil, fmt.Errorf("failed to decrypt, error: %+v", err) 214 | } 215 | bytes, err := base64.RawURLEncoding.DecodeString(*result.Result) 216 | if err != nil { 217 | return nil, fmt.Errorf("failed to base64 decode result, error: %+v", err) 218 | } 219 | 220 | return bytes, nil 221 | } 222 | 223 | func (kvc *KeyVaultClient) GetUserAgent() string { 224 | return kvc.baseClient.UserAgent 225 | } 226 | 227 | func (kvc *KeyVaultClient) GetVaultURL() string { 228 | return kvc.vaultURL 229 | } 230 | 231 | // ValidateAnnotations validates following annotations before decryption: 232 | // - Algorithm. 233 | // - Version. 234 | // It also validates keyID that the API server checks. 235 | func (kvc *KeyVaultClient) validateAnnotations( 236 | annotations map[string][]byte, 237 | keyID string, 238 | encryptionAlgorithm kv.JSONWebKeyEncryptionAlgorithm, 239 | ) error { 240 | if len(annotations) == 0 { 241 | return fmt.Errorf("invalid annotations, annotations cannot be empty") 242 | } 243 | 244 | if keyID != kvc.keyIDHash { 245 | return fmt.Errorf( 246 | "key id %s does not match expected key id %s used for encryption", 247 | keyID, 248 | kvc.keyIDHash, 249 | ) 250 | } 251 | 252 | algorithm := string(annotations[algorithmAnnotationKey]) 253 | if algorithm != string(encryptionAlgorithm) { 254 | return fmt.Errorf( 255 | "algorithm %s does not match expected algorithm %s used for encryption", 256 | algorithm, 257 | encryptionAlgorithm, 258 | ) 259 | } 260 | 261 | version := string(annotations[versionAnnotationKey]) 262 | if version != encryptionResponseVersion { 263 | return fmt.Errorf( 264 | "version %s does not match expected version %s used for encryption", 265 | version, 266 | encryptionResponseVersion, 267 | ) 268 | } 269 | 270 | return nil 271 | } 272 | 273 | func getVaultURL(vaultName string, managedHSM bool, env *azure.Environment) (vaultURL *string, err error) { 274 | // Key Vault name must be a 3-24 character string 275 | if len(vaultName) < 3 || len(vaultName) > 24 { 276 | return nil, fmt.Errorf("invalid vault name: %q, must be between 3 and 24 chars", vaultName) 277 | } 278 | 279 | // See docs for validation spec: https://docs.microsoft.com/en-us/azure/key-vault/about-keys-secrets-and-certificates#objects-identifiers-and-versioning 280 | isValid := regexp.MustCompile(`^[-A-Za-z0-9]+$`).MatchString 281 | if !isValid(vaultName) { 282 | return nil, fmt.Errorf("invalid vault name: %q, must match [-a-zA-Z0-9]{3,24}", vaultName) 283 | } 284 | 285 | vaultDNSSuffixValue := getVaultDNSSuffix(managedHSM, env) 286 | if vaultDNSSuffixValue == azure.NotAvailable { 287 | return nil, fmt.Errorf("vault dns suffix not available for cloud: %s", env.Name) 288 | } 289 | 290 | vaultURI := fmt.Sprintf("https://%s.%s/", vaultName, vaultDNSSuffixValue) 291 | return &vaultURI, nil 292 | } 293 | 294 | func getProxiedVaultURL(vaultURL *string, proxyAddress string, proxyPort int) *string { 295 | proxiedVaultURL := fmt.Sprintf("http://%s:%d/%s", proxyAddress, proxyPort, strings.TrimPrefix(*vaultURL, "https://")) 296 | return &proxiedVaultURL 297 | } 298 | 299 | func getVaultDNSSuffix(managedHSM bool, env *azure.Environment) string { 300 | if managedHSM { 301 | return env.ManagedHSMDNSSuffix 302 | } 303 | return env.KeyVaultDNSSuffix 304 | } 305 | 306 | func getVaultResourceIdentifier(managedHSM bool, env *azure.Environment) string { 307 | if managedHSM { 308 | return env.ResourceIdentifiers.ManagedHSM 309 | } 310 | return env.ResourceIdentifiers.KeyVault 311 | } 312 | 313 | func getKeyIDHash(vaultURL, keyName, keyVersion string) (string, error) { 314 | if vaultURL == "" || keyName == "" || keyVersion == "" { 315 | return "", fmt.Errorf("vault url, key name and key version cannot be empty") 316 | } 317 | 318 | baseURL, err := url.Parse(vaultURL) 319 | if err != nil { 320 | return "", fmt.Errorf("failed to parse vault url, error: %w", err) 321 | } 322 | 323 | urlPath := path.Join("keys", keyName, keyVersion) 324 | keyID := baseURL.ResolveReference( 325 | &url.URL{ 326 | Path: urlPath, 327 | }, 328 | ).String() 329 | 330 | return fmt.Sprintf("%x", sha256.Sum256([]byte(keyID))), nil 331 | } 332 | -------------------------------------------------------------------------------- /pkg/plugin/keyvault_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "fmt" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/Azure/kubernetes-kms/pkg/auth" 14 | "github.com/Azure/kubernetes-kms/pkg/config" 15 | ) 16 | 17 | var ( 18 | testEnvs = []string{"", "AZUREPUBLICCLOUD", "AZURECHINACLOUD", "AZUREGERMANCLOUD", "AZUREUSGOVERNMENTCLOUD"} 19 | vaultDNSSuffix = []string{"vault.azure.net", "vault.azure.net", "vault.azure.cn", "vault.microsoftazure.de", "vault.usgovcloudapi.net"} 20 | ) 21 | 22 | func TestNewKeyVaultClientError(t *testing.T) { 23 | tests := []struct { 24 | desc string 25 | config *config.AzureConfig 26 | vaultName string 27 | keyName string 28 | keyVersion string 29 | proxyMode bool 30 | proxyAddress string 31 | proxyPort int 32 | managedHSM bool 33 | }{ 34 | { 35 | desc: "vault name not provided", 36 | config: &config.AzureConfig{}, 37 | proxyMode: false, 38 | }, 39 | { 40 | desc: "key name not provided", 41 | config: &config.AzureConfig{}, 42 | vaultName: "testkv", 43 | proxyMode: false, 44 | }, 45 | { 46 | desc: "key version not provided", 47 | config: &config.AzureConfig{}, 48 | vaultName: "testkv", 49 | keyName: "k8s", 50 | proxyMode: false, 51 | }, 52 | { 53 | desc: "no credentials in config", 54 | config: &config.AzureConfig{}, 55 | vaultName: "testkv", 56 | keyName: "key1", 57 | keyVersion: "262067a9e8ba401aa8a746c5f1a7e147", 58 | }, 59 | { 60 | desc: "managed hsm not available in the azure environment", 61 | config: &config.AzureConfig{ClientID: "clientid", ClientSecret: "clientsecret", Cloud: "AzureGermanCloud"}, 62 | vaultName: "testkv", 63 | keyName: "key1", 64 | keyVersion: "262067a9e8ba401aa8a746c5f1a7e147", 65 | managedHSM: true, 66 | }, 67 | } 68 | 69 | for _, test := range tests { 70 | t.Run(test.desc, func(t *testing.T) { 71 | if _, err := NewKeyVaultClient(test.config, test.vaultName, test.keyName, test.keyVersion, test.proxyMode, test.proxyAddress, test.proxyPort, test.managedHSM); err == nil { 72 | t.Fatalf("newKeyVaultClient() expected error, got nil") 73 | } 74 | }) 75 | } 76 | } 77 | 78 | func TestNewKeyVaultClient(t *testing.T) { 79 | tests := []struct { 80 | desc string 81 | config *config.AzureConfig 82 | vaultName string 83 | keyName string 84 | keyVersion string 85 | proxyMode bool 86 | proxyAddress string 87 | proxyPort int 88 | managedHSM bool 89 | expectedVaultURL string 90 | }{ 91 | { 92 | desc: "no error", 93 | config: &config.AzureConfig{ClientID: "clientid", ClientSecret: "clientsecret"}, 94 | vaultName: "testkv", 95 | keyName: "key1", 96 | keyVersion: "262067a9e8ba401aa8a746c5f1a7e147", 97 | proxyMode: false, 98 | expectedVaultURL: "https://testkv.vault.azure.net/", 99 | }, 100 | { 101 | desc: "no error with double quotes", 102 | config: &config.AzureConfig{ClientID: "clientid", ClientSecret: "clientsecret"}, 103 | vaultName: "\"testkv\"", 104 | keyName: "\"key1\"", 105 | keyVersion: "\"262067a9e8ba401aa8a746c5f1a7e147\"", 106 | proxyMode: false, 107 | expectedVaultURL: "https://testkv.vault.azure.net/", 108 | }, 109 | { 110 | desc: "no error with proxy mode", 111 | config: &config.AzureConfig{ClientID: "clientid", ClientSecret: "clientsecret"}, 112 | vaultName: "testkv", 113 | keyName: "key1", 114 | keyVersion: "262067a9e8ba401aa8a746c5f1a7e147", 115 | proxyMode: true, 116 | proxyAddress: "localhost", 117 | proxyPort: 7788, 118 | expectedVaultURL: "http://localhost:7788/testkv.vault.azure.net/", 119 | }, 120 | { 121 | desc: "no error with managed hsm", 122 | config: &config.AzureConfig{ClientID: "clientid", ClientSecret: "clientsecret"}, 123 | vaultName: "testkv", 124 | keyName: "key1", 125 | keyVersion: "262067a9e8ba401aa8a746c5f1a7e147", 126 | managedHSM: true, 127 | proxyMode: false, 128 | expectedVaultURL: "https://testkv.managedhsm.azure.net/", 129 | }, 130 | } 131 | 132 | for _, test := range tests { 133 | t.Run(test.desc, func(t *testing.T) { 134 | kvClient, err := NewKeyVaultClient(test.config, test.vaultName, test.keyName, test.keyVersion, test.proxyMode, test.proxyAddress, test.proxyPort, test.managedHSM) 135 | if err != nil { 136 | t.Fatalf("newKeyVaultClient() failed with error: %v", err) 137 | } 138 | if kvClient == nil { 139 | t.Fatalf("newKeyVaultClient() expected kv client to not be nil") 140 | } 141 | if !strings.Contains(kvClient.GetUserAgent(), "k8s-kms-keyvault") { 142 | t.Fatalf("newKeyVaultClient() expected k8s-kms-keyvault user agent") 143 | } 144 | if kvClient.GetVaultURL() != test.expectedVaultURL { 145 | t.Fatalf("expected vault URL: %v, got vault URL: %v", test.expectedVaultURL, kvClient.GetVaultURL()) 146 | } 147 | }) 148 | } 149 | } 150 | 151 | func TestGetVaultURLError(t *testing.T) { 152 | tests := []struct { 153 | desc string 154 | vaultName string 155 | managedHSM bool 156 | }{ 157 | { 158 | desc: "vault name > 24", 159 | vaultName: "longkeyvaultnamewhichisnotvalid", 160 | }, 161 | { 162 | desc: "vault name < 3", 163 | vaultName: "kv", 164 | }, 165 | { 166 | desc: "vault name contains non alpha-numeric chars", 167 | vaultName: "kv_test", 168 | }, 169 | } 170 | 171 | for _, test := range tests { 172 | for idx := range testEnvs { 173 | t.Run(fmt.Sprintf("%s/%s", test.desc, testEnvs[idx]), func(t *testing.T) { 174 | azEnv, err := auth.ParseAzureEnvironment(testEnvs[idx]) 175 | if err != nil { 176 | t.Fatalf("failed to parse azure environment from name, err: %+v", err) 177 | } 178 | if _, err = getVaultURL(test.vaultName, test.managedHSM, azEnv); err == nil { 179 | t.Fatalf("getVaultURL() expected error, got nil") 180 | } 181 | }) 182 | } 183 | } 184 | } 185 | 186 | func TestGetVaultURL(t *testing.T) { 187 | vaultName := "testkv" 188 | 189 | for idx := range testEnvs { 190 | t.Run(testEnvs[idx], func(t *testing.T) { 191 | azEnv, err := auth.ParseAzureEnvironment(testEnvs[idx]) 192 | if err != nil { 193 | t.Fatalf("failed to parse azure environment from name, err: %+v", err) 194 | } 195 | vaultURL, err := getVaultURL(vaultName, false, azEnv) 196 | if err != nil { 197 | t.Fatalf("expected no error of getting vault URL, got error: %v", err) 198 | } 199 | expectedURL := "https://" + vaultName + "." + vaultDNSSuffix[idx] + "/" 200 | if expectedURL != *vaultURL { 201 | t.Fatalf("expected vault url: %s, got: %s", expectedURL, *vaultURL) 202 | } 203 | }) 204 | } 205 | } 206 | 207 | func TestGetKeyIDHash(t *testing.T) { 208 | testCases := []struct { 209 | name string 210 | vaultURL string 211 | keyName string 212 | keyVersion string 213 | expectedHash string 214 | expectedError bool 215 | expectedErrorString string 216 | }{ 217 | { 218 | name: "valid hash", 219 | vaultURL: "https://example.vault.azure.net/", 220 | keyName: "mykey", 221 | keyVersion: "ABCD", 222 | expectedHash: "567d783db3043fe298fe0d9eeedb0029a3815cdd4fe4b059d018c91e6acffe3b", 223 | expectedError: false, 224 | }, 225 | { 226 | name: "invalid vault URL", 227 | vaultURL: ":invalid-url:", 228 | keyName: "mykey", 229 | keyVersion: "ABCD", 230 | expectedHash: "", 231 | expectedError: true, 232 | expectedErrorString: "failed to parse vault url, error: parse \":invalid-url:\": missing protocol scheme", 233 | }, 234 | { 235 | name: "empty vault name", 236 | vaultURL: "", 237 | keyName: "mykey", 238 | keyVersion: "ABCD", 239 | expectedHash: "", 240 | expectedError: true, 241 | expectedErrorString: "vault url, key name and key version cannot be empty", 242 | }, 243 | { 244 | name: "empty key name", 245 | vaultURL: "https://example.vault.azure.net/", 246 | keyName: "", 247 | keyVersion: "ABCD", 248 | expectedHash: "", 249 | expectedError: true, 250 | expectedErrorString: "vault url, key name and key version cannot be empty", 251 | }, 252 | { 253 | name: "empty key vesion", 254 | vaultURL: "https://example.vault.azure.net/", 255 | keyName: "mykey", 256 | keyVersion: "", 257 | expectedHash: "", 258 | expectedError: true, 259 | expectedErrorString: "vault url, key name and key version cannot be empty", 260 | }, 261 | } 262 | 263 | for _, tc := range testCases { 264 | t.Run(tc.name, func(t *testing.T) { 265 | hash, err := getKeyIDHash(tc.vaultURL, tc.keyName, tc.keyVersion) 266 | 267 | if tc.expectedError { 268 | if (err != nil) && (err.Error() != tc.expectedErrorString) { 269 | t.Errorf("Expected error: %v, but got: %v", tc.expectedErrorString, err.Error()) 270 | } else if err == nil { 271 | t.Errorf("Expected error: %v, but didn't get any", tc.expectedErrorString) 272 | } 273 | } 274 | 275 | if hash != tc.expectedHash { 276 | t.Errorf("Expected hash: %s, but got: %s", tc.expectedHash, hash) 277 | } 278 | }) 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /pkg/plugin/kms_v2_server.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "time" 12 | 13 | "github.com/Azure/kubernetes-kms/pkg/metrics" 14 | "github.com/Azure/kubernetes-kms/pkg/version" 15 | 16 | "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault" 17 | kmsv2 "k8s.io/kms/apis/v2" 18 | "monis.app/mlog" 19 | ) 20 | 21 | // KeyManagementServiceV2Server is a gRPC server. 22 | type KeyManagementServiceV2Server struct { 23 | kvClient Client 24 | reporter metrics.StatsReporter 25 | encryptionAlgorithm keyvault.JSONWebKeyEncryptionAlgorithm 26 | } 27 | 28 | // NewKMSv2Server creates an instance of the KMS Service Server with v2 apis. 29 | func NewKMSv2Server(kvClient Client) (*KeyManagementServiceV2Server, error) { 30 | statsReporter, err := metrics.NewStatsReporter() 31 | if err != nil { 32 | return nil, fmt.Errorf("failed to create stats reporter: %w", err) 33 | } 34 | 35 | return &KeyManagementServiceV2Server{ 36 | kvClient: kvClient, 37 | reporter: statsReporter, 38 | encryptionAlgorithm: keyvault.RSAOAEP256, 39 | }, nil 40 | } 41 | 42 | // Status returns the health status of the KMS plugin. 43 | func (s *KeyManagementServiceV2Server) Status(ctx context.Context, _ *kmsv2.StatusRequest) (*kmsv2.StatusResponse, error) { 44 | // We perform a simple encrypt/decrypt operation to verify the plugin's connectivity with Key Vault. 45 | // The KMS invokes the Status API every minute, resulting in 120 calls per hour to the Key Vault. 46 | // This volume of calls is well within the permissible limit of Key Vault. 47 | encryptResponse, err := s.kvClient.Encrypt(ctx, []byte(healthCheckPlainText), s.encryptionAlgorithm) 48 | if err != nil { 49 | mlog.Error("failed to encrypt healthcheck call", err) 50 | return nil, err 51 | } 52 | 53 | decryptedText, err := s.kvClient.Decrypt( 54 | ctx, 55 | encryptResponse.Ciphertext, 56 | s.encryptionAlgorithm, 57 | version.KMSv2APIVersion, 58 | encryptResponse.Annotations, 59 | encryptResponse.KeyID, 60 | ) 61 | if err != nil { 62 | mlog.Error("failed to decrypt healthcheck call", err) 63 | return nil, err 64 | } 65 | 66 | if string(decryptedText) != healthCheckPlainText { 67 | err = fmt.Errorf("decrypted text does not match") 68 | mlog.Error("healthcheck failed", err) 69 | return nil, err 70 | } 71 | 72 | return &kmsv2.StatusResponse{ 73 | Version: version.KMSv2APIVersion, 74 | Healthz: "ok", 75 | KeyId: encryptResponse.KeyID, 76 | }, nil 77 | } 78 | 79 | // Encrypt message. 80 | func (s *KeyManagementServiceV2Server) Encrypt(ctx context.Context, request *kmsv2.EncryptRequest) (*kmsv2.EncryptResponse, error) { 81 | mlog.Debug("encrypt request received", "uid", request.Uid) 82 | start := time.Now() 83 | 84 | var err error 85 | defer func() { 86 | errors := "" 87 | status := metrics.SuccessStatusTypeValue 88 | if err != nil { 89 | status = metrics.ErrorStatusTypeValue 90 | errors = err.Error() 91 | } 92 | s.reporter.ReportRequest(ctx, metrics.EncryptOperationTypeValue, status, time.Since(start).Seconds(), errors) 93 | }() 94 | 95 | mlog.Info("encrypt request started", "uid", request.Uid) 96 | encryptResponse, err := s.kvClient.Encrypt(ctx, request.Plaintext, s.encryptionAlgorithm) 97 | if err != nil { 98 | mlog.Error("failed to encrypt", err, "uid", request.Uid) 99 | return &kmsv2.EncryptResponse{}, err 100 | } 101 | mlog.Info("encrypt request complete", "uid", request.Uid) 102 | 103 | return &kmsv2.EncryptResponse{ 104 | Ciphertext: encryptResponse.Ciphertext, 105 | KeyId: encryptResponse.KeyID, 106 | Annotations: encryptResponse.Annotations, 107 | }, nil 108 | } 109 | 110 | // Decrypt message. 111 | func (s *KeyManagementServiceV2Server) Decrypt(ctx context.Context, request *kmsv2.DecryptRequest) (*kmsv2.DecryptResponse, error) { 112 | mlog.Debug("decrypt request received", "uid", request.Uid) 113 | start := time.Now() 114 | 115 | var err error 116 | defer func() { 117 | errors := "" 118 | status := metrics.SuccessStatusTypeValue 119 | if err != nil { 120 | status = metrics.ErrorStatusTypeValue 121 | errors = err.Error() 122 | } 123 | s.reporter.ReportRequest(ctx, metrics.DecryptOperationTypeValue, status, time.Since(start).Seconds(), errors) 124 | }() 125 | 126 | mlog.Info("decrypt request started", "uid", request.Uid) 127 | 128 | plainText, err := s.kvClient.Decrypt( 129 | ctx, 130 | request.Ciphertext, 131 | s.encryptionAlgorithm, 132 | version.KMSv2APIVersion, 133 | request.Annotations, 134 | request.KeyId, 135 | ) 136 | if err != nil { 137 | mlog.Error("failed to decrypt", err, "uid", request.Uid) 138 | return &kmsv2.DecryptResponse{}, err 139 | } 140 | mlog.Info("decrypt request complete", "uid", request.Uid) 141 | 142 | return &kmsv2.DecryptResponse{ 143 | Plaintext: plainText, 144 | }, nil 145 | } 146 | -------------------------------------------------------------------------------- /pkg/plugin/kms_v2_server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "fmt" 12 | "testing" 13 | 14 | "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault" 15 | "github.com/Azure/kubernetes-kms/pkg/metrics" 16 | mockkeyvault "github.com/Azure/kubernetes-kms/pkg/plugin/mock_keyvault" 17 | 18 | "github.com/Azure/kubernetes-kms/pkg/version" 19 | kmsv2 "k8s.io/kms/apis/v2" 20 | ) 21 | 22 | func TestV2Encrypt(t *testing.T) { 23 | tests := []struct { 24 | desc string 25 | input []byte 26 | output []byte 27 | err error 28 | }{ 29 | { 30 | desc: "failed to encrypt", 31 | input: []byte("foo"), 32 | output: []byte{}, 33 | err: fmt.Errorf("failed to encrypt"), 34 | }, 35 | { 36 | desc: "successfully encrypted", 37 | input: []byte("foo"), 38 | output: []byte("bar"), 39 | err: nil, 40 | }, 41 | } 42 | 43 | for _, test := range tests { 44 | t.Run(test.desc, func(t *testing.T) { 45 | kvClient := &mockkeyvault.KeyVaultClient{ 46 | KeyID: "mock-key-id", 47 | Algorithm: keyvault.RSA15, 48 | } 49 | kvClient.SetEncryptResponse(test.output, test.err) 50 | 51 | statsReporter, err := metrics.NewStatsReporter() 52 | if err != nil { 53 | t.Fatalf("failed to create stats reporter: %v", err) 54 | } 55 | 56 | kmsV2Server := KeyManagementServiceV2Server{ 57 | kvClient: kvClient, 58 | reporter: statsReporter, 59 | } 60 | 61 | out, err := kmsV2Server.Encrypt(context.TODO(), &kmsv2.EncryptRequest{ 62 | Plaintext: test.input, 63 | }) 64 | if err != test.err { 65 | t.Fatalf("expected err: %v, got: %v", test.err, err) 66 | } 67 | if !bytes.Equal(out.GetCiphertext(), test.output) { 68 | t.Fatalf("expected out: %v, got: %v", test.output, out) 69 | } 70 | if err == nil && (out.KeyId != kvClient.KeyID) { 71 | t.Fatalf("expected key id: %v, got: %v", kvClient.KeyID, out.KeyId) 72 | } 73 | if err == nil && (len(out.Annotations) == 0) { 74 | t.Fatalf("invalid annotations, annotations cannot be empty") 75 | } 76 | }) 77 | } 78 | } 79 | 80 | func TestV2Decrypt(t *testing.T) { 81 | tests := []struct { 82 | desc string 83 | input []byte 84 | output []byte 85 | err error 86 | annotations map[string][]byte 87 | }{ 88 | { 89 | desc: "empty annotations failed to decrypt", 90 | input: []byte("bar"), 91 | output: []byte{}, 92 | err: fmt.Errorf("invalid annotations, annotations cannot be empty"), 93 | }, 94 | { 95 | desc: "invalid keyid failed to decrypt", 96 | input: []byte("bar"), 97 | output: []byte{}, 98 | err: fmt.Errorf("key id \"invalid-key-id\" does not match expected key id \"mock-key-id\" used for encryption"), 99 | annotations: map[string][]byte{ 100 | algorithmAnnotationKey: []byte(keyvault.RSA15), 101 | versionAnnotationKey: []byte("1"), 102 | }, 103 | }, 104 | { 105 | desc: "invalid algorithm failed to decrypt", 106 | input: []byte("bar"), 107 | output: []byte{}, 108 | err: fmt.Errorf("algorithm \"insecure-algorithm\" does not match expected algorithm \"RSAOAEP256\" used for encryption"), 109 | annotations: map[string][]byte{ 110 | algorithmAnnotationKey: []byte("insecure-algorithm"), 111 | versionAnnotationKey: []byte("1"), 112 | }, 113 | }, 114 | { 115 | desc: "invalid version failed to decrypt", 116 | input: []byte("bar"), 117 | output: []byte{}, 118 | err: fmt.Errorf("version \"10\" does not match expected version \"1\" used for encryption"), 119 | annotations: map[string][]byte{ 120 | algorithmAnnotationKey: []byte(keyvault.RSA15), 121 | versionAnnotationKey: []byte("10"), 122 | }, 123 | }, 124 | { 125 | desc: "failed to decrypt", 126 | input: []byte("foo"), 127 | output: []byte{}, 128 | err: fmt.Errorf("failed to decrypt"), 129 | annotations: map[string][]byte{ 130 | algorithmAnnotationKey: []byte(keyvault.RSA15), 131 | versionAnnotationKey: []byte("1"), 132 | }, 133 | }, 134 | { 135 | desc: "successfully decrypted", 136 | input: []byte("bar"), 137 | output: []byte("foo"), 138 | err: nil, 139 | annotations: map[string][]byte{ 140 | algorithmAnnotationKey: []byte(keyvault.RSA15), 141 | versionAnnotationKey: []byte("1"), 142 | }, 143 | }, 144 | } 145 | 146 | for _, test := range tests { 147 | t.Run(test.desc, func(t *testing.T) { 148 | kvClient := &mockkeyvault.KeyVaultClient{ 149 | KeyID: "mock-key-id", 150 | Algorithm: keyvault.RSAOAEP256, 151 | } 152 | kvClient.SetDecryptResponse(test.output, test.err) 153 | 154 | statsReporter, err := metrics.NewStatsReporter() 155 | if err != nil { 156 | t.Fatalf("failed to create stats reporter: %v", err) 157 | } 158 | 159 | kmsV2Server := KeyManagementServiceV2Server{ 160 | kvClient: kvClient, 161 | reporter: statsReporter, 162 | } 163 | 164 | out, err := kmsV2Server.Decrypt(context.TODO(), &kmsv2.DecryptRequest{ 165 | Ciphertext: test.input, 166 | Annotations: test.annotations, 167 | KeyId: "mock-key-id", 168 | }) 169 | if err != nil && (err.Error() != test.err.Error()) { 170 | t.Fatalf("expected err: %v, got: %v", test.err, err) 171 | } 172 | if !bytes.Equal(out.GetPlaintext(), test.output) { 173 | t.Fatalf("expected out: %v, got: %v", test.output, out) 174 | } 175 | }) 176 | } 177 | } 178 | 179 | func TestStatus(t *testing.T) { 180 | kmsServer := KeyManagementServiceV2Server{} 181 | mockKeyVaultClient := &mockkeyvault.KeyVaultClient{ 182 | KeyID: "mock-key-id", 183 | } 184 | mockKeyVaultClient.SetEncryptResponse([]byte(healthCheckPlainText), nil) 185 | mockKeyVaultClient.SetDecryptResponse([]byte(healthCheckPlainText), nil) 186 | kmsServer.kvClient = mockKeyVaultClient 187 | 188 | v, err := kmsServer.Status(context.TODO(), &kmsv2.StatusRequest{}) 189 | if err != nil { 190 | t.Fatalf("expected err to be nil, got: %v", err) 191 | } 192 | 193 | if v.Version != version.KMSv2APIVersion { 194 | t.Fatalf("expected version: %s, got: %s", version.KMSv2APIVersion, v.Version) 195 | } 196 | 197 | if v.Healthz != "ok" { 198 | t.Fatalf("expected healthz response to be: %s, got: %s", "ok", v.Healthz) 199 | } 200 | 201 | if v.KeyId != "mock-key-id" { 202 | t.Fatalf("expected key id: %s, got: %s", "mock-key-id", v.KeyId) 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /pkg/plugin/mock_keyvault/keyvault_mock.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package mockkeyvault 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "sync" 12 | 13 | "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault" 14 | "k8s.io/kms/pkg/service" 15 | ) 16 | 17 | type KeyVaultClient struct { 18 | mutex sync.Mutex 19 | 20 | encryptOut []byte 21 | encryptErr error 22 | decryptOut []byte 23 | decryptErr error 24 | KeyID string 25 | Algorithm keyvault.JSONWebKeyEncryptionAlgorithm 26 | } 27 | 28 | func (kvc *KeyVaultClient) Encrypt(_ context.Context, _ []byte, _ keyvault.JSONWebKeyEncryptionAlgorithm) (*service.EncryptResponse, error) { 29 | kvc.mutex.Lock() 30 | defer kvc.mutex.Unlock() 31 | return &service.EncryptResponse{ 32 | Ciphertext: kvc.encryptOut, 33 | KeyID: kvc.KeyID, 34 | Annotations: map[string][]byte{ 35 | "key-id.azure.akv.io": []byte(kvc.KeyID), 36 | "algorithm.azure.akv.io": []byte(kvc.Algorithm), 37 | "version.azure.akv.io": []byte("1"), 38 | }, 39 | }, kvc.encryptErr 40 | } 41 | 42 | func (kvc *KeyVaultClient) Decrypt(_ context.Context, _ []byte, _ keyvault.JSONWebKeyEncryptionAlgorithm, _ string, _ map[string][]byte, _ string) ([]byte, error) { 43 | kvc.mutex.Lock() 44 | defer kvc.mutex.Unlock() 45 | return kvc.decryptOut, kvc.decryptErr 46 | } 47 | 48 | func (kvc *KeyVaultClient) SetEncryptResponse(encryptOut []byte, err error) { 49 | kvc.mutex.Lock() 50 | defer kvc.mutex.Unlock() 51 | kvc.encryptOut = encryptOut 52 | kvc.encryptErr = err 53 | } 54 | 55 | func (kvc *KeyVaultClient) SetDecryptResponse(decryptOut []byte, err error) { 56 | kvc.mutex.Lock() 57 | defer kvc.mutex.Unlock() 58 | kvc.decryptOut = decryptOut 59 | kvc.decryptErr = err 60 | } 61 | 62 | func (kvc *KeyVaultClient) ValidateAnnotations(annotations map[string][]byte, keyID string) error { 63 | if len(annotations) == 0 { 64 | return fmt.Errorf("invalid annotations, annotations cannot be empty") 65 | } 66 | 67 | // validate key id 68 | if keyID != kvc.KeyID { 69 | return fmt.Errorf( 70 | "key id %q does not match expected key id %q used for encryption", 71 | string(annotations["key-id.azure.akv.io"]), 72 | kvc.KeyID, 73 | ) 74 | } 75 | 76 | // validate algorithm 77 | if string(annotations["algorithm.azure.akv.io"]) != string(kvc.Algorithm) { 78 | return fmt.Errorf("algorithm %q does not match expected algorithm %q used for encryption", string(annotations["algorithm.azure.akv.io"]), kvc.Algorithm) 79 | } 80 | 81 | // validate version 82 | if string(annotations["version.azure.akv.io"]) != "1" { 83 | return fmt.Errorf( 84 | "version %q does not match expected version %q used for encryption", 85 | string(annotations["version.azure.akv.io"]), 86 | "1", 87 | ) 88 | } 89 | 90 | return nil 91 | } 92 | 93 | func (kvc *KeyVaultClient) GetUserAgent() string { 94 | return "k8s-kms-keyvault" 95 | } 96 | 97 | func (kvc *KeyVaultClient) GetVaultURL() string { 98 | return "https://test.vault.azure.net" 99 | } 100 | -------------------------------------------------------------------------------- /pkg/plugin/server.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "time" 12 | 13 | "github.com/Azure/kubernetes-kms/pkg/metrics" 14 | "github.com/Azure/kubernetes-kms/pkg/version" 15 | 16 | "github.com/Azure/azure-sdk-for-go/services/keyvault/2016-10-01/keyvault" 17 | kmsv1 "k8s.io/kms/apis/v1beta1" 18 | "monis.app/mlog" 19 | ) 20 | 21 | // KeyManagementServiceServer is a gRPC server. 22 | type KeyManagementServiceServer struct { 23 | kvClient Client 24 | reporter metrics.StatsReporter 25 | encryptionAlgorithm keyvault.JSONWebKeyEncryptionAlgorithm 26 | } 27 | 28 | // Config is the configuration for the KMS plugin. 29 | type Config struct { 30 | ConfigFilePath string 31 | KeyVaultName string 32 | KeyName string 33 | KeyVersion string 34 | ManagedHSM bool 35 | ProxyMode bool 36 | ProxyAddress string 37 | ProxyPort int 38 | } 39 | 40 | // NewKMSv1Server creates an instance of the KMS Service Server. 41 | func NewKMSv1Server(kvClient Client) (*KeyManagementServiceServer, error) { 42 | statsReporter, err := metrics.NewStatsReporter() 43 | if err != nil { 44 | return nil, fmt.Errorf("failed to create stats reporter: %w", err) 45 | } 46 | 47 | return &KeyManagementServiceServer{ 48 | kvClient: kvClient, 49 | reporter: statsReporter, 50 | encryptionAlgorithm: keyvault.RSA15, 51 | }, nil 52 | } 53 | 54 | // Version of kms. 55 | func (s *KeyManagementServiceServer) Version(_ context.Context, _ *kmsv1.VersionRequest) (*kmsv1.VersionResponse, error) { 56 | return &kmsv1.VersionResponse{ 57 | Version: version.KMSv1APIVersion, 58 | RuntimeName: version.Runtime, 59 | RuntimeVersion: version.BuildVersion, 60 | }, nil 61 | } 62 | 63 | // Encrypt message. 64 | func (s *KeyManagementServiceServer) Encrypt(ctx context.Context, request *kmsv1.EncryptRequest) (*kmsv1.EncryptResponse, error) { 65 | start := time.Now() 66 | 67 | var err error 68 | defer func() { 69 | errors := "" 70 | status := metrics.SuccessStatusTypeValue 71 | if err != nil { 72 | status = metrics.ErrorStatusTypeValue 73 | errors = err.Error() 74 | } 75 | s.reporter.ReportRequest(ctx, metrics.EncryptOperationTypeValue, status, time.Since(start).Seconds(), errors) 76 | }() 77 | 78 | mlog.Info("encrypt request started") 79 | encryptResponse, err := s.kvClient.Encrypt(ctx, request.Plain, s.encryptionAlgorithm) 80 | if err != nil { 81 | mlog.Error("failed to encrypt", err) 82 | return &kmsv1.EncryptResponse{}, err 83 | } 84 | mlog.Info("encrypt request complete") 85 | return &kmsv1.EncryptResponse{ 86 | Cipher: encryptResponse.Ciphertext, 87 | }, nil 88 | } 89 | 90 | // Decrypt message. 91 | func (s *KeyManagementServiceServer) Decrypt(ctx context.Context, request *kmsv1.DecryptRequest) (*kmsv1.DecryptResponse, error) { 92 | start := time.Now() 93 | 94 | var err error 95 | defer func() { 96 | errors := "" 97 | status := metrics.SuccessStatusTypeValue 98 | if err != nil { 99 | status = metrics.ErrorStatusTypeValue 100 | errors = err.Error() 101 | } 102 | s.reporter.ReportRequest(ctx, metrics.DecryptOperationTypeValue, status, time.Since(start).Seconds(), errors) 103 | }() 104 | 105 | mlog.Info("decrypt request started") 106 | plain, err := s.kvClient.Decrypt( 107 | ctx, 108 | request.Cipher, 109 | s.encryptionAlgorithm, 110 | request.Version, 111 | nil, 112 | "", 113 | ) 114 | if err != nil { 115 | mlog.Error("failed to decrypt", err) 116 | return &kmsv1.DecryptResponse{}, err 117 | } 118 | mlog.Info("decrypt request complete") 119 | return &kmsv1.DecryptResponse{Plain: plain}, nil 120 | } 121 | -------------------------------------------------------------------------------- /pkg/plugin/server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft and contributors. All rights reserved. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | package plugin 7 | 8 | import ( 9 | "bytes" 10 | "context" 11 | "fmt" 12 | "testing" 13 | 14 | "github.com/Azure/kubernetes-kms/pkg/metrics" 15 | mockkeyvault "github.com/Azure/kubernetes-kms/pkg/plugin/mock_keyvault" 16 | "github.com/Azure/kubernetes-kms/pkg/version" 17 | 18 | kmsv1 "k8s.io/kms/apis/v1beta1" 19 | ) 20 | 21 | func TestEncrypt(t *testing.T) { 22 | tests := []struct { 23 | desc string 24 | input []byte 25 | output []byte 26 | err error 27 | }{ 28 | { 29 | desc: "failed to encrypt", 30 | input: []byte("foo"), 31 | output: []byte{}, 32 | err: fmt.Errorf("failed to encrypt"), 33 | }, 34 | { 35 | desc: "successfully encrypted", 36 | input: []byte("foo"), 37 | output: []byte("bar"), 38 | err: nil, 39 | }, 40 | } 41 | 42 | for _, test := range tests { 43 | t.Run(test.desc, func(t *testing.T) { 44 | kvClient := &mockkeyvault.KeyVaultClient{} 45 | kvClient.SetEncryptResponse(test.output, test.err) 46 | 47 | statsReporter, err := metrics.NewStatsReporter() 48 | if err != nil { 49 | t.Fatalf("failed to create stats reporter: %v", err) 50 | } 51 | 52 | kmsServer := KeyManagementServiceServer{ 53 | kvClient: kvClient, 54 | reporter: statsReporter, 55 | } 56 | 57 | out, err := kmsServer.Encrypt(context.TODO(), &kmsv1.EncryptRequest{ 58 | Plain: test.input, 59 | }) 60 | if err != test.err { 61 | t.Fatalf("expected err: %v, got: %v", test.err, err) 62 | } 63 | if !bytes.Equal(out.GetCipher(), test.output) { 64 | t.Fatalf("expected out: %v, got: %v", test.output, out) 65 | } 66 | }) 67 | } 68 | } 69 | 70 | func TestDecrypt(t *testing.T) { 71 | tests := []struct { 72 | desc string 73 | input []byte 74 | output []byte 75 | err error 76 | }{ 77 | { 78 | desc: "failed to decrypt", 79 | input: []byte("foo"), 80 | output: []byte{}, 81 | err: fmt.Errorf("failed to decrypt"), 82 | }, 83 | { 84 | desc: "successfully decrypted", 85 | input: []byte("bar"), 86 | output: []byte("foo"), 87 | err: nil, 88 | }, 89 | } 90 | 91 | for _, test := range tests { 92 | t.Run(test.desc, func(t *testing.T) { 93 | kvClient := &mockkeyvault.KeyVaultClient{} 94 | kvClient.SetDecryptResponse(test.output, test.err) 95 | 96 | statsReporter, err := metrics.NewStatsReporter() 97 | if err != nil { 98 | t.Fatalf("failed to create stats reporter: %v", err) 99 | } 100 | 101 | kmsServer := KeyManagementServiceServer{ 102 | kvClient: kvClient, 103 | reporter: statsReporter, 104 | } 105 | 106 | out, err := kmsServer.Decrypt(context.TODO(), &kmsv1.DecryptRequest{ 107 | Cipher: test.input, 108 | }) 109 | if err != test.err { 110 | t.Fatalf("expected err: %v, got: %v", test.err, err) 111 | } 112 | if !bytes.Equal(out.GetPlain(), test.output) { 113 | t.Fatalf("expected out: %v, got: %v", test.output, out) 114 | } 115 | }) 116 | } 117 | } 118 | 119 | func TestVersion(t *testing.T) { 120 | kmsServer := KeyManagementServiceServer{} 121 | 122 | version.BuildVersion = "latest" 123 | 124 | v, err := kmsServer.Version(context.TODO(), &kmsv1.VersionRequest{}) 125 | if err != nil { 126 | t.Fatalf("expected err to be nil, got: %v", err) 127 | } 128 | if v.Version != version.KMSv1APIVersion { 129 | t.Fatalf("expected version: %s, got: %s", version.KMSv1APIVersion, v.Version) 130 | } 131 | if v.RuntimeName != version.Runtime { 132 | t.Fatalf("expected runtime: %s, got: %s", version.Runtime, v.RuntimeName) 133 | } 134 | if v.RuntimeVersion != "latest" { 135 | t.Fatalf("expected runtime version: %s, got: %s", version.BuildVersion, v.Version) 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /pkg/utils/grpc.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "time" 8 | 9 | "github.com/Azure/kubernetes-kms/pkg/metrics" 10 | 11 | "google.golang.org/grpc" 12 | "monis.app/mlog" 13 | ) 14 | 15 | // ParseEndpoint returns unix socket's protocol and address. 16 | func ParseEndpoint(ep string) (string, string, error) { 17 | if strings.HasPrefix(strings.ToLower(ep), "unix://") { 18 | s := strings.SplitN(ep, "://", 2) 19 | if s[1] != "" { 20 | return s[0], s[1], nil 21 | } 22 | } 23 | return "", "", fmt.Errorf("invalid endpoint: %v", ep) 24 | } 25 | 26 | // UnaryServerInterceptor provides metrics around Unary RPCs. 27 | func UnaryServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { 28 | var err error 29 | start := time.Now() 30 | reporter, err := metrics.NewStatsReporter() 31 | if err != nil { 32 | return nil, fmt.Errorf("failed to create stats reporter: %w", err) 33 | } 34 | 35 | defer func() { 36 | errors := "" 37 | status := metrics.SuccessStatusTypeValue 38 | if err != nil { 39 | status = metrics.ErrorStatusTypeValue 40 | errors = err.Error() 41 | } 42 | reporter.ReportRequest(ctx, fmt.Sprintf("%s_%s", metrics.GrpcOperationTypeValue, getGRPCMethodName(info.FullMethod)), status, time.Since(start).Seconds(), errors) 43 | }() 44 | 45 | mlog.Trace("GRPC call", "method", info.FullMethod) 46 | resp, err := handler(ctx, req) 47 | if err != nil { 48 | mlog.Error("GRPC request error", err) 49 | } 50 | return resp, err 51 | } 52 | 53 | func getGRPCMethodName(fullMethodName string) string { 54 | fullMethodName = strings.TrimPrefix(fullMethodName, "/") 55 | methodNames := strings.Split(fullMethodName, "/") 56 | if len(methodNames) >= 2 { 57 | return strings.ToLower(methodNames[1]) 58 | } 59 | 60 | return "unknown" 61 | } 62 | -------------------------------------------------------------------------------- /pkg/utils/grpc_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestParseEndpoint(t *testing.T) { 6 | tests := []struct { 7 | desc string 8 | endpoint string 9 | expectedProto string 10 | expectedAddr string 11 | expectedErr bool 12 | }{ 13 | { 14 | desc: "invalid endpoint", 15 | endpoint: "udp:///provider/azure.sock", 16 | expectedErr: true, 17 | }, 18 | { 19 | desc: "invalid unix endpoint", 20 | endpoint: "unix://", 21 | expectedProto: "", 22 | expectedAddr: "", 23 | expectedErr: true, 24 | }, 25 | { 26 | desc: "valid unix endpoint", 27 | endpoint: "unix:///provider/azure.sock", 28 | expectedProto: "unix", 29 | expectedAddr: "/provider/azure.sock", 30 | expectedErr: false, 31 | }, 32 | } 33 | 34 | for _, test := range tests { 35 | t.Run(test.desc, func(t *testing.T) { 36 | proto, addr, err := ParseEndpoint(test.endpoint) 37 | if test.expectedErr && err == nil || !test.expectedErr && err != nil { 38 | t.Fatalf("expected error: %v, got error: %v", test.expectedErr, err) 39 | } 40 | if proto != test.expectedProto { 41 | t.Fatalf("expected proto: %v, got: %v", test.expectedProto, proto) 42 | } 43 | if addr != test.expectedAddr { 44 | t.Fatalf("expected addr: %v, got: %v", test.expectedAddr, addr) 45 | } 46 | }) 47 | } 48 | } 49 | 50 | func TestGetGRPCMethodName(t *testing.T) { 51 | testCases := []struct { 52 | name string 53 | input string 54 | expectedOutput string 55 | }{ 56 | { 57 | name: "With_Correct_Method_Name", 58 | input: "/v1beta1.KeyManagementService/Encrypt", 59 | expectedOutput: "encrypt", 60 | }, 61 | { 62 | name: "With_Incorrect_Method_Name", 63 | input: "/Encrypt", 64 | expectedOutput: "unknown", 65 | }, 66 | } 67 | 68 | for _, testCase := range testCases { 69 | t.Run(testCase.name, func(t *testing.T) { 70 | methodName := getGRPCMethodName(testCase.input) 71 | 72 | if methodName != testCase.expectedOutput { 73 | t.Fatalf("expected output: '%s', found: '%s'", testCase.expectedOutput, methodName) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /pkg/utils/sanitize.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "strings" 4 | 5 | // SanitizeString returns a string that does not have white spaces and double quotes. 6 | func SanitizeString(s string) string { 7 | return strings.TrimSpace(strings.Trim(strings.TrimSpace(s), "\"")) 8 | } 9 | -------------------------------------------------------------------------------- /pkg/utils/sanitize_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestSanitizeString(t *testing.T) { 6 | testCases := []struct { 7 | name string 8 | input string 9 | expectedOutput string 10 | }{ 11 | { 12 | name: "With_White_Spaces", 13 | input: " hello ", 14 | expectedOutput: "hello", 15 | }, 16 | { 17 | name: "With_Double_Quotes", 18 | input: "\"hello\"", 19 | expectedOutput: "hello", 20 | }, 21 | { 22 | name: "With_White_Spaces_And_Double_Quotes", 23 | input: " \"hello\" ", 24 | expectedOutput: "hello", 25 | }, 26 | { 27 | name: "With_Double_Quotes_And_White_Spaces", 28 | input: "\" hello \"", 29 | expectedOutput: "hello", 30 | }, 31 | } 32 | 33 | for _, testCase := range testCases { 34 | t.Run(testCase.name, func(t *testing.T) { 35 | sanitizedString := SanitizeString(testCase.input) 36 | 37 | if sanitizedString != testCase.expectedOutput { 38 | t.Fatalf("expected output: '%s', found: '%s'", testCase.expectedOutput, sanitizedString) 39 | } 40 | }) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "runtime" 7 | ) 8 | 9 | var ( 10 | // BuildDate is the date when the binary was built. 11 | BuildDate string 12 | // GitCommit is the commit hash when the binary was built. 13 | GitCommit string 14 | // BuildVersion is the version of the KMS binary. 15 | BuildVersion string 16 | // KMSv1APIVersion is the version of the KMS V1 APIs. 17 | KMSv1APIVersion = "v1beta1" 18 | // KMSv2APIVersion is the version of the KMS V2 APIs. 19 | KMSv2APIVersion = "v2beta1" 20 | // Runtime of the plugin. 21 | Runtime = "Microsoft AzureKMS" 22 | ) 23 | 24 | // PrintVersion prints the current KMS plugin version. 25 | func PrintVersion() (err error) { 26 | pv := struct { 27 | BuildVersion string 28 | GitCommit string 29 | BuildDate string 30 | }{ 31 | BuildDate: BuildDate, 32 | BuildVersion: BuildVersion, 33 | GitCommit: GitCommit, 34 | } 35 | 36 | var res []byte 37 | if res, err = json.Marshal(pv); err != nil { 38 | return 39 | } 40 | 41 | fmt.Printf("%s\n", res) 42 | return 43 | } 44 | 45 | // GetUserAgent returns UserAgent string to append to the agent identifier. 46 | func GetUserAgent() string { 47 | return fmt.Sprintf("k8s-kms-keyvault/%s (%s/%s) %s/%s", BuildVersion, runtime.GOOS, runtime.GOARCH, GitCommit, BuildDate) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/version/version_test.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "runtime" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | func TestPrintVersion(t *testing.T) { 14 | BuildDate = "Now" 15 | BuildVersion = "version" 16 | GitCommit = "hash" 17 | 18 | old := os.Stdout // keep backup of the real stdout 19 | r, w, _ := os.Pipe() 20 | os.Stdout = w 21 | 22 | err := PrintVersion() 23 | 24 | outC := make(chan string) 25 | // copy the output in a separate goroutine so printing can't block indefinitely 26 | go func() { 27 | var buf bytes.Buffer 28 | _, _ = io.Copy(&buf, r) 29 | outC <- strings.TrimSpace(buf.String()) 30 | }() 31 | 32 | // back to normal state 33 | w.Close() 34 | os.Stdout = old // restoring the real stdout 35 | out := <-outC 36 | 37 | if err != nil { 38 | t.Fatalf("expected no error, got %v", err) 39 | } 40 | expected := `{"BuildVersion":"version","GitCommit":"hash","BuildDate":"Now"}` 41 | if !strings.EqualFold(out, expected) { 42 | t.Fatalf("string doesn't match, expected %s, got %s", expected, out) 43 | } 44 | } 45 | 46 | func TestGetUserAgent(t *testing.T) { 47 | BuildDate = "Now" 48 | BuildVersion = "version" 49 | GitCommit = "hash" 50 | 51 | userAgent := GetUserAgent() 52 | expectedUserAgent := fmt.Sprintf("k8s-kms-keyvault/version (%s/%s) hash/Now", runtime.GOOS, runtime.GOARCH) 53 | if !strings.EqualFold(userAgent, expectedUserAgent) { 54 | t.Fatalf("string doesn't match, expected %s, got %s", expectedUserAgent, userAgent) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /scripts/connect-registry.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | if [ "${KIND_NETWORK}" != "bridge" ]; then 8 | # wait for the kind network to exist 9 | for i in $(seq 1 25); do 10 | if docker network ls | grep "${KIND_NETWORK}"; then 11 | break 12 | else 13 | sleep 1 14 | fi 15 | done 16 | containers=$(docker network inspect "${KIND_NETWORK}" -f "{{range .Containers}}{{.Name}} {{end}}") 17 | needs_connect="true" 18 | for c in $containers; do 19 | if [ "$c" = "${REGISTRY_NAME}" ]; then 20 | needs_connect="false" 21 | fi 22 | done 23 | if [ "${needs_connect}" = "true" ]; then 24 | echo "connecting ${KIND_NETWORK} network to ${REGISTRY_NAME}" 25 | docker network connect "${KIND_NETWORK}" "${REGISTRY_NAME}" || true 26 | fi 27 | fi 28 | -------------------------------------------------------------------------------- /scripts/setup-kind-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | export ENCRYPTION_CONFIG_FILE=encryption-config.yaml 8 | envsubst < ./tests/e2e/kind-config.yaml > ./tests/e2e/generated_manifests/kind-config.yaml 9 | 10 | # create a cluster with the local registry enabled in containerd 11 | # add encryption config and the kms static pod manifest with custom image 12 | kind create cluster --retain --image mcr.microsoft.com/mirror/kindest/node:"${KUBERNETES_VERSION}" --name "${KIND_CLUSTER_NAME}" --wait 2m --config=./tests/e2e/generated_manifests/kind-config.yaml 13 | -------------------------------------------------------------------------------- /scripts/setup-kmsv2-kind-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | export ENCRYPTION_CONFIG_FILE=kmsv2-encryption-config.yaml 8 | envsubst < ./tests/e2e/kind-config.yaml > ./tests/e2e/generated_manifests/kind-config.yaml 9 | 10 | # # create a cluster with the local registry enabled in containerd 11 | # # add encryption config and the kms static pod manifest with custom image 12 | kind create cluster --retain --image mcr.microsoft.com/mirror/kindest/node:"${KUBERNETES_VERSION}" --name "${KIND_CLUSTER_NAME}" --wait 2m --config=./tests/e2e/generated_manifests/kind-config.yaml 13 | -------------------------------------------------------------------------------- /scripts/setup-local-registry.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -o errexit 4 | set -o nounset 5 | set -o pipefail 6 | 7 | # create registry container unless it already exists 8 | running="$(docker inspect -f '{{.State.Running}}' "${REGISTRY_NAME}" 2>/dev/null || true)" 9 | if [ "${running}" != 'true' ]; then 10 | echo "Creating local registry" 11 | docker run \ 12 | -d --restart=always -p "${REGISTRY_PORT}:5000" --name "${REGISTRY_NAME}" \ 13 | mirror.gcr.io/registry:2 14 | fi 15 | 16 | # Build and push kms image 17 | export REGISTRY=localhost:${REGISTRY_PORT} 18 | export IMAGE_NAME=keyvault 19 | export IMAGE_VERSION=e2e-$(git rev-parse --short HEAD) 20 | export OUTPUT_TYPE=type=docker 21 | 22 | # push build image to local registry 23 | echo "Build and push image to local registry" 24 | make docker-init-buildx docker-build 25 | docker push "${REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}" 26 | 27 | # generate manifest for local 28 | make e2e-generate-manifests 29 | -------------------------------------------------------------------------------- /tests/client/client_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net" 7 | "testing" 8 | "time" 9 | 10 | "golang.org/x/net/context" 11 | "google.golang.org/grpc" 12 | "google.golang.org/grpc/credentials/insecure" 13 | "k8s.io/apimachinery/pkg/util/uuid" 14 | kmsv1 "k8s.io/kms/apis/v1beta1" 15 | kmsv2 "k8s.io/kms/apis/v2" 16 | ) 17 | 18 | const ( 19 | netProtocol = "unix" 20 | pathToUnixSocket = "/opt/azurekms.sock" 21 | version = "v1beta1" 22 | ) 23 | 24 | var ( 25 | v1Client kmsv1.KeyManagementServiceClient 26 | v2Client kmsv2.KeyManagementServiceClient 27 | connection *grpc.ClientConn 28 | t *testing.T 29 | err error 30 | ) 31 | 32 | func setupTestCase() { 33 | if t != nil { 34 | t.Log("setup test case") 35 | connection, err = newUnixSocketConnection(pathToUnixSocket) 36 | if err != nil { 37 | fmt.Printf("%s", err) 38 | } 39 | 40 | v1Client = kmsv1.NewKeyManagementServiceClient(connection) 41 | v2Client = kmsv2.NewKeyManagementServiceClient(connection) 42 | } 43 | } 44 | 45 | func teardownTestCase() { 46 | if t != nil { 47 | t.Log("teardown test case") 48 | connection.Close() 49 | } 50 | } 51 | 52 | func TestEncryptDecrypt(t *testing.T) { 53 | cases := []struct { 54 | name string 55 | want []byte 56 | expected []byte 57 | }{ 58 | {"text", []byte("secret"), []byte("secret")}, 59 | {"number", []byte("1234"), []byte("1234")}, 60 | {"special", []byte("!@#$%^&*()_"), []byte("!@#$%^&*()_")}, 61 | {"GUID", []byte("b32a58c6-48c1-4552-8ff0-47680f3416d0"), []byte("b32a58c6-48c1-4552-8ff0-47680f3416d0")}, 62 | } 63 | 64 | for _, tc := range cases { 65 | t.Run(tc.name, func(t *testing.T) { 66 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 67 | t.Cleanup(cancel) 68 | 69 | v1EncryptRequest := kmsv1.EncryptRequest{Version: version, Plain: tc.want} 70 | v1EncryptResponse, err := v1Client.Encrypt(ctx, &v1EncryptRequest) 71 | if err != nil { 72 | t.Fatalf("encrypt request for KMS v1 failed with error: %+v", err) 73 | } 74 | 75 | v1DecryptRequest := kmsv1.DecryptRequest{Version: version, Cipher: v1EncryptResponse.Cipher} 76 | v1DecryptResponse, err := v1Client.Decrypt(ctx, &v1DecryptRequest) 77 | if !bytes.Equal(v1DecryptResponse.Plain, tc.want) { 78 | t.Fatalf("Expected secret, but got %s - %v", string(v1DecryptResponse.Plain), err) 79 | } 80 | 81 | uid := "integration-test-" + string(uuid.NewUUID()) 82 | v2EncryptRequest := kmsv2.EncryptRequest{ 83 | Plaintext: tc.want, 84 | Uid: uid, 85 | } 86 | v2EncryptResponse, err := v2Client.Encrypt(ctx, &v2EncryptRequest) 87 | if err != nil { 88 | t.Fatalf("encrypt request for KMS v2 failed with error: %+v", err) 89 | } 90 | if v2EncryptResponse.KeyId == "" { 91 | t.Fatalf("Returned KeyId is empty") 92 | } 93 | 94 | if v2EncryptResponse.Annotations == nil { 95 | t.Fatalf("Returned Annotations is nil") 96 | } 97 | 98 | v2DecryptRequest := kmsv2.DecryptRequest{ 99 | Ciphertext: v2EncryptResponse.Ciphertext, 100 | KeyId: v2EncryptResponse.KeyId, 101 | Uid: uid, 102 | Annotations: v2EncryptResponse.Annotations, 103 | } 104 | v2DecryptResponse, err := v2Client.Decrypt(ctx, &v2DecryptRequest) 105 | if !bytes.Equal(v2DecryptResponse.Plaintext, tc.want) { 106 | t.Fatalf("Expected secret, but got %s - %v", string(v2DecryptResponse.Plaintext), err) 107 | } 108 | }) 109 | } 110 | } 111 | 112 | // Check the KMS provider API version. 113 | // Only matching version is supported now. 114 | func TestV1Version(t *testing.T) { 115 | cases := []struct { 116 | name string 117 | want string 118 | expected string 119 | }{ 120 | {"v1beta1", "v1beta1", "v1beta1"}, 121 | } 122 | 123 | for _, tc := range cases { 124 | t.Run(tc.name, func(t *testing.T) { 125 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 126 | t.Cleanup(cancel) 127 | 128 | request := &kmsv1.VersionRequest{Version: tc.want} 129 | response, err := v1Client.Version(ctx, request) 130 | if err != nil { 131 | t.Fatalf("failed get version from remote KMS provider: %v", err) 132 | } 133 | if response.Version != tc.want { 134 | t.Fatalf("KMS provider api version %s is not supported, only %s is supported now", tc.want, version) 135 | } 136 | }) 137 | } 138 | } 139 | 140 | func TestV2Version(t *testing.T) { 141 | cases := []struct { 142 | name string 143 | want string 144 | expected string 145 | }{ 146 | {"v2beta1", "v2beta1", "v2beta1"}, 147 | } 148 | 149 | for _, tc := range cases { 150 | t.Run(tc.name, func(t *testing.T) { 151 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) 152 | t.Cleanup(cancel) 153 | 154 | request := &kmsv2.StatusRequest{} 155 | response, err := v2Client.Status(ctx, request) 156 | if err != nil { 157 | t.Fatalf("failed get status of remote KMS v2 provider: %v", err) 158 | } 159 | if response.Version != tc.want { 160 | t.Fatalf("KMS v2 provider api version %s is not supported, only %s is supported now", tc.want, version) 161 | } 162 | }) 163 | } 164 | } 165 | 166 | func TestMain(m *testing.M) { 167 | t = &testing.T{} 168 | setupTestCase() 169 | m.Run() 170 | teardownTestCase() 171 | } 172 | 173 | func newUnixSocketConnection(path string) (*grpc.ClientConn, error) { 174 | addr := path 175 | dialer := func(ctx context.Context, addr string) (net.Conn, error) { 176 | return (&net.Dialer{}).DialContext(ctx, netProtocol, addr) 177 | } 178 | connection, err := grpc.Dial( 179 | addr, 180 | grpc.WithTransportCredentials(insecure.NewCredentials()), 181 | grpc.WithContextDialer(dialer)) 182 | if err != nil { 183 | return nil, err 184 | } 185 | return connection, nil 186 | } 187 | -------------------------------------------------------------------------------- /tests/e2e/azure.json: -------------------------------------------------------------------------------- 1 | { 2 | "cloud": "AzurePublicCloud", 3 | "tenantId": "$AZURE_TENANT_ID", 4 | "useManagedIdentityExtension": true, 5 | "userAssignedIdentityID": "$USER_ASSIGNED_IDENTITY_ID" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /tests/e2e/encryption-config.yaml: -------------------------------------------------------------------------------- 1 | kind: EncryptionConfiguration 2 | apiVersion: apiserver.config.k8s.io/v1 3 | resources: 4 | - resources: 5 | - secrets 6 | providers: 7 | - kms: 8 | name: azurekmsprovider 9 | endpoint: unix:///opt/azurekms.socket 10 | cachesize: 1000 11 | - identity: {} 12 | -------------------------------------------------------------------------------- /tests/e2e/helpers.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | assert_success() { 4 | if [[ "$status" != 0 ]]; then 5 | echo "expected: 0" 6 | echo "actual: $status" 7 | echo "output: $output" 8 | return 1 9 | fi 10 | } 11 | 12 | assert_equal() { 13 | if [[ "$1" != "$2" ]]; then 14 | echo "expected: $1" 15 | echo "actual: $2" 16 | return 1 17 | fi 18 | } 19 | 20 | assert_match() { 21 | if [[ ! "$2" =~ $1 ]]; then 22 | echo "expected: $1" 23 | echo "actual: $2" 24 | return 1 25 | fi 26 | } 27 | 28 | wait_for_process() { 29 | wait_time="$1" 30 | sleep_time="$2" 31 | cmd="$3" 32 | while [ "$wait_time" -gt 0 ]; do 33 | if eval "$cmd"; then 34 | return 0 35 | else 36 | sleep "$sleep_time" 37 | wait_time=$((wait_time - sleep_time)) 38 | fi 39 | done 40 | return 1 41 | } 42 | -------------------------------------------------------------------------------- /tests/e2e/kind-config.yaml: -------------------------------------------------------------------------------- 1 | kind: Cluster 2 | apiVersion: kind.x-k8s.io/v1alpha4 3 | containerdConfigPatches: 4 | - |- 5 | [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${REGISTRY_PORT}"] 6 | endpoint = ["http://${REGISTRY_NAME}:${REGISTRY_PORT}"] 7 | nodes: 8 | - role: control-plane 9 | extraMounts: 10 | - containerPath: /etc/kubernetes/${ENCRYPTION_CONFIG_FILE} 11 | hostPath: tests/e2e/${ENCRYPTION_CONFIG_FILE} 12 | readOnly: true 13 | propagation: None 14 | - containerPath: /etc/kubernetes/manifests/kubernetes-kms.yaml 15 | hostPath: tests/e2e/generated_manifests/kms.yaml 16 | readOnly: true 17 | propagation: None 18 | - containerPath: /etc/kubernetes/azure.json 19 | hostPath: tests/e2e/generated_manifests/azure.json 20 | readOnly: true 21 | propagation: None 22 | kubeadmConfigPatches: 23 | - | 24 | kind: ClusterConfiguration 25 | apiServer: 26 | extraArgs: 27 | encryption-provider-config: "/etc/kubernetes/${ENCRYPTION_CONFIG_FILE}" 28 | feature-gates: "KMSv1=true" 29 | extraVolumes: 30 | - name: encryption-config 31 | hostPath: "/etc/kubernetes/${ENCRYPTION_CONFIG_FILE}" 32 | mountPath: "/etc/kubernetes/${ENCRYPTION_CONFIG_FILE}" 33 | readOnly: true 34 | pathType: File 35 | - name: sock-path 36 | hostPath: "/opt" 37 | mountPath: "/opt" 38 | -------------------------------------------------------------------------------- /tests/e2e/kms.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: azure-kms-provider 5 | namespace: kube-system 6 | labels: 7 | tier: control-plane 8 | component: azure-kms-provider 9 | spec: 10 | priorityClassName: system-node-critical 11 | hostNetwork: true 12 | containers: 13 | - name: azure-kms-provider 14 | image: ${REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION} 15 | imagePullPolicy: IfNotPresent 16 | args: 17 | - --keyvault-name=${KEYVAULT_NAME} 18 | - --key-name=${KEY_NAME} 19 | - --key-version=${KEY_VERSION} 20 | - --managed-hsm=false 21 | - -v=5 22 | env: 23 | # setting this env var so we get debug logs in SDK from CI runs 24 | - name: AZURE_GO_SDK_LOG_LEVEL 25 | value: DEBUG 26 | securityContext: 27 | allowPrivilegeEscalation: false 28 | capabilities: 29 | drop: 30 | - ALL 31 | readOnlyRootFilesystem: true 32 | runAsUser: 0 33 | ports: 34 | - containerPort: 8787 35 | protocol: TCP 36 | livenessProbe: 37 | httpGet: 38 | path: /healthz 39 | port: 8787 40 | failureThreshold: 2 41 | periodSeconds: 10 42 | resources: 43 | requests: 44 | cpu: 100m 45 | memory: 128Mi 46 | limits: 47 | cpu: "4" 48 | memory: 2Gi 49 | volumeMounts: 50 | - name: etc-kubernetes 51 | mountPath: /etc/kubernetes 52 | - name: etc-ssl 53 | mountPath: /etc/ssl 54 | readOnly: true 55 | - name: sock 56 | mountPath: /opt 57 | volumes: 58 | - name: etc-kubernetes 59 | hostPath: 60 | path: /etc/kubernetes 61 | - name: etc-ssl 62 | hostPath: 63 | path: /etc/ssl 64 | - name: sock 65 | hostPath: 66 | path: /opt 67 | nodeSelector: 68 | kubernetes.io/os: linux 69 | -------------------------------------------------------------------------------- /tests/e2e/kmsv2-encryption-config.yaml: -------------------------------------------------------------------------------- 1 | kind: EncryptionConfiguration 2 | apiVersion: apiserver.config.k8s.io/v1 3 | resources: 4 | - resources: 5 | - secrets 6 | providers: 7 | - kms: 8 | apiVersion: v2 9 | name: azurekmsprovider 10 | endpoint: unix:///opt/azurekms.socket 11 | -------------------------------------------------------------------------------- /tests/e2e/test.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load helpers 4 | 5 | WAIT_TIME=120 6 | SLEEP_TIME=1 7 | 8 | if [[ ${IS_SOAK_TEST} = true ]]; then 9 | export ETCD_CA_CERT=/etc/kubernetes/certs/ca.crt 10 | export ETCD_CERT=/etc/kubernetes/certs/etcdclient.crt 11 | export ETCD_KEY=/etc/kubernetes/certs/etcdclient.key 12 | else 13 | export ETCD_CA_CERT=/etc/kubernetes/pki/etcd/ca.crt 14 | export ETCD_CERT=/etc/kubernetes/pki/etcd/server.crt 15 | export ETCD_KEY=/etc/kubernetes/pki/etcd/server.key 16 | fi 17 | 18 | @test "azure keyvault kms plugin is running" { 19 | wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl -n kube-system wait --for=condition=Ready --timeout=60s pod -l component=azure-kms-provider" 20 | } 21 | 22 | @test "creating secret resource" { 23 | run kubectl create secret generic secret1 -n default --from-literal=foo=bar 24 | assert_success 25 | } 26 | 27 | @test "read the secret resource test" { 28 | result=$(kubectl get secret secret1 -o jsonpath='{.data.foo}' | base64 -d) 29 | [[ "${result//$'\r'}" == "bar" ]] 30 | } 31 | 32 | @test "check if secret is encrypted in etcd" { 33 | if [ ${IS_SOAK_TEST} = true ]; then 34 | local node_name=$(kubectl get nodes -l kubernetes.azure.com/role=master -o jsonpath="{.items[0].metadata.name}") 35 | run kubectl node-shell ${node_name} -- sh -c "ETCDCTL_API=3 etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1" 36 | assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}" 37 | assert_success 38 | else 39 | local pod_name=$(kubectl get pod -n kube-system -l component=etcd -o jsonpath="{.items[0].metadata.name}") 40 | run kubectl exec ${pod_name} -n kube-system -- etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1 41 | assert_match "k8s:enc:kms:v1:azurekmsprovider" "${output}" 42 | assert_success 43 | fi 44 | } 45 | 46 | @test "check if metrics endpoint works" { 47 | local curl_pod_name=curl-$(openssl rand -hex 5) 48 | kubectl run ${curl_pod_name} --image=curlimages/curl:7.75.0 --labels="test=metrics_test" -- tail -f /dev/null 49 | kubectl wait --for=condition=Ready --timeout=60s pod ${curl_pod_name} 50 | 51 | local pod_ip=$(kubectl get pod -n kube-system -l component=azure-kms-provider -o jsonpath="{.items[0].status.podIP}") 52 | run kubectl exec ${curl_pod_name} -- curl http://${pod_ip}:8095/metrics 53 | assert_match "kms_request_bucket" "${output}" 54 | assert_success 55 | } 56 | 57 | @test "check healthz for kms plugin" { 58 | local curl_pod_name=curl-$(openssl rand -hex 5) 59 | kubectl run ${curl_pod_name} --image=curlimages/curl:7.75.0 --labels="test=healthz_test" -- tail -f /dev/null 60 | kubectl wait --for=condition=Ready --timeout=60s pod ${curl_pod_name} 61 | 62 | local pod_ip=$(kubectl get pod -n kube-system -l component=azure-kms-provider -o jsonpath="{.items[0].status.podIP}") 63 | result=$(kubectl exec ${curl_pod_name} -- curl http://${pod_ip}:8787/healthz) 64 | [[ "${result//$'\r'}" == "ok" ]] 65 | 66 | result=$(kubectl exec ${curl_pod_name} -- curl http://${pod_ip}:8787/healthz -o /dev/null -w '%{http_code}\n' -s) 67 | [[ "${result//$'\r'}" == "200" ]] 68 | } 69 | 70 | teardown_file() { 71 | # cleanup 72 | run kubectl delete secret secret1 -n default 73 | 74 | run kubectl delete pod -l test=metrics_test --force --grace-period 0 75 | run kubectl delete pod -l test=healthz_test --force --grace-period 0 76 | } 77 | -------------------------------------------------------------------------------- /tests/e2e/testkmsv2.bats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bats 2 | 3 | load helpers 4 | 5 | WAIT_TIME=120 6 | SLEEP_TIME=1 7 | 8 | export ETCD_CA_CERT=/etc/kubernetes/pki/etcd/ca.crt 9 | export ETCD_CERT=/etc/kubernetes/pki/etcd/server.crt 10 | export ETCD_KEY=/etc/kubernetes/pki/etcd/server.key 11 | 12 | setup() { 13 | # get the initial number of encrypted count 14 | local metrics=$(kubectl get --raw /metrics) 15 | expected_encyption_count=$(echo "${metrics}" | grep -oP 'apiserver_envelope_encryption_key_id_hash_total\{[^\}]*transformation_type="to_storage"[^\}]*\}\s+\K\d+') 16 | } 17 | 18 | @test "azure keyvault kms plugin is running" { 19 | wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl -n kube-system wait --for=condition=Ready --timeout=60s pod -l component=azure-kms-provider" 20 | } 21 | 22 | @test "creating secret resource" { 23 | run kubectl create secret generic secret1 -n default --from-literal=foo=bar 24 | let "expected_encyption_count++" 25 | assert_success 26 | } 27 | 28 | @test "read the secret resource test" { 29 | result=$(kubectl get secret secret1 -o jsonpath='{.data.foo}' | base64 -d) 30 | [[ "${result//$'\r'}" == "bar" ]] 31 | } 32 | 33 | @test "check if secret is encrypted in etcd" { 34 | local pod_name=$(kubectl get pod -n kube-system -l component=etcd -o jsonpath="{.items[0].metadata.name}") 35 | run kubectl exec ${pod_name} -n kube-system -- etcdctl --cacert=${ETCD_CA_CERT} --cert=${ETCD_CERT} --key=${ETCD_KEY} get /registry/secrets/default/secret1 36 | assert_match "k8s:enc:kms:v2:azurekmsprovider" "${output}" 37 | assert_success 38 | } 39 | 40 | @test "check encryption count" { 41 | # The expected_encryption_count value is set in the setup(). 42 | local metrics=$(kubectl get --raw /metrics) 43 | encyption_count=$(echo "${metrics}" | grep -oP 'apiserver_envelope_encryption_key_id_hash_total\{[^\}]*transformation_type="to_storage"[^\}]*\}\s+\K\d+') 44 | [[ "${encyption_count}" == "${expected_encyption_count}" ]] 45 | } 46 | 47 | @test "check keyID hash used for encrypt/decrypt" { 48 | # expected_hash value is computed based on the key used in CI. 49 | # this needs to be updated when we rotate that key. 50 | local expected_hash="sha256:cbda52be2f8c13d323a3b17c4679118a60b91d29454305e02ee485185b6e386f" 51 | local metrics=$(kubectl get --raw /metrics) 52 | got_hash_id=$(echo "${metrics}" | grep 'apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds' | sed -n 's/.*key_id_hash="\([^"]*\)".*/\1/p' | sort -u) 53 | [[ "${got_hash_id}" == "${expected_hash}" ]] 54 | } 55 | 56 | @test "check if metrics endpoint works" { 57 | local curl_pod_name=curl-$(openssl rand -hex 5) 58 | kubectl run ${curl_pod_name} --image=curlimages/curl:7.75.0 --labels="test=metrics_test" -- tail -f /dev/null 59 | kubectl wait --for=condition=Ready --timeout=60s pod ${curl_pod_name} 60 | 61 | local pod_ip=$(kubectl get pod -n kube-system -l component=azure-kms-provider -o jsonpath="{.items[0].status.podIP}") 62 | run kubectl exec ${curl_pod_name} -- curl http://${pod_ip}:8095/metrics 63 | assert_match "kms_request_bucket" "${output}" 64 | assert_success 65 | } 66 | 67 | @test "check healthz for kms plugin" { 68 | local curl_pod_name=curl-$(openssl rand -hex 5) 69 | kubectl run ${curl_pod_name} --image=curlimages/curl:7.75.0 --labels="test=healthz_test" -- tail -f /dev/null 70 | kubectl wait --for=condition=Ready --timeout=60s pod ${curl_pod_name} 71 | 72 | local pod_ip=$(kubectl get pod -n kube-system -l component=azure-kms-provider -o jsonpath="{.items[0].status.podIP}") 73 | result=$(kubectl exec ${curl_pod_name} -- curl http://${pod_ip}:8787/healthz) 74 | [[ "${result//$'\r'}" == "ok" ]] 75 | 76 | result=$(kubectl exec ${curl_pod_name} -- curl http://${pod_ip}:8787/healthz -o /dev/null -w '%{http_code}\n' -s) 77 | [[ "${result//$'\r'}" == "200" ]] 78 | } 79 | 80 | teardown_file() { 81 | # cleanup 82 | run kubectl delete secret secret1 -n default 83 | 84 | run kubectl delete pod -l test=metrics_test --force --grace-period 0 85 | run kubectl delete pod -l test=healthz_test --force --grace-period 0 86 | } 87 | -------------------------------------------------------------------------------- /tools/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Azure/kubernetes-kms/tools 2 | 3 | go 1.23.8 4 | 5 | require github.com/golangci/golangci-lint v1.63.4 6 | 7 | require ( 8 | 4d63.com/gocheckcompilerdirectives v1.2.1 // indirect 9 | 4d63.com/gochecknoglobals v0.2.1 // indirect 10 | github.com/4meepo/tagalign v1.4.1 // indirect 11 | github.com/Abirdcfly/dupword v0.1.3 // indirect 12 | github.com/Antonboom/errname v1.0.0 // indirect 13 | github.com/Antonboom/nilnil v1.0.1 // indirect 14 | github.com/Antonboom/testifylint v1.5.2 // indirect 15 | github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect 16 | github.com/Crocmagnon/fatcontext v0.5.3 // indirect 17 | github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect 18 | github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 // indirect 19 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 20 | github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect 21 | github.com/alecthomas/go-check-sumtype v0.3.1 // indirect 22 | github.com/alexkohler/nakedret/v2 v2.0.5 // indirect 23 | github.com/alexkohler/prealloc v1.0.0 // indirect 24 | github.com/alingse/asasalint v0.0.11 // indirect 25 | github.com/alingse/nilnesserr v0.1.1 // indirect 26 | github.com/ashanbrown/forbidigo v1.6.0 // indirect 27 | github.com/ashanbrown/makezero v1.2.0 // indirect 28 | github.com/beorn7/perks v1.0.1 // indirect 29 | github.com/bkielbasa/cyclop v1.2.3 // indirect 30 | github.com/blizzy78/varnamelen v0.8.0 // indirect 31 | github.com/bombsimon/wsl/v4 v4.5.0 // indirect 32 | github.com/breml/bidichk v0.3.2 // indirect 33 | github.com/breml/errchkjson v0.4.0 // indirect 34 | github.com/butuzov/ireturn v0.3.1 // indirect 35 | github.com/butuzov/mirror v1.3.0 // indirect 36 | github.com/catenacyber/perfsprint v0.7.1 // indirect 37 | github.com/ccojocar/zxcvbn-go v1.0.2 // indirect 38 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 39 | github.com/charithe/durationcheck v0.0.10 // indirect 40 | github.com/chavacava/garif v0.1.0 // indirect 41 | github.com/ckaznocha/intrange v0.3.0 // indirect 42 | github.com/curioswitch/go-reassign v0.3.0 // indirect 43 | github.com/daixiang0/gci v0.13.5 // indirect 44 | github.com/davecgh/go-spew v1.1.1 // indirect 45 | github.com/denis-tingaikin/go-header v0.5.0 // indirect 46 | github.com/ettle/strcase v0.2.0 // indirect 47 | github.com/fatih/color v1.18.0 // indirect 48 | github.com/fatih/structtag v1.2.0 // indirect 49 | github.com/firefart/nonamedreturns v1.0.5 // indirect 50 | github.com/fsnotify/fsnotify v1.5.4 // indirect 51 | github.com/fzipp/gocyclo v0.6.0 // indirect 52 | github.com/ghostiam/protogetter v0.3.8 // indirect 53 | github.com/go-critic/go-critic v0.11.5 // indirect 54 | github.com/go-toolsmith/astcast v1.1.0 // indirect 55 | github.com/go-toolsmith/astcopy v1.1.0 // indirect 56 | github.com/go-toolsmith/astequal v1.2.0 // indirect 57 | github.com/go-toolsmith/astfmt v1.1.0 // indirect 58 | github.com/go-toolsmith/astp v1.1.0 // indirect 59 | github.com/go-toolsmith/strparse v1.1.0 // indirect 60 | github.com/go-toolsmith/typep v1.1.0 // indirect 61 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 62 | github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect 63 | github.com/gobwas/glob v0.2.3 // indirect 64 | github.com/gofrs/flock v0.12.1 // indirect 65 | github.com/golang/protobuf v1.5.3 // indirect 66 | github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect 67 | github.com/golangci/go-printf-func-name v0.1.0 // indirect 68 | github.com/golangci/gofmt v0.0.0-20241223200906-057b0627d9b9 // indirect 69 | github.com/golangci/misspell v0.6.0 // indirect 70 | github.com/golangci/plugin-module-register v0.1.1 // indirect 71 | github.com/golangci/revgrep v0.5.3 // indirect 72 | github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed // indirect 73 | github.com/google/go-cmp v0.6.0 // indirect 74 | github.com/gordonklaus/ineffassign v0.1.0 // indirect 75 | github.com/gostaticanalysis/analysisutil v0.7.1 // indirect 76 | github.com/gostaticanalysis/comment v1.4.2 // indirect 77 | github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect 78 | github.com/gostaticanalysis/nilerr v0.1.1 // indirect 79 | github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect 80 | github.com/hashicorp/go-version v1.7.0 // indirect 81 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 82 | github.com/hashicorp/hcl v1.0.0 // indirect 83 | github.com/hexops/gotextdiff v1.0.3 // indirect 84 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 85 | github.com/jgautheron/goconst v1.7.1 // indirect 86 | github.com/jingyugao/rowserrcheck v1.1.1 // indirect 87 | github.com/jjti/go-spancheck v0.6.4 // indirect 88 | github.com/julz/importas v0.2.0 // indirect 89 | github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect 90 | github.com/kisielk/errcheck v1.8.0 // indirect 91 | github.com/kkHAIKE/contextcheck v1.1.5 // indirect 92 | github.com/kulti/thelper v0.6.3 // indirect 93 | github.com/kunwardeep/paralleltest v1.0.10 // indirect 94 | github.com/kyoh86/exportloopref v0.1.11 // indirect 95 | github.com/lasiar/canonicalheader v1.1.2 // indirect 96 | github.com/ldez/exptostd v0.3.1 // indirect 97 | github.com/ldez/gomoddirectives v0.6.0 // indirect 98 | github.com/ldez/grignotin v0.7.0 // indirect 99 | github.com/ldez/tagliatelle v0.7.1 // indirect 100 | github.com/ldez/usetesting v0.4.2 // indirect 101 | github.com/leonklingele/grouper v1.1.2 // indirect 102 | github.com/macabu/inamedparam v0.1.3 // indirect 103 | github.com/magiconair/properties v1.8.6 // indirect 104 | github.com/maratori/testableexamples v1.0.0 // indirect 105 | github.com/maratori/testpackage v1.1.1 // indirect 106 | github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 // indirect 107 | github.com/mattn/go-colorable v0.1.13 // indirect 108 | github.com/mattn/go-isatty v0.0.20 // indirect 109 | github.com/mattn/go-runewidth v0.0.16 // indirect 110 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 111 | github.com/mgechev/revive v1.5.1 // indirect 112 | github.com/mitchellh/go-homedir v1.1.0 // indirect 113 | github.com/mitchellh/mapstructure v1.5.0 // indirect 114 | github.com/moricho/tparallel v0.3.2 // indirect 115 | github.com/nakabonne/nestif v0.3.1 // indirect 116 | github.com/nishanths/exhaustive v0.12.0 // indirect 117 | github.com/nishanths/predeclared v0.2.2 // indirect 118 | github.com/nunnatsa/ginkgolinter v0.18.4 // indirect 119 | github.com/olekukonko/tablewriter v0.0.5 // indirect 120 | github.com/pelletier/go-toml v1.9.5 // indirect 121 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 122 | github.com/pmezard/go-difflib v1.0.0 // indirect 123 | github.com/polyfloyd/go-errorlint v1.7.0 // indirect 124 | github.com/prometheus/client_golang v1.12.1 // indirect 125 | github.com/prometheus/client_model v0.2.0 // indirect 126 | github.com/prometheus/common v0.32.1 // indirect 127 | github.com/prometheus/procfs v0.7.3 // indirect 128 | github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 // indirect 129 | github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect 130 | github.com/quasilyte/gogrep v0.5.0 // indirect 131 | github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect 132 | github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect 133 | github.com/raeperd/recvcheck v0.2.0 // indirect 134 | github.com/rivo/uniseg v0.4.7 // indirect 135 | github.com/rogpeppe/go-internal v1.13.1 // indirect 136 | github.com/ryancurrah/gomodguard v1.3.5 // indirect 137 | github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect 138 | github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect 139 | github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect 140 | github.com/sashamelentyev/interfacebloat v1.1.0 // indirect 141 | github.com/sashamelentyev/usestdlibvars v1.28.0 // indirect 142 | github.com/securego/gosec/v2 v2.21.4 // indirect 143 | github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect 144 | github.com/sirupsen/logrus v1.9.3 // indirect 145 | github.com/sivchari/containedctx v1.0.3 // indirect 146 | github.com/sivchari/tenv v1.12.1 // indirect 147 | github.com/sonatard/noctx v0.1.0 // indirect 148 | github.com/sourcegraph/go-diff v0.7.0 // indirect 149 | github.com/spf13/afero v1.11.0 // indirect 150 | github.com/spf13/cast v1.5.0 // indirect 151 | github.com/spf13/cobra v1.8.1 // indirect 152 | github.com/spf13/jwalterweatherman v1.1.0 // indirect 153 | github.com/spf13/pflag v1.0.5 // indirect 154 | github.com/spf13/viper v1.12.0 // indirect 155 | github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect 156 | github.com/stbenjam/no-sprintf-host-port v0.2.0 // indirect 157 | github.com/stretchr/objx v0.5.2 // indirect 158 | github.com/stretchr/testify v1.10.0 // indirect 159 | github.com/subosito/gotenv v1.4.1 // indirect 160 | github.com/tdakkota/asciicheck v0.3.0 // indirect 161 | github.com/tetafro/godot v1.4.20 // indirect 162 | github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3 // indirect 163 | github.com/timonwong/loggercheck v0.10.1 // indirect 164 | github.com/tomarrell/wrapcheck/v2 v2.10.0 // indirect 165 | github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect 166 | github.com/ultraware/funlen v0.2.0 // indirect 167 | github.com/ultraware/whitespace v0.2.0 // indirect 168 | github.com/uudashr/gocognit v1.2.0 // indirect 169 | github.com/uudashr/iface v1.3.0 // indirect 170 | github.com/xen0n/gosmopolitan v1.2.2 // indirect 171 | github.com/yagipy/maintidx v1.0.0 // indirect 172 | github.com/yeya24/promlinter v0.3.0 // indirect 173 | github.com/ykadowak/zerologlint v0.1.5 // indirect 174 | gitlab.com/bosi/decorder v0.4.2 // indirect 175 | go-simpler.org/musttag v0.13.0 // indirect 176 | go-simpler.org/sloglint v0.7.2 // indirect 177 | go.uber.org/atomic v1.7.0 // indirect 178 | go.uber.org/automaxprocs v1.6.0 // indirect 179 | go.uber.org/multierr v1.6.0 // indirect 180 | go.uber.org/zap v1.24.0 // indirect 181 | golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect 182 | golang.org/x/exp/typeparams v0.0.0-20241108190413-2d47ceb2692f // indirect 183 | golang.org/x/mod v0.22.0 // indirect 184 | golang.org/x/sync v0.10.0 // indirect 185 | golang.org/x/sys v0.28.0 // indirect 186 | golang.org/x/text v0.20.0 // indirect 187 | golang.org/x/tools v0.28.0 // indirect 188 | google.golang.org/protobuf v1.34.2 // indirect 189 | gopkg.in/ini.v1 v1.67.0 // indirect 190 | gopkg.in/yaml.v2 v2.4.0 // indirect 191 | gopkg.in/yaml.v3 v3.0.1 // indirect 192 | honnef.co/go/tools v0.5.1 // indirect 193 | mvdan.cc/gofumpt v0.7.0 // indirect 194 | mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect 195 | ) 196 | -------------------------------------------------------------------------------- /tools/tools.go: -------------------------------------------------------------------------------- 1 | //go:build tools 2 | // +build tools 3 | 4 | package tools 5 | 6 | import ( 7 | _ "github.com/golangci/golangci-lint/cmd/golangci-lint" 8 | ) 9 | --------------------------------------------------------------------------------